log SEB Client ping threshold overflows as Error log
This commit is contained in:
		
							parent
							
								
									3fee5dc0a6
								
							
						
					
					
						commit
						b27c79e1d8
					
				
					 8 changed files with 148 additions and 7 deletions
				
			
		|  | @ -175,7 +175,7 @@ public final class Indicator implements Entity { | ||||||
|         } |         } | ||||||
|     } |     } | ||||||
| 
 | 
 | ||||||
|     public static final class Threshold { |     public static final class Threshold implements Comparable<Threshold> { | ||||||
| 
 | 
 | ||||||
|         @JsonProperty(THRESHOLD.ATTR_VALUE) |         @JsonProperty(THRESHOLD.ATTR_VALUE) | ||||||
|         @NotNull |         @NotNull | ||||||
|  | @ -206,6 +206,11 @@ public final class Indicator implements Entity { | ||||||
|             return "Threshold [value=" + this.value + ", color=" + this.color + "]"; |             return "Threshold [value=" + this.value + ", color=" + this.color + "]"; | ||||||
|         } |         } | ||||||
| 
 | 
 | ||||||
|  |         @Override | ||||||
|  |         public int compareTo(final Threshold o) { | ||||||
|  |             return Double.compare(this.value, (o != null) ? o.value : 0); | ||||||
|  |         } | ||||||
|  | 
 | ||||||
|     } |     } | ||||||
| 
 | 
 | ||||||
| } | } | ||||||
|  |  | ||||||
|  | @ -71,6 +71,11 @@ public interface ExamDAO extends ActivatableEntityDAO<Exam, Exam>, BulkActionSup | ||||||
|      * @return Result refer to a collection of exams or to an error if happened */ |      * @return Result refer to a collection of exams or to an error if happened */ | ||||||
|     Result<Collection<Exam>> allForEndCheck(); |     Result<Collection<Exam>> allForEndCheck(); | ||||||
| 
 | 
 | ||||||
|  |     /** Get a collection of all currently running exam identifiers | ||||||
|  |      * | ||||||
|  |      * @return collection of all currently running exam identifiers */ | ||||||
|  |     Result<Collection<Long>> allRunningExamIds(); | ||||||
|  | 
 | ||||||
|     /** This is used to place an internal (write)lock for the specified exam. |     /** This is used to place an internal (write)lock for the specified exam. | ||||||
|      * The exam will be marked as locked on the persistence level to prevent other running web-service instances |      * The exam will be marked as locked on the persistence level to prevent other running web-service instances | ||||||
|      * to write concurrently to the specified exam while it is been updated by an internal batch process. |      * to write concurrently to the specified exam while it is been updated by an internal batch process. | ||||||
|  |  | ||||||
|  | @ -353,6 +353,26 @@ public class ExamDAOImpl implements ExamDAO { | ||||||
|                 .execute()); |                 .execute()); | ||||||
|     } |     } | ||||||
| 
 | 
 | ||||||
|  |     @Override | ||||||
|  |     @Transactional(readOnly = true) | ||||||
|  |     public Result<Collection<Long>> allRunningExamIds() { | ||||||
|  |         return Result.tryCatch(() -> { | ||||||
|  |             return this.examRecordMapper.selectIdsByExample() | ||||||
|  |                     .where( | ||||||
|  |                             ExamRecordDynamicSqlSupport.active, | ||||||
|  |                             isEqualTo(BooleanUtils.toInteger(true))) | ||||||
|  |                     .and( | ||||||
|  |                             ExamRecordDynamicSqlSupport.status, | ||||||
|  |                             isEqualTo(ExamStatus.RUNNING.name())) | ||||||
|  |                     .and( | ||||||
|  |                             ExamRecordDynamicSqlSupport.updating, | ||||||
|  |                             isEqualTo(BooleanUtils.toInteger(false))) | ||||||
|  | 
 | ||||||
|  |                     .build() | ||||||
|  |                     .execute(); | ||||||
|  |         }); | ||||||
|  |     } | ||||||
|  | 
 | ||||||
|     @Override |     @Override | ||||||
|     @Transactional(readOnly = true) |     @Transactional(readOnly = true) | ||||||
|     public Result<Collection<Exam>> allForRunCheck() { |     public Result<Collection<Exam>> allForRunCheck() { | ||||||
|  |  | ||||||
|  | @ -103,6 +103,8 @@ public interface SebClientConnectionService { | ||||||
|             Long institutionId, |             Long institutionId, | ||||||
|             String clientAddress); |             String clientAddress); | ||||||
| 
 | 
 | ||||||
|  |     void updatePingEvents(); | ||||||
|  | 
 | ||||||
|     /** Notify a ping for a certain client connection. |     /** Notify a ping for a certain client connection. | ||||||
|      * |      * | ||||||
|      * @param connectionToken the connection token |      * @param connectionToken the connection token | ||||||
|  |  | ||||||
|  | @ -23,6 +23,7 @@ import com.fasterxml.jackson.annotation.JsonIgnore; | ||||||
| import ch.ethz.seb.sebserver.gbl.model.session.ClientEvent.EventType; | import ch.ethz.seb.sebserver.gbl.model.session.ClientEvent.EventType; | ||||||
| import ch.ethz.seb.sebserver.webservice.datalayer.batis.ClientEventExtentionMapper; | import ch.ethz.seb.sebserver.webservice.datalayer.batis.ClientEventExtentionMapper; | ||||||
| import ch.ethz.seb.sebserver.webservice.datalayer.batis.mapper.ClientEventRecordDynamicSqlSupport; | import ch.ethz.seb.sebserver.webservice.datalayer.batis.mapper.ClientEventRecordDynamicSqlSupport; | ||||||
|  | import ch.ethz.seb.sebserver.webservice.datalayer.batis.model.ClientEventRecord; | ||||||
| 
 | 
 | ||||||
| public abstract class AbstractPingIndicator extends AbstractClientIndicator { | public abstract class AbstractPingIndicator extends AbstractClientIndicator { | ||||||
| 
 | 
 | ||||||
|  | @ -84,4 +85,6 @@ public abstract class AbstractPingIndicator extends AbstractClientIndicator { | ||||||
|         return this.pingNumber; |         return this.pingNumber; | ||||||
|     } |     } | ||||||
| 
 | 
 | ||||||
|  |     public abstract ClientEventRecord updateLogEvent(); | ||||||
|  | 
 | ||||||
| } | } | ||||||
|  |  | ||||||
|  | @ -20,8 +20,10 @@ import org.springframework.scheduling.annotation.Async; | ||||||
| import org.springframework.scheduling.annotation.Scheduled; | import org.springframework.scheduling.annotation.Scheduled; | ||||||
| import org.springframework.stereotype.Service; | import org.springframework.stereotype.Service; | ||||||
| 
 | 
 | ||||||
|  | import ch.ethz.seb.sebserver.gbl.Constants; | ||||||
| import ch.ethz.seb.sebserver.gbl.model.exam.Exam; | import ch.ethz.seb.sebserver.gbl.model.exam.Exam; | ||||||
| import ch.ethz.seb.sebserver.webservice.servicelayer.dao.ExamDAO; | import ch.ethz.seb.sebserver.webservice.servicelayer.dao.ExamDAO; | ||||||
|  | import ch.ethz.seb.sebserver.webservice.servicelayer.session.SebClientConnectionService; | ||||||
| 
 | 
 | ||||||
| @Service | @Service | ||||||
| class ExamSessionControlTask { | class ExamSessionControlTask { | ||||||
|  | @ -29,6 +31,7 @@ class ExamSessionControlTask { | ||||||
|     private static final Logger log = LoggerFactory.getLogger(ExamSessionControlTask.class); |     private static final Logger log = LoggerFactory.getLogger(ExamSessionControlTask.class); | ||||||
| 
 | 
 | ||||||
|     private final ExamDAO examDAO; |     private final ExamDAO examDAO; | ||||||
|  |     private final SebClientConnectionService sebClientConnectionService; | ||||||
|     private final ExamUpdateHandler examUpdateHandler; |     private final ExamUpdateHandler examUpdateHandler; | ||||||
|     private final Long examTimePrefix; |     private final Long examTimePrefix; | ||||||
|     private final Long examTimeSuffix; |     private final Long examTimeSuffix; | ||||||
|  | @ -46,20 +49,21 @@ class ExamSessionControlTask { | ||||||
| 
 | 
 | ||||||
|     protected ExamSessionControlTask( |     protected ExamSessionControlTask( | ||||||
|             final ExamDAO examDAO, |             final ExamDAO examDAO, | ||||||
|  |             final SebClientConnectionService sebClientConnectionService, | ||||||
|             final ExamUpdateHandler examUpdateHandler, |             final ExamUpdateHandler examUpdateHandler, | ||||||
|             @Value("${sebserver.webservice.api.exam.time-prefix:3600000}") final Long examTimePrefix, |             @Value("${sebserver.webservice.api.exam.time-prefix:3600000}") final Long examTimePrefix, | ||||||
|             @Value("${sebserver.webservice.api.exam.time-suffix:3600000}") final Long examTimeSuffix) { |             @Value("${sebserver.webservice.api.exam.time-suffix:3600000}") final Long examTimeSuffix) { | ||||||
| 
 | 
 | ||||||
|         this.examDAO = examDAO; |         this.examDAO = examDAO; | ||||||
|  |         this.sebClientConnectionService = sebClientConnectionService; | ||||||
|         this.examUpdateHandler = examUpdateHandler; |         this.examUpdateHandler = examUpdateHandler; | ||||||
|         this.examTimePrefix = examTimePrefix; |         this.examTimePrefix = examTimePrefix; | ||||||
|         this.examTimeSuffix = examTimeSuffix; |         this.examTimeSuffix = examTimeSuffix; | ||||||
| 
 |  | ||||||
|     } |     } | ||||||
| 
 | 
 | ||||||
|     @Async |     @Async | ||||||
|     @Scheduled(cron = "${sebserver.webservice.api.exam.update-interval:1 * * * * *}") |     @Scheduled(cron = "${sebserver.webservice.api.exam.update-interval:1 * * * * *}") | ||||||
|     public void execTask() { |     public void examRunUpdateTask() { | ||||||
| 
 | 
 | ||||||
|         final String updateId = this.examUpdateHandler.createUpdateId(); |         final String updateId = this.examUpdateHandler.createUpdateId(); | ||||||
| 
 | 
 | ||||||
|  | @ -67,11 +71,17 @@ class ExamSessionControlTask { | ||||||
|             log.debug("Run exam runtime update task with Id: {}", updateId); |             log.debug("Run exam runtime update task with Id: {}", updateId); | ||||||
|         } |         } | ||||||
| 
 | 
 | ||||||
|         controlStart(updateId); |         controlExamStart(updateId); | ||||||
|         controlEnd(updateId); |         controlExamEnd(updateId); | ||||||
|     } |     } | ||||||
| 
 | 
 | ||||||
|     private void controlStart(final String updateId) { |     @Async | ||||||
|  |     @Scheduled(fixedRate = Constants.SECOND_IN_MILLIS) | ||||||
|  |     public void pingEventUpdateTask() { | ||||||
|  |         this.sebClientConnectionService.updatePingEvents(); | ||||||
|  |     } | ||||||
|  | 
 | ||||||
|  |     private void controlExamStart(final String updateId) { | ||||||
|         if (log.isDebugEnabled()) { |         if (log.isDebugEnabled()) { | ||||||
|             log.debug("Check starting exams: {}", updateId); |             log.debug("Check starting exams: {}", updateId); | ||||||
|         } |         } | ||||||
|  | @ -95,7 +105,7 @@ class ExamSessionControlTask { | ||||||
|         } |         } | ||||||
|     } |     } | ||||||
| 
 | 
 | ||||||
|     private void controlEnd(final String updateId) { |     private void controlExamEnd(final String updateId) { | ||||||
|         if (log.isDebugEnabled()) { |         if (log.isDebugEnabled()) { | ||||||
|             log.debug("Check ending exams: {}", updateId); |             log.debug("Check ending exams: {}", updateId); | ||||||
|         } |         } | ||||||
|  |  | ||||||
|  | @ -8,29 +8,60 @@ | ||||||
| 
 | 
 | ||||||
| package ch.ethz.seb.sebserver.webservice.servicelayer.session.impl; | package ch.ethz.seb.sebserver.webservice.servicelayer.session.impl; | ||||||
| 
 | 
 | ||||||
|  | import java.math.BigDecimal; | ||||||
|  | import java.util.Comparator; | ||||||
|  | 
 | ||||||
| import org.joda.time.DateTime; | import org.joda.time.DateTime; | ||||||
| import org.joda.time.DateTimeZone; | import org.joda.time.DateTimeZone; | ||||||
|  | import org.slf4j.Logger; | ||||||
|  | import org.slf4j.LoggerFactory; | ||||||
| import org.springframework.beans.factory.config.ConfigurableBeanFactory; | import org.springframework.beans.factory.config.ConfigurableBeanFactory; | ||||||
| import org.springframework.context.annotation.Lazy; | import org.springframework.context.annotation.Lazy; | ||||||
| import org.springframework.context.annotation.Scope; | import org.springframework.context.annotation.Scope; | ||||||
| import org.springframework.stereotype.Component; | import org.springframework.stereotype.Component; | ||||||
| 
 | 
 | ||||||
|  | import ch.ethz.seb.sebserver.gbl.Constants; | ||||||
|  | import ch.ethz.seb.sebserver.gbl.model.exam.Indicator; | ||||||
| import ch.ethz.seb.sebserver.gbl.model.exam.Indicator.IndicatorType; | import ch.ethz.seb.sebserver.gbl.model.exam.Indicator.IndicatorType; | ||||||
|  | import ch.ethz.seb.sebserver.gbl.model.session.ClientEvent.EventType; | ||||||
| import ch.ethz.seb.sebserver.gbl.model.session.IndicatorValueHolder; | import ch.ethz.seb.sebserver.gbl.model.session.IndicatorValueHolder; | ||||||
| import ch.ethz.seb.sebserver.gbl.util.Utils; | import ch.ethz.seb.sebserver.gbl.util.Utils; | ||||||
| import ch.ethz.seb.sebserver.webservice.datalayer.batis.ClientEventExtentionMapper; | import ch.ethz.seb.sebserver.webservice.datalayer.batis.ClientEventExtentionMapper; | ||||||
|  | import ch.ethz.seb.sebserver.webservice.datalayer.batis.model.ClientEventRecord; | ||||||
| 
 | 
 | ||||||
| @Lazy | @Lazy | ||||||
| @Component(IndicatorType.Names.LAST_PING) | @Component(IndicatorType.Names.LAST_PING) | ||||||
| @Scope(ConfigurableBeanFactory.SCOPE_PROTOTYPE) | @Scope(ConfigurableBeanFactory.SCOPE_PROTOTYPE) | ||||||
| public final class PingIntervalClientIndicator extends AbstractPingIndicator { | public final class PingIntervalClientIndicator extends AbstractPingIndicator { | ||||||
| 
 | 
 | ||||||
|  |     private static final Logger log = LoggerFactory.getLogger(PingIntervalClientIndicator.class); | ||||||
|  | 
 | ||||||
|  |     private long pingErrorThreshold; | ||||||
|  |     private boolean isOnError = false; | ||||||
|  | 
 | ||||||
|     public PingIntervalClientIndicator(final ClientEventExtentionMapper clientEventExtentionMapper) { |     public PingIntervalClientIndicator(final ClientEventExtentionMapper clientEventExtentionMapper) { | ||||||
|         super(clientEventExtentionMapper); |         super(clientEventExtentionMapper); | ||||||
|         this.cachingEnabled = true; |         this.cachingEnabled = true; | ||||||
|         this.currentValue = computeValueAt(Utils.getMillisecondsNow()); |         this.currentValue = computeValueAt(Utils.getMillisecondsNow()); | ||||||
|     } |     } | ||||||
| 
 | 
 | ||||||
|  |     @Override | ||||||
|  |     public void init(final Indicator indicatorDefinition, final Long connectionId, final boolean cachingEnabled) { | ||||||
|  |         super.init(indicatorDefinition, connectionId, cachingEnabled); | ||||||
|  | 
 | ||||||
|  |         try { | ||||||
|  |             indicatorDefinition | ||||||
|  |                     .getThresholds() | ||||||
|  |                     .stream() | ||||||
|  |                     .sorted(Comparator.reverseOrder()) | ||||||
|  |                     .findFirst() | ||||||
|  |                     .ifPresent(t -> this.pingErrorThreshold = t.value.longValue()); | ||||||
|  |         } catch (final Exception e) { | ||||||
|  |             log.error("Failed to initialize pingErrorThreshold: {}", e.getMessage()); | ||||||
|  |             this.pingErrorThreshold = Constants.SECOND_IN_MILLIS * 5; | ||||||
|  |         } | ||||||
|  |     } | ||||||
|  | 
 | ||||||
|     @Override |     @Override | ||||||
|     public IndicatorType getType() { |     public IndicatorType getType() { | ||||||
|         return IndicatorType.LAST_PING; |         return IndicatorType.LAST_PING; | ||||||
|  | @ -47,4 +78,37 @@ public final class PingIntervalClientIndicator extends AbstractPingIndicator { | ||||||
| 
 | 
 | ||||||
|     } |     } | ||||||
| 
 | 
 | ||||||
|  |     @Override | ||||||
|  |     public ClientEventRecord updateLogEvent() { | ||||||
|  |         final long now = DateTime.now(DateTimeZone.UTC).getMillis(); | ||||||
|  |         final long value = now - (long) super.currentValue; | ||||||
|  |         if (this.isOnError) { | ||||||
|  |             if (this.pingErrorThreshold > value) { | ||||||
|  |                 this.isOnError = false; | ||||||
|  |                 return new ClientEventRecord( | ||||||
|  |                         null, | ||||||
|  |                         this.connectionId, | ||||||
|  |                         EventType.INFO_LOG.id, | ||||||
|  |                         now, | ||||||
|  |                         now, | ||||||
|  |                         new BigDecimal(value), | ||||||
|  |                         "Client Ping Back To Normal"); | ||||||
|  |             } | ||||||
|  |         } else { | ||||||
|  |             if (this.pingErrorThreshold < value) { | ||||||
|  |                 this.isOnError = true; | ||||||
|  |                 return new ClientEventRecord( | ||||||
|  |                         null, | ||||||
|  |                         this.connectionId, | ||||||
|  |                         EventType.ERROR_LOG.id, | ||||||
|  |                         now, | ||||||
|  |                         now, | ||||||
|  |                         new BigDecimal(value), | ||||||
|  |                         "Missing Client Ping"); | ||||||
|  |             } | ||||||
|  |         } | ||||||
|  | 
 | ||||||
|  |         return null; | ||||||
|  |     } | ||||||
|  | 
 | ||||||
| } | } | ||||||
|  |  | ||||||
|  | @ -9,11 +9,14 @@ | ||||||
| package ch.ethz.seb.sebserver.webservice.servicelayer.session.impl; | package ch.ethz.seb.sebserver.webservice.servicelayer.session.impl; | ||||||
| 
 | 
 | ||||||
| import java.security.Principal; | import java.security.Principal; | ||||||
|  | import java.util.Objects; | ||||||
| import java.util.UUID; | import java.util.UUID; | ||||||
| 
 | 
 | ||||||
| import org.apache.commons.lang3.StringUtils; | import org.apache.commons.lang3.StringUtils; | ||||||
| import org.slf4j.Logger; | import org.slf4j.Logger; | ||||||
| import org.slf4j.LoggerFactory; | import org.slf4j.LoggerFactory; | ||||||
|  | import org.springframework.cache.Cache; | ||||||
|  | import org.springframework.cache.CacheManager; | ||||||
| import org.springframework.context.annotation.Lazy; | import org.springframework.context.annotation.Lazy; | ||||||
| import org.springframework.stereotype.Service; | import org.springframework.stereotype.Service; | ||||||
| 
 | 
 | ||||||
|  | @ -43,6 +46,7 @@ public class SebClientConnectionServiceImpl implements SebClientConnectionServic | ||||||
| 
 | 
 | ||||||
|     private final ExamSessionService examSessionService; |     private final ExamSessionService examSessionService; | ||||||
|     private final ExamSessionCacheService examSessionCacheService; |     private final ExamSessionCacheService examSessionCacheService; | ||||||
|  |     private final CacheManager cacheManager; | ||||||
|     private final EventHandlingStrategy eventHandlingStrategy; |     private final EventHandlingStrategy eventHandlingStrategy; | ||||||
|     private final ClientConnectionDAO clientConnectionDAO; |     private final ClientConnectionDAO clientConnectionDAO; | ||||||
|     private final PingHandlingStrategy pingHandlingStrategy; |     private final PingHandlingStrategy pingHandlingStrategy; | ||||||
|  | @ -52,6 +56,7 @@ public class SebClientConnectionServiceImpl implements SebClientConnectionServic | ||||||
|     protected SebClientConnectionServiceImpl( |     protected SebClientConnectionServiceImpl( | ||||||
|             final ExamSessionService examSessionService, |             final ExamSessionService examSessionService, | ||||||
|             final ExamSessionCacheService examSessionCacheService, |             final ExamSessionCacheService examSessionCacheService, | ||||||
|  |             final CacheManager cacheManager, | ||||||
|             final ClientConnectionDAO clientConnectionDAO, |             final ClientConnectionDAO clientConnectionDAO, | ||||||
|             final EventHandlingStrategyFactory eventHandlingStrategyFactory, |             final EventHandlingStrategyFactory eventHandlingStrategyFactory, | ||||||
|             final PingHandlingStrategyFactory pingHandlingStrategyFactory, |             final PingHandlingStrategyFactory pingHandlingStrategyFactory, | ||||||
|  | @ -60,6 +65,7 @@ public class SebClientConnectionServiceImpl implements SebClientConnectionServic | ||||||
| 
 | 
 | ||||||
|         this.examSessionService = examSessionService; |         this.examSessionService = examSessionService; | ||||||
|         this.examSessionCacheService = examSessionCacheService; |         this.examSessionCacheService = examSessionCacheService; | ||||||
|  |         this.cacheManager = cacheManager; | ||||||
|         this.clientConnectionDAO = clientConnectionDAO; |         this.clientConnectionDAO = clientConnectionDAO; | ||||||
|         this.pingHandlingStrategy = pingHandlingStrategyFactory.get(); |         this.pingHandlingStrategy = pingHandlingStrategyFactory.get(); | ||||||
|         this.eventHandlingStrategy = eventHandlingStrategyFactory.get(); |         this.eventHandlingStrategy = eventHandlingStrategyFactory.get(); | ||||||
|  | @ -382,6 +388,32 @@ public class SebClientConnectionServiceImpl implements SebClientConnectionServic | ||||||
|         }); |         }); | ||||||
|     } |     } | ||||||
| 
 | 
 | ||||||
|  |     @Override | ||||||
|  |     public void updatePingEvents() { | ||||||
|  |         try { | ||||||
|  | 
 | ||||||
|  |             final Cache cache = this.cacheManager.getCache(ExamSessionCacheService.CACHE_NAME_ACTIVE_CLIENT_CONNECTION); | ||||||
|  |             this.examSessionService | ||||||
|  |                     .getExamDAO() | ||||||
|  |                     .allRunningExamIds() | ||||||
|  |                     .getOrThrow() | ||||||
|  |                     .stream() | ||||||
|  |                     .flatMap(examId -> this.clientConnectionDAO | ||||||
|  |                             .getConnectionTokens(examId) | ||||||
|  |                             .getOrThrow() | ||||||
|  |                             .stream()) | ||||||
|  |                     .map(token -> cache.get(token, ClientConnectionDataInternal.class)) | ||||||
|  |                     .filter(Objects::nonNull) | ||||||
|  |                     .flatMap(connection -> connection.pingMappings.stream()) | ||||||
|  |                     .map(ping -> ping.updateLogEvent()) | ||||||
|  |                     .filter(Objects::nonNull) | ||||||
|  |                     .forEach(this.eventHandlingStrategy::accept); | ||||||
|  | 
 | ||||||
|  |         } catch (final Exception e) { | ||||||
|  |             log.error("Failed to update ping events: ", e); | ||||||
|  |         } | ||||||
|  |     } | ||||||
|  | 
 | ||||||
|     @Override |     @Override | ||||||
|     public String notifyPing( |     public String notifyPing( | ||||||
|             final String connectionToken, |             final String connectionToken, | ||||||
|  |  | ||||||
		Loading…
	
	Add table
		
		Reference in a new issue
	
	 anhefti
						anhefti