From b27c79e1d8f6149915e540dde329422177c209ff Mon Sep 17 00:00:00 2001 From: anhefti Date: Mon, 9 Dec 2019 15:26:14 +0100 Subject: [PATCH] log SEB Client ping threshold overflows as Error log --- .../sebserver/gbl/model/exam/Indicator.java | 7 +- .../webservice/servicelayer/dao/ExamDAO.java | 5 ++ .../servicelayer/dao/impl/ExamDAOImpl.java | 20 ++++++ .../session/SebClientConnectionService.java | 2 + .../session/impl/AbstractPingIndicator.java | 3 + .../session/impl/ExamSessionControlTask.java | 22 +++++-- .../impl/PingIntervalClientIndicator.java | 64 +++++++++++++++++++ .../impl/SebClientConnectionServiceImpl.java | 32 ++++++++++ 8 files changed, 148 insertions(+), 7 deletions(-) diff --git a/src/main/java/ch/ethz/seb/sebserver/gbl/model/exam/Indicator.java b/src/main/java/ch/ethz/seb/sebserver/gbl/model/exam/Indicator.java index c145c73c..7c08b7f9 100644 --- a/src/main/java/ch/ethz/seb/sebserver/gbl/model/exam/Indicator.java +++ b/src/main/java/ch/ethz/seb/sebserver/gbl/model/exam/Indicator.java @@ -175,7 +175,7 @@ public final class Indicator implements Entity { } } - public static final class Threshold { + public static final class Threshold implements Comparable { @JsonProperty(THRESHOLD.ATTR_VALUE) @NotNull @@ -206,6 +206,11 @@ public final class Indicator implements Entity { return "Threshold [value=" + this.value + ", color=" + this.color + "]"; } + @Override + public int compareTo(final Threshold o) { + return Double.compare(this.value, (o != null) ? o.value : 0); + } + } } diff --git a/src/main/java/ch/ethz/seb/sebserver/webservice/servicelayer/dao/ExamDAO.java b/src/main/java/ch/ethz/seb/sebserver/webservice/servicelayer/dao/ExamDAO.java index 72979963..bacbf73d 100644 --- a/src/main/java/ch/ethz/seb/sebserver/webservice/servicelayer/dao/ExamDAO.java +++ b/src/main/java/ch/ethz/seb/sebserver/webservice/servicelayer/dao/ExamDAO.java @@ -71,6 +71,11 @@ public interface ExamDAO extends ActivatableEntityDAO, BulkActionSup * @return Result refer to a collection of exams or to an error if happened */ Result> allForEndCheck(); + /** Get a collection of all currently running exam identifiers + * + * @return collection of all currently running exam identifiers */ + Result> allRunningExamIds(); + /** This is used to place an internal (write)lock for the specified exam. * The exam will be marked as locked on the persistence level to prevent other running web-service instances * to write concurrently to the specified exam while it is been updated by an internal batch process. diff --git a/src/main/java/ch/ethz/seb/sebserver/webservice/servicelayer/dao/impl/ExamDAOImpl.java b/src/main/java/ch/ethz/seb/sebserver/webservice/servicelayer/dao/impl/ExamDAOImpl.java index 819fc76f..a96837c1 100644 --- a/src/main/java/ch/ethz/seb/sebserver/webservice/servicelayer/dao/impl/ExamDAOImpl.java +++ b/src/main/java/ch/ethz/seb/sebserver/webservice/servicelayer/dao/impl/ExamDAOImpl.java @@ -353,6 +353,26 @@ public class ExamDAOImpl implements ExamDAO { .execute()); } + @Override + @Transactional(readOnly = true) + public Result> allRunningExamIds() { + return Result.tryCatch(() -> { + return this.examRecordMapper.selectIdsByExample() + .where( + ExamRecordDynamicSqlSupport.active, + isEqualTo(BooleanUtils.toInteger(true))) + .and( + ExamRecordDynamicSqlSupport.status, + isEqualTo(ExamStatus.RUNNING.name())) + .and( + ExamRecordDynamicSqlSupport.updating, + isEqualTo(BooleanUtils.toInteger(false))) + + .build() + .execute(); + }); + } + @Override @Transactional(readOnly = true) public Result> allForRunCheck() { diff --git a/src/main/java/ch/ethz/seb/sebserver/webservice/servicelayer/session/SebClientConnectionService.java b/src/main/java/ch/ethz/seb/sebserver/webservice/servicelayer/session/SebClientConnectionService.java index c75ae958..78f422f3 100644 --- a/src/main/java/ch/ethz/seb/sebserver/webservice/servicelayer/session/SebClientConnectionService.java +++ b/src/main/java/ch/ethz/seb/sebserver/webservice/servicelayer/session/SebClientConnectionService.java @@ -103,6 +103,8 @@ public interface SebClientConnectionService { Long institutionId, String clientAddress); + void updatePingEvents(); + /** Notify a ping for a certain client connection. * * @param connectionToken the connection token diff --git a/src/main/java/ch/ethz/seb/sebserver/webservice/servicelayer/session/impl/AbstractPingIndicator.java b/src/main/java/ch/ethz/seb/sebserver/webservice/servicelayer/session/impl/AbstractPingIndicator.java index 4d6d9996..425b7870 100644 --- a/src/main/java/ch/ethz/seb/sebserver/webservice/servicelayer/session/impl/AbstractPingIndicator.java +++ b/src/main/java/ch/ethz/seb/sebserver/webservice/servicelayer/session/impl/AbstractPingIndicator.java @@ -23,6 +23,7 @@ import com.fasterxml.jackson.annotation.JsonIgnore; import ch.ethz.seb.sebserver.gbl.model.session.ClientEvent.EventType; import ch.ethz.seb.sebserver.webservice.datalayer.batis.ClientEventExtentionMapper; import ch.ethz.seb.sebserver.webservice.datalayer.batis.mapper.ClientEventRecordDynamicSqlSupport; +import ch.ethz.seb.sebserver.webservice.datalayer.batis.model.ClientEventRecord; public abstract class AbstractPingIndicator extends AbstractClientIndicator { @@ -84,4 +85,6 @@ public abstract class AbstractPingIndicator extends AbstractClientIndicator { return this.pingNumber; } + public abstract ClientEventRecord updateLogEvent(); + } diff --git a/src/main/java/ch/ethz/seb/sebserver/webservice/servicelayer/session/impl/ExamSessionControlTask.java b/src/main/java/ch/ethz/seb/sebserver/webservice/servicelayer/session/impl/ExamSessionControlTask.java index 5b9f07ea..1b7da979 100644 --- a/src/main/java/ch/ethz/seb/sebserver/webservice/servicelayer/session/impl/ExamSessionControlTask.java +++ b/src/main/java/ch/ethz/seb/sebserver/webservice/servicelayer/session/impl/ExamSessionControlTask.java @@ -20,8 +20,10 @@ import org.springframework.scheduling.annotation.Async; import org.springframework.scheduling.annotation.Scheduled; import org.springframework.stereotype.Service; +import ch.ethz.seb.sebserver.gbl.Constants; import ch.ethz.seb.sebserver.gbl.model.exam.Exam; import ch.ethz.seb.sebserver.webservice.servicelayer.dao.ExamDAO; +import ch.ethz.seb.sebserver.webservice.servicelayer.session.SebClientConnectionService; @Service class ExamSessionControlTask { @@ -29,6 +31,7 @@ class ExamSessionControlTask { private static final Logger log = LoggerFactory.getLogger(ExamSessionControlTask.class); private final ExamDAO examDAO; + private final SebClientConnectionService sebClientConnectionService; private final ExamUpdateHandler examUpdateHandler; private final Long examTimePrefix; private final Long examTimeSuffix; @@ -46,20 +49,21 @@ class ExamSessionControlTask { protected ExamSessionControlTask( final ExamDAO examDAO, + final SebClientConnectionService sebClientConnectionService, final ExamUpdateHandler examUpdateHandler, @Value("${sebserver.webservice.api.exam.time-prefix:3600000}") final Long examTimePrefix, @Value("${sebserver.webservice.api.exam.time-suffix:3600000}") final Long examTimeSuffix) { this.examDAO = examDAO; + this.sebClientConnectionService = sebClientConnectionService; this.examUpdateHandler = examUpdateHandler; this.examTimePrefix = examTimePrefix; this.examTimeSuffix = examTimeSuffix; - } @Async @Scheduled(cron = "${sebserver.webservice.api.exam.update-interval:1 * * * * *}") - public void execTask() { + public void examRunUpdateTask() { final String updateId = this.examUpdateHandler.createUpdateId(); @@ -67,11 +71,17 @@ class ExamSessionControlTask { log.debug("Run exam runtime update task with Id: {}", updateId); } - controlStart(updateId); - controlEnd(updateId); + controlExamStart(updateId); + controlExamEnd(updateId); } - private void controlStart(final String updateId) { + @Async + @Scheduled(fixedRate = Constants.SECOND_IN_MILLIS) + public void pingEventUpdateTask() { + this.sebClientConnectionService.updatePingEvents(); + } + + private void controlExamStart(final String updateId) { if (log.isDebugEnabled()) { log.debug("Check starting exams: {}", updateId); } @@ -95,7 +105,7 @@ class ExamSessionControlTask { } } - private void controlEnd(final String updateId) { + private void controlExamEnd(final String updateId) { if (log.isDebugEnabled()) { log.debug("Check ending exams: {}", updateId); } diff --git a/src/main/java/ch/ethz/seb/sebserver/webservice/servicelayer/session/impl/PingIntervalClientIndicator.java b/src/main/java/ch/ethz/seb/sebserver/webservice/servicelayer/session/impl/PingIntervalClientIndicator.java index 7e982fcd..0637a8ca 100644 --- a/src/main/java/ch/ethz/seb/sebserver/webservice/servicelayer/session/impl/PingIntervalClientIndicator.java +++ b/src/main/java/ch/ethz/seb/sebserver/webservice/servicelayer/session/impl/PingIntervalClientIndicator.java @@ -8,29 +8,60 @@ package ch.ethz.seb.sebserver.webservice.servicelayer.session.impl; +import java.math.BigDecimal; +import java.util.Comparator; + import org.joda.time.DateTime; import org.joda.time.DateTimeZone; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.springframework.beans.factory.config.ConfigurableBeanFactory; import org.springframework.context.annotation.Lazy; import org.springframework.context.annotation.Scope; import org.springframework.stereotype.Component; +import ch.ethz.seb.sebserver.gbl.Constants; +import ch.ethz.seb.sebserver.gbl.model.exam.Indicator; import ch.ethz.seb.sebserver.gbl.model.exam.Indicator.IndicatorType; +import ch.ethz.seb.sebserver.gbl.model.session.ClientEvent.EventType; import ch.ethz.seb.sebserver.gbl.model.session.IndicatorValueHolder; import ch.ethz.seb.sebserver.gbl.util.Utils; import ch.ethz.seb.sebserver.webservice.datalayer.batis.ClientEventExtentionMapper; +import ch.ethz.seb.sebserver.webservice.datalayer.batis.model.ClientEventRecord; @Lazy @Component(IndicatorType.Names.LAST_PING) @Scope(ConfigurableBeanFactory.SCOPE_PROTOTYPE) public final class PingIntervalClientIndicator extends AbstractPingIndicator { + private static final Logger log = LoggerFactory.getLogger(PingIntervalClientIndicator.class); + + private long pingErrorThreshold; + private boolean isOnError = false; + public PingIntervalClientIndicator(final ClientEventExtentionMapper clientEventExtentionMapper) { super(clientEventExtentionMapper); this.cachingEnabled = true; this.currentValue = computeValueAt(Utils.getMillisecondsNow()); } + @Override + public void init(final Indicator indicatorDefinition, final Long connectionId, final boolean cachingEnabled) { + super.init(indicatorDefinition, connectionId, cachingEnabled); + + try { + indicatorDefinition + .getThresholds() + .stream() + .sorted(Comparator.reverseOrder()) + .findFirst() + .ifPresent(t -> this.pingErrorThreshold = t.value.longValue()); + } catch (final Exception e) { + log.error("Failed to initialize pingErrorThreshold: {}", e.getMessage()); + this.pingErrorThreshold = Constants.SECOND_IN_MILLIS * 5; + } + } + @Override public IndicatorType getType() { return IndicatorType.LAST_PING; @@ -47,4 +78,37 @@ public final class PingIntervalClientIndicator extends AbstractPingIndicator { } + @Override + public ClientEventRecord updateLogEvent() { + final long now = DateTime.now(DateTimeZone.UTC).getMillis(); + final long value = now - (long) super.currentValue; + if (this.isOnError) { + if (this.pingErrorThreshold > value) { + this.isOnError = false; + return new ClientEventRecord( + null, + this.connectionId, + EventType.INFO_LOG.id, + now, + now, + new BigDecimal(value), + "Client Ping Back To Normal"); + } + } else { + if (this.pingErrorThreshold < value) { + this.isOnError = true; + return new ClientEventRecord( + null, + this.connectionId, + EventType.ERROR_LOG.id, + now, + now, + new BigDecimal(value), + "Missing Client Ping"); + } + } + + return null; + } + } diff --git a/src/main/java/ch/ethz/seb/sebserver/webservice/servicelayer/session/impl/SebClientConnectionServiceImpl.java b/src/main/java/ch/ethz/seb/sebserver/webservice/servicelayer/session/impl/SebClientConnectionServiceImpl.java index b0a49a99..817b3424 100644 --- a/src/main/java/ch/ethz/seb/sebserver/webservice/servicelayer/session/impl/SebClientConnectionServiceImpl.java +++ b/src/main/java/ch/ethz/seb/sebserver/webservice/servicelayer/session/impl/SebClientConnectionServiceImpl.java @@ -9,11 +9,14 @@ package ch.ethz.seb.sebserver.webservice.servicelayer.session.impl; import java.security.Principal; +import java.util.Objects; import java.util.UUID; import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.springframework.cache.Cache; +import org.springframework.cache.CacheManager; import org.springframework.context.annotation.Lazy; import org.springframework.stereotype.Service; @@ -43,6 +46,7 @@ public class SebClientConnectionServiceImpl implements SebClientConnectionServic private final ExamSessionService examSessionService; private final ExamSessionCacheService examSessionCacheService; + private final CacheManager cacheManager; private final EventHandlingStrategy eventHandlingStrategy; private final ClientConnectionDAO clientConnectionDAO; private final PingHandlingStrategy pingHandlingStrategy; @@ -52,6 +56,7 @@ public class SebClientConnectionServiceImpl implements SebClientConnectionServic protected SebClientConnectionServiceImpl( final ExamSessionService examSessionService, final ExamSessionCacheService examSessionCacheService, + final CacheManager cacheManager, final ClientConnectionDAO clientConnectionDAO, final EventHandlingStrategyFactory eventHandlingStrategyFactory, final PingHandlingStrategyFactory pingHandlingStrategyFactory, @@ -60,6 +65,7 @@ public class SebClientConnectionServiceImpl implements SebClientConnectionServic this.examSessionService = examSessionService; this.examSessionCacheService = examSessionCacheService; + this.cacheManager = cacheManager; this.clientConnectionDAO = clientConnectionDAO; this.pingHandlingStrategy = pingHandlingStrategyFactory.get(); this.eventHandlingStrategy = eventHandlingStrategyFactory.get(); @@ -382,6 +388,32 @@ public class SebClientConnectionServiceImpl implements SebClientConnectionServic }); } + @Override + public void updatePingEvents() { + try { + + final Cache cache = this.cacheManager.getCache(ExamSessionCacheService.CACHE_NAME_ACTIVE_CLIENT_CONNECTION); + this.examSessionService + .getExamDAO() + .allRunningExamIds() + .getOrThrow() + .stream() + .flatMap(examId -> this.clientConnectionDAO + .getConnectionTokens(examId) + .getOrThrow() + .stream()) + .map(token -> cache.get(token, ClientConnectionDataInternal.class)) + .filter(Objects::nonNull) + .flatMap(connection -> connection.pingMappings.stream()) + .map(ping -> ping.updateLogEvent()) + .filter(Objects::nonNull) + .forEach(this.eventHandlingStrategy::accept); + + } catch (final Exception e) { + log.error("Failed to update ping events: ", e); + } + } + @Override public String notifyPing( final String connectionToken,