log SEB Client ping threshold overflows as Error log

This commit is contained in:
anhefti 2019-12-09 15:26:14 +01:00
parent 3fee5dc0a6
commit b27c79e1d8
8 changed files with 148 additions and 7 deletions

View file

@ -175,7 +175,7 @@ public final class Indicator implements Entity {
} }
} }
public static final class Threshold { public static final class Threshold implements Comparable<Threshold> {
@JsonProperty(THRESHOLD.ATTR_VALUE) @JsonProperty(THRESHOLD.ATTR_VALUE)
@NotNull @NotNull
@ -206,6 +206,11 @@ public final class Indicator implements Entity {
return "Threshold [value=" + this.value + ", color=" + this.color + "]"; return "Threshold [value=" + this.value + ", color=" + this.color + "]";
} }
@Override
public int compareTo(final Threshold o) {
return Double.compare(this.value, (o != null) ? o.value : 0);
}
} }
} }

View file

@ -71,6 +71,11 @@ public interface ExamDAO extends ActivatableEntityDAO<Exam, Exam>, BulkActionSup
* @return Result refer to a collection of exams or to an error if happened */ * @return Result refer to a collection of exams or to an error if happened */
Result<Collection<Exam>> allForEndCheck(); Result<Collection<Exam>> allForEndCheck();
/** Get a collection of all currently running exam identifiers
*
* @return collection of all currently running exam identifiers */
Result<Collection<Long>> allRunningExamIds();
/** This is used to place an internal (write)lock for the specified exam. /** This is used to place an internal (write)lock for the specified exam.
* The exam will be marked as locked on the persistence level to prevent other running web-service instances * The exam will be marked as locked on the persistence level to prevent other running web-service instances
* to write concurrently to the specified exam while it is been updated by an internal batch process. * to write concurrently to the specified exam while it is been updated by an internal batch process.

View file

@ -353,6 +353,26 @@ public class ExamDAOImpl implements ExamDAO {
.execute()); .execute());
} }
@Override
@Transactional(readOnly = true)
public Result<Collection<Long>> allRunningExamIds() {
return Result.tryCatch(() -> {
return this.examRecordMapper.selectIdsByExample()
.where(
ExamRecordDynamicSqlSupport.active,
isEqualTo(BooleanUtils.toInteger(true)))
.and(
ExamRecordDynamicSqlSupport.status,
isEqualTo(ExamStatus.RUNNING.name()))
.and(
ExamRecordDynamicSqlSupport.updating,
isEqualTo(BooleanUtils.toInteger(false)))
.build()
.execute();
});
}
@Override @Override
@Transactional(readOnly = true) @Transactional(readOnly = true)
public Result<Collection<Exam>> allForRunCheck() { public Result<Collection<Exam>> allForRunCheck() {

View file

@ -103,6 +103,8 @@ public interface SebClientConnectionService {
Long institutionId, Long institutionId,
String clientAddress); String clientAddress);
void updatePingEvents();
/** Notify a ping for a certain client connection. /** Notify a ping for a certain client connection.
* *
* @param connectionToken the connection token * @param connectionToken the connection token

View file

@ -23,6 +23,7 @@ import com.fasterxml.jackson.annotation.JsonIgnore;
import ch.ethz.seb.sebserver.gbl.model.session.ClientEvent.EventType; import ch.ethz.seb.sebserver.gbl.model.session.ClientEvent.EventType;
import ch.ethz.seb.sebserver.webservice.datalayer.batis.ClientEventExtentionMapper; import ch.ethz.seb.sebserver.webservice.datalayer.batis.ClientEventExtentionMapper;
import ch.ethz.seb.sebserver.webservice.datalayer.batis.mapper.ClientEventRecordDynamicSqlSupport; import ch.ethz.seb.sebserver.webservice.datalayer.batis.mapper.ClientEventRecordDynamicSqlSupport;
import ch.ethz.seb.sebserver.webservice.datalayer.batis.model.ClientEventRecord;
public abstract class AbstractPingIndicator extends AbstractClientIndicator { public abstract class AbstractPingIndicator extends AbstractClientIndicator {
@ -84,4 +85,6 @@ public abstract class AbstractPingIndicator extends AbstractClientIndicator {
return this.pingNumber; return this.pingNumber;
} }
public abstract ClientEventRecord updateLogEvent();
} }

View file

@ -20,8 +20,10 @@ import org.springframework.scheduling.annotation.Async;
import org.springframework.scheduling.annotation.Scheduled; import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Service; import org.springframework.stereotype.Service;
import ch.ethz.seb.sebserver.gbl.Constants;
import ch.ethz.seb.sebserver.gbl.model.exam.Exam; import ch.ethz.seb.sebserver.gbl.model.exam.Exam;
import ch.ethz.seb.sebserver.webservice.servicelayer.dao.ExamDAO; import ch.ethz.seb.sebserver.webservice.servicelayer.dao.ExamDAO;
import ch.ethz.seb.sebserver.webservice.servicelayer.session.SebClientConnectionService;
@Service @Service
class ExamSessionControlTask { class ExamSessionControlTask {
@ -29,6 +31,7 @@ class ExamSessionControlTask {
private static final Logger log = LoggerFactory.getLogger(ExamSessionControlTask.class); private static final Logger log = LoggerFactory.getLogger(ExamSessionControlTask.class);
private final ExamDAO examDAO; private final ExamDAO examDAO;
private final SebClientConnectionService sebClientConnectionService;
private final ExamUpdateHandler examUpdateHandler; private final ExamUpdateHandler examUpdateHandler;
private final Long examTimePrefix; private final Long examTimePrefix;
private final Long examTimeSuffix; private final Long examTimeSuffix;
@ -46,20 +49,21 @@ class ExamSessionControlTask {
protected ExamSessionControlTask( protected ExamSessionControlTask(
final ExamDAO examDAO, final ExamDAO examDAO,
final SebClientConnectionService sebClientConnectionService,
final ExamUpdateHandler examUpdateHandler, final ExamUpdateHandler examUpdateHandler,
@Value("${sebserver.webservice.api.exam.time-prefix:3600000}") final Long examTimePrefix, @Value("${sebserver.webservice.api.exam.time-prefix:3600000}") final Long examTimePrefix,
@Value("${sebserver.webservice.api.exam.time-suffix:3600000}") final Long examTimeSuffix) { @Value("${sebserver.webservice.api.exam.time-suffix:3600000}") final Long examTimeSuffix) {
this.examDAO = examDAO; this.examDAO = examDAO;
this.sebClientConnectionService = sebClientConnectionService;
this.examUpdateHandler = examUpdateHandler; this.examUpdateHandler = examUpdateHandler;
this.examTimePrefix = examTimePrefix; this.examTimePrefix = examTimePrefix;
this.examTimeSuffix = examTimeSuffix; this.examTimeSuffix = examTimeSuffix;
} }
@Async @Async
@Scheduled(cron = "${sebserver.webservice.api.exam.update-interval:1 * * * * *}") @Scheduled(cron = "${sebserver.webservice.api.exam.update-interval:1 * * * * *}")
public void execTask() { public void examRunUpdateTask() {
final String updateId = this.examUpdateHandler.createUpdateId(); final String updateId = this.examUpdateHandler.createUpdateId();
@ -67,11 +71,17 @@ class ExamSessionControlTask {
log.debug("Run exam runtime update task with Id: {}", updateId); log.debug("Run exam runtime update task with Id: {}", updateId);
} }
controlStart(updateId); controlExamStart(updateId);
controlEnd(updateId); controlExamEnd(updateId);
} }
private void controlStart(final String updateId) { @Async
@Scheduled(fixedRate = Constants.SECOND_IN_MILLIS)
public void pingEventUpdateTask() {
this.sebClientConnectionService.updatePingEvents();
}
private void controlExamStart(final String updateId) {
if (log.isDebugEnabled()) { if (log.isDebugEnabled()) {
log.debug("Check starting exams: {}", updateId); log.debug("Check starting exams: {}", updateId);
} }
@ -95,7 +105,7 @@ class ExamSessionControlTask {
} }
} }
private void controlEnd(final String updateId) { private void controlExamEnd(final String updateId) {
if (log.isDebugEnabled()) { if (log.isDebugEnabled()) {
log.debug("Check ending exams: {}", updateId); log.debug("Check ending exams: {}", updateId);
} }

View file

@ -8,29 +8,60 @@
package ch.ethz.seb.sebserver.webservice.servicelayer.session.impl; package ch.ethz.seb.sebserver.webservice.servicelayer.session.impl;
import java.math.BigDecimal;
import java.util.Comparator;
import org.joda.time.DateTime; import org.joda.time.DateTime;
import org.joda.time.DateTimeZone; import org.joda.time.DateTimeZone;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.config.ConfigurableBeanFactory; import org.springframework.beans.factory.config.ConfigurableBeanFactory;
import org.springframework.context.annotation.Lazy; import org.springframework.context.annotation.Lazy;
import org.springframework.context.annotation.Scope; import org.springframework.context.annotation.Scope;
import org.springframework.stereotype.Component; import org.springframework.stereotype.Component;
import ch.ethz.seb.sebserver.gbl.Constants;
import ch.ethz.seb.sebserver.gbl.model.exam.Indicator;
import ch.ethz.seb.sebserver.gbl.model.exam.Indicator.IndicatorType; import ch.ethz.seb.sebserver.gbl.model.exam.Indicator.IndicatorType;
import ch.ethz.seb.sebserver.gbl.model.session.ClientEvent.EventType;
import ch.ethz.seb.sebserver.gbl.model.session.IndicatorValueHolder; import ch.ethz.seb.sebserver.gbl.model.session.IndicatorValueHolder;
import ch.ethz.seb.sebserver.gbl.util.Utils; import ch.ethz.seb.sebserver.gbl.util.Utils;
import ch.ethz.seb.sebserver.webservice.datalayer.batis.ClientEventExtentionMapper; import ch.ethz.seb.sebserver.webservice.datalayer.batis.ClientEventExtentionMapper;
import ch.ethz.seb.sebserver.webservice.datalayer.batis.model.ClientEventRecord;
@Lazy @Lazy
@Component(IndicatorType.Names.LAST_PING) @Component(IndicatorType.Names.LAST_PING)
@Scope(ConfigurableBeanFactory.SCOPE_PROTOTYPE) @Scope(ConfigurableBeanFactory.SCOPE_PROTOTYPE)
public final class PingIntervalClientIndicator extends AbstractPingIndicator { public final class PingIntervalClientIndicator extends AbstractPingIndicator {
private static final Logger log = LoggerFactory.getLogger(PingIntervalClientIndicator.class);
private long pingErrorThreshold;
private boolean isOnError = false;
public PingIntervalClientIndicator(final ClientEventExtentionMapper clientEventExtentionMapper) { public PingIntervalClientIndicator(final ClientEventExtentionMapper clientEventExtentionMapper) {
super(clientEventExtentionMapper); super(clientEventExtentionMapper);
this.cachingEnabled = true; this.cachingEnabled = true;
this.currentValue = computeValueAt(Utils.getMillisecondsNow()); this.currentValue = computeValueAt(Utils.getMillisecondsNow());
} }
@Override
public void init(final Indicator indicatorDefinition, final Long connectionId, final boolean cachingEnabled) {
super.init(indicatorDefinition, connectionId, cachingEnabled);
try {
indicatorDefinition
.getThresholds()
.stream()
.sorted(Comparator.reverseOrder())
.findFirst()
.ifPresent(t -> this.pingErrorThreshold = t.value.longValue());
} catch (final Exception e) {
log.error("Failed to initialize pingErrorThreshold: {}", e.getMessage());
this.pingErrorThreshold = Constants.SECOND_IN_MILLIS * 5;
}
}
@Override @Override
public IndicatorType getType() { public IndicatorType getType() {
return IndicatorType.LAST_PING; return IndicatorType.LAST_PING;
@ -47,4 +78,37 @@ public final class PingIntervalClientIndicator extends AbstractPingIndicator {
} }
@Override
public ClientEventRecord updateLogEvent() {
final long now = DateTime.now(DateTimeZone.UTC).getMillis();
final long value = now - (long) super.currentValue;
if (this.isOnError) {
if (this.pingErrorThreshold > value) {
this.isOnError = false;
return new ClientEventRecord(
null,
this.connectionId,
EventType.INFO_LOG.id,
now,
now,
new BigDecimal(value),
"Client Ping Back To Normal");
}
} else {
if (this.pingErrorThreshold < value) {
this.isOnError = true;
return new ClientEventRecord(
null,
this.connectionId,
EventType.ERROR_LOG.id,
now,
now,
new BigDecimal(value),
"Missing Client Ping");
}
}
return null;
}
} }

View file

@ -9,11 +9,14 @@
package ch.ethz.seb.sebserver.webservice.servicelayer.session.impl; package ch.ethz.seb.sebserver.webservice.servicelayer.session.impl;
import java.security.Principal; import java.security.Principal;
import java.util.Objects;
import java.util.UUID; import java.util.UUID;
import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
import org.springframework.cache.Cache;
import org.springframework.cache.CacheManager;
import org.springframework.context.annotation.Lazy; import org.springframework.context.annotation.Lazy;
import org.springframework.stereotype.Service; import org.springframework.stereotype.Service;
@ -43,6 +46,7 @@ public class SebClientConnectionServiceImpl implements SebClientConnectionServic
private final ExamSessionService examSessionService; private final ExamSessionService examSessionService;
private final ExamSessionCacheService examSessionCacheService; private final ExamSessionCacheService examSessionCacheService;
private final CacheManager cacheManager;
private final EventHandlingStrategy eventHandlingStrategy; private final EventHandlingStrategy eventHandlingStrategy;
private final ClientConnectionDAO clientConnectionDAO; private final ClientConnectionDAO clientConnectionDAO;
private final PingHandlingStrategy pingHandlingStrategy; private final PingHandlingStrategy pingHandlingStrategy;
@ -52,6 +56,7 @@ public class SebClientConnectionServiceImpl implements SebClientConnectionServic
protected SebClientConnectionServiceImpl( protected SebClientConnectionServiceImpl(
final ExamSessionService examSessionService, final ExamSessionService examSessionService,
final ExamSessionCacheService examSessionCacheService, final ExamSessionCacheService examSessionCacheService,
final CacheManager cacheManager,
final ClientConnectionDAO clientConnectionDAO, final ClientConnectionDAO clientConnectionDAO,
final EventHandlingStrategyFactory eventHandlingStrategyFactory, final EventHandlingStrategyFactory eventHandlingStrategyFactory,
final PingHandlingStrategyFactory pingHandlingStrategyFactory, final PingHandlingStrategyFactory pingHandlingStrategyFactory,
@ -60,6 +65,7 @@ public class SebClientConnectionServiceImpl implements SebClientConnectionServic
this.examSessionService = examSessionService; this.examSessionService = examSessionService;
this.examSessionCacheService = examSessionCacheService; this.examSessionCacheService = examSessionCacheService;
this.cacheManager = cacheManager;
this.clientConnectionDAO = clientConnectionDAO; this.clientConnectionDAO = clientConnectionDAO;
this.pingHandlingStrategy = pingHandlingStrategyFactory.get(); this.pingHandlingStrategy = pingHandlingStrategyFactory.get();
this.eventHandlingStrategy = eventHandlingStrategyFactory.get(); this.eventHandlingStrategy = eventHandlingStrategyFactory.get();
@ -382,6 +388,32 @@ public class SebClientConnectionServiceImpl implements SebClientConnectionServic
}); });
} }
@Override
public void updatePingEvents() {
try {
final Cache cache = this.cacheManager.getCache(ExamSessionCacheService.CACHE_NAME_ACTIVE_CLIENT_CONNECTION);
this.examSessionService
.getExamDAO()
.allRunningExamIds()
.getOrThrow()
.stream()
.flatMap(examId -> this.clientConnectionDAO
.getConnectionTokens(examId)
.getOrThrow()
.stream())
.map(token -> cache.get(token, ClientConnectionDataInternal.class))
.filter(Objects::nonNull)
.flatMap(connection -> connection.pingMappings.stream())
.map(ping -> ping.updateLogEvent())
.filter(Objects::nonNull)
.forEach(this.eventHandlingStrategy::accept);
} catch (final Exception e) {
log.error("Failed to update ping events: ", e);
}
}
@Override @Override
public String notifyPing( public String notifyPing(
final String connectionToken, final String connectionToken,