From 82222b3fae9e5842adc94dfb9d0d18ce9cc22c1e Mon Sep 17 00:00:00 2001 From: anhefti Date: Thu, 25 Nov 2021 14:13:59 +0100 Subject: [PATCH 1/7] fixed missing ping problem in distributed environments --- .../servicelayer/session/impl/ExamSessionControlTask.java | 3 ++- .../session/impl/indicator/PingIntervalClientIndicator.java | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/main/java/ch/ethz/seb/sebserver/webservice/servicelayer/session/impl/ExamSessionControlTask.java b/src/main/java/ch/ethz/seb/sebserver/webservice/servicelayer/session/impl/ExamSessionControlTask.java index 2e1106ee..237e4910 100644 --- a/src/main/java/ch/ethz/seb/sebserver/webservice/servicelayer/session/impl/ExamSessionControlTask.java +++ b/src/main/java/ch/ethz/seb/sebserver/webservice/servicelayer/session/impl/ExamSessionControlTask.java @@ -112,11 +112,12 @@ class ExamSessionControlTask implements DisposableBean { @Scheduled(fixedRateString = "${sebserver.webservice.api.seb.lostping.update:5000}") public void examSessionUpdateTask() { + this.sebClientConnectionService.updatePingEvents(); + if (!this.webserviceInfoDAO.isMaster(this.webserviceInfo.getWebserviceUUID())) { return; } - this.sebClientConnectionService.updatePingEvents(); this.sebClientConnectionService.cleanupInstructions(); this.examProcotringRoomService.updateProctoringCollectingRooms(); } diff --git a/src/main/java/ch/ethz/seb/sebserver/webservice/servicelayer/session/impl/indicator/PingIntervalClientIndicator.java b/src/main/java/ch/ethz/seb/sebserver/webservice/servicelayer/session/impl/indicator/PingIntervalClientIndicator.java index f22b7b53..a5bb26fa 100644 --- a/src/main/java/ch/ethz/seb/sebserver/webservice/servicelayer/session/impl/indicator/PingIntervalClientIndicator.java +++ b/src/main/java/ch/ethz/seb/sebserver/webservice/servicelayer/session/impl/indicator/PingIntervalClientIndicator.java @@ -83,7 +83,7 @@ public final class PingIntervalClientIndicator extends AbstractPingIndicator { this.missingPing = this.pingErrorThreshold < value; } catch (final Exception e) { log.error("Failed to initialize missingPing: {}", e.getMessage()); - this.missingPing = false; + this.missingPing = true; } } From 03a0e03dd6765e968958358bc3922d4b3d66d78a Mon Sep 17 00:00:00 2001 From: anhefti Date: Thu, 25 Nov 2021 14:24:02 +0100 Subject: [PATCH 2/7] prepare for release v1.2.5 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 730c5098..d65d6228 100644 --- a/pom.xml +++ b/pom.xml @@ -18,7 +18,7 @@ jar - 1.2.4-SNAPSHOT + 1.2.5 ${sebserver-version} ${sebserver-version} UTF-8 From f75a03ff4db9286738b114eb3f107ee90b11d20b Mon Sep 17 00:00:00 2001 From: anhefti Date: Thu, 25 Nov 2021 16:48:03 +0100 Subject: [PATCH 3/7] SNAPSHOT --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index d65d6228..87c5557a 100644 --- a/pom.xml +++ b/pom.xml @@ -18,7 +18,7 @@ jar - 1.2.5 + 1.2.5-SNAPSHOT ${sebserver-version} ${sebserver-version} UTF-8 From f44c82bde3671ebde943181134c270984a6551be Mon Sep 17 00:00:00 2001 From: anhefti Date: Wed, 1 Dec 2021 11:03:03 +0100 Subject: [PATCH 4/7] various fixes and improvements found by log analysis --- .../webservice/AdminUserInitializer.java | 25 ++++++------ .../sebserver/webservice/WebserviceInit.java | 2 +- .../webservice/servicelayer/dao/UserDAO.java | 6 +++ .../servicelayer/dao/impl/UserDAOImpl.java | 15 +++++++ .../exam/impl/ExamAdminServiceImpl.java | 4 +- .../AbstractLogLevelCountIndicator.java | 2 + .../indicator/AbstractLogNumberIndicator.java | 4 +- .../impl/indicator/AbstractPingIndicator.java | 40 +++++-------------- .../impl/indicator/DistributedPingCache.java | 3 +- .../PingIntervalClientIndicator.java | 4 +- .../weblayer/api/APIExceptionHandler.java | 10 +++++ .../api/ExamMonitoringController.java | 37 +++++++++++++---- .../weblayer/api/ExamNotRunningException.java | 27 +++++++++++++ .../config/application-ws.properties | 2 +- 14 files changed, 123 insertions(+), 58 deletions(-) create mode 100644 src/main/java/ch/ethz/seb/sebserver/webservice/weblayer/api/ExamNotRunningException.java diff --git a/src/main/java/ch/ethz/seb/sebserver/webservice/AdminUserInitializer.java b/src/main/java/ch/ethz/seb/sebserver/webservice/AdminUserInitializer.java index 93553f67..cbd73b8a 100644 --- a/src/main/java/ch/ethz/seb/sebserver/webservice/AdminUserInitializer.java +++ b/src/main/java/ch/ethz/seb/sebserver/webservice/AdminUserInitializer.java @@ -8,6 +8,17 @@ package ch.ethz.seb.sebserver.webservice; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashSet; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.security.crypto.password.PasswordEncoder; +import org.springframework.stereotype.Component; + import ch.ethz.seb.sebserver.SEBServerInit; import ch.ethz.seb.sebserver.WebSecurityConfig; import ch.ethz.seb.sebserver.gbl.client.ClientCredentialServiceImpl; @@ -20,16 +31,6 @@ import ch.ethz.seb.sebserver.webservice.servicelayer.authorization.impl.SEBServe import ch.ethz.seb.sebserver.webservice.servicelayer.dao.FilterMap; import ch.ethz.seb.sebserver.webservice.servicelayer.dao.InstitutionDAO; import ch.ethz.seb.sebserver.webservice.servicelayer.dao.UserDAO; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.beans.factory.annotation.Qualifier; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.security.crypto.password.PasswordEncoder; -import org.springframework.stereotype.Component; - -import java.util.Arrays; -import java.util.Collections; -import java.util.HashSet; @Component @WebServiceProfile @@ -69,7 +70,7 @@ class AdminUserInitializer { try { log.debug("Create initial admin account is switched on. Check database if exists..."); - final Result byUsername = this.userDAO.sebServerUserByUsername(this.adminName); + final Result byUsername = this.userDAO.sebServerAdminByUsername(this.adminName); if (byUsername.hasValue()) { log.debug("Initial admin account already exists. Check if the password must be reset..."); @@ -130,7 +131,7 @@ class AdminUserInitializer { return account; }) .getOrThrow(); - } + } } catch (final Exception e) { SEBServerInit.INIT_LOGGER.error("---->"); SEBServerInit.INIT_LOGGER.error("----> SEB Server initial admin-account creation failed: ", e); diff --git a/src/main/java/ch/ethz/seb/sebserver/webservice/WebserviceInit.java b/src/main/java/ch/ethz/seb/sebserver/webservice/WebserviceInit.java index 91322c60..2ab1f59d 100644 --- a/src/main/java/ch/ethz/seb/sebserver/webservice/WebserviceInit.java +++ b/src/main/java/ch/ethz/seb/sebserver/webservice/WebserviceInit.java @@ -94,7 +94,7 @@ public class WebserviceInit implements ApplicationListener Ping update time: {}", this.environment.getProperty("sebserver.webservice.distributed.pingUpdate")); SEBServerInit.INIT_LOGGER.info("------> Connection update time: {}", - this.environment.getProperty("sebserver.webservice.distributed.connectionUpdate")); + this.environment.getProperty("sebserver.webservice.distributed.connectionUpdate", "2000")); } try { diff --git a/src/main/java/ch/ethz/seb/sebserver/webservice/servicelayer/dao/UserDAO.java b/src/main/java/ch/ethz/seb/sebserver/webservice/servicelayer/dao/UserDAO.java index c6a640c4..32d5bcb6 100644 --- a/src/main/java/ch/ethz/seb/sebserver/webservice/servicelayer/dao/UserDAO.java +++ b/src/main/java/ch/ethz/seb/sebserver/webservice/servicelayer/dao/UserDAO.java @@ -49,6 +49,12 @@ public interface UserDAO extends ActivatableEntityDAO, BulkAc * @return a Result of SEBServerUser for specified username. Or an exception result on error case */ Result sebServerUserByUsername(String username); + /** Use this to get the SEBServerUser admin principal for a given username. + * + * @param username The username of the user to get SEBServerUser from + * @return a Result of SEBServerUser for specified username. Or an exception result on error case */ + Result sebServerAdminByUsername(String username); + /** Use this to get a Collection containing EntityKey's of all entities that belongs to a given User. * * @param uuid The UUID of the user diff --git a/src/main/java/ch/ethz/seb/sebserver/webservice/servicelayer/dao/impl/UserDAOImpl.java b/src/main/java/ch/ethz/seb/sebserver/webservice/servicelayer/dao/impl/UserDAOImpl.java index 8d1a6ba3..fec7fb1a 100644 --- a/src/main/java/ch/ethz/seb/sebserver/webservice/servicelayer/dao/impl/UserDAOImpl.java +++ b/src/main/java/ch/ethz/seb/sebserver/webservice/servicelayer/dao/impl/UserDAOImpl.java @@ -127,6 +127,21 @@ public class UserDAOImpl implements UserDAO { .flatMap(this::sebServerUserFromRecord); } + @Override + @Transactional(readOnly = true) + public Result sebServerAdminByUsername(final String username) { + return getSingleResource( + username, + this.userRecordMapper + .selectByExample() + .where(UserRecordDynamicSqlSupport.username, isEqualTo(username)) + .and(UserRecordDynamicSqlSupport.active, + isEqualTo(BooleanUtils.toInteger(true))) + .build() + .execute()) + .flatMap(this::sebServerUserFromRecord); + } + @Override @Transactional(readOnly = true) public Result> all(final Long institutionId, final Boolean active) { diff --git a/src/main/java/ch/ethz/seb/sebserver/webservice/servicelayer/exam/impl/ExamAdminServiceImpl.java b/src/main/java/ch/ethz/seb/sebserver/webservice/servicelayer/exam/impl/ExamAdminServiceImpl.java index f983f507..36e3e8fd 100644 --- a/src/main/java/ch/ethz/seb/sebserver/webservice/servicelayer/exam/impl/ExamAdminServiceImpl.java +++ b/src/main/java/ch/ethz/seb/sebserver/webservice/servicelayer/exam/impl/ExamAdminServiceImpl.java @@ -311,7 +311,9 @@ public class ExamAdminServiceImpl implements ExamAdminService { examId, ProctoringServiceSettings.ATTR_ENABLE_PROCTORING) .map(rec -> BooleanUtils.toBoolean(rec.getValue())) - .onError(error -> log.error("Failed to verify proctoring enabled for exam: {}", examId, error)); + .onError(error -> log.warn("Failed to verify proctoring enabled for exam: {}, {}", + examId, + error.getMessage())); if (result.hasError()) { return Result.of(false); } diff --git a/src/main/java/ch/ethz/seb/sebserver/webservice/servicelayer/session/impl/indicator/AbstractLogLevelCountIndicator.java b/src/main/java/ch/ethz/seb/sebserver/webservice/servicelayer/session/impl/indicator/AbstractLogLevelCountIndicator.java index 50cc95eb..92b3f77c 100644 --- a/src/main/java/ch/ethz/seb/sebserver/webservice/servicelayer/session/impl/indicator/AbstractLogLevelCountIndicator.java +++ b/src/main/java/ch/ethz/seb/sebserver/webservice/servicelayer/session/impl/indicator/AbstractLogLevelCountIndicator.java @@ -61,6 +61,8 @@ public abstract class AbstractLogLevelCountIndicator extends AbstractLogIndicato return super.currentValue; } + // TODO do this within a better reactive way like ping updates + try { final Long errors = this.clientEventRecordMapper diff --git a/src/main/java/ch/ethz/seb/sebserver/webservice/servicelayer/session/impl/indicator/AbstractLogNumberIndicator.java b/src/main/java/ch/ethz/seb/sebserver/webservice/servicelayer/session/impl/indicator/AbstractLogNumberIndicator.java index 5e9f381c..3fa51e81 100644 --- a/src/main/java/ch/ethz/seb/sebserver/webservice/servicelayer/session/impl/indicator/AbstractLogNumberIndicator.java +++ b/src/main/java/ch/ethz/seb/sebserver/webservice/servicelayer/session/impl/indicator/AbstractLogNumberIndicator.java @@ -67,9 +67,9 @@ public abstract class AbstractLogNumberIndicator extends AbstractLogIndicator { return super.currentValue; } - try { + // TODO do this within a better reactive way like ping updates - System.out.println("************** loadFromPersistent"); + try { final List execute = this.clientEventRecordMapper.selectByExample() .where(ClientEventRecordDynamicSqlSupport.clientConnectionId, isEqualTo(this.connectionId)) diff --git a/src/main/java/ch/ethz/seb/sebserver/webservice/servicelayer/session/impl/indicator/AbstractPingIndicator.java b/src/main/java/ch/ethz/seb/sebserver/webservice/servicelayer/session/impl/indicator/AbstractPingIndicator.java index 0005171b..775be7db 100644 --- a/src/main/java/ch/ethz/seb/sebserver/webservice/servicelayer/session/impl/indicator/AbstractPingIndicator.java +++ b/src/main/java/ch/ethz/seb/sebserver/webservice/servicelayer/session/impl/indicator/AbstractPingIndicator.java @@ -19,8 +19,6 @@ import org.springframework.beans.factory.annotation.Qualifier; import ch.ethz.seb.sebserver.gbl.async.AsyncServiceSpringConfig; import ch.ethz.seb.sebserver.gbl.model.exam.Indicator; -import ch.ethz.seb.sebserver.gbl.model.exam.Indicator.IndicatorType; -import ch.ethz.seb.sebserver.gbl.model.session.ClientEvent; import ch.ethz.seb.sebserver.gbl.model.session.ClientEvent.EventType; import ch.ethz.seb.sebserver.gbl.util.Utils; import ch.ethz.seb.sebserver.webservice.datalayer.batis.ClientEventLastPingMapper; @@ -35,12 +33,12 @@ public abstract class AbstractPingIndicator extends AbstractClientIndicator { private final Executor executor; protected final DistributedPingCache distributedPingCache; - //protected Long pingRecord = null; protected PingUpdate pingUpdate = null; protected AbstractPingIndicator( final DistributedPingCache distributedPingCache, @Qualifier(AsyncServiceSpringConfig.EXAM_API_PING_SERVICE_EXECUTOR_BEAN_NAME) final Executor executor) { + super(); this.executor = executor; this.distributedPingCache = distributedPingCache; @@ -81,7 +79,9 @@ public abstract class AbstractPingIndicator extends AbstractClientIndicator { try { this.executor.execute(this.pingUpdate); } catch (final Exception e) { - //log.warn("Failed to schedule ping task: {}" + e.getMessage()); + if (log.isDebugEnabled()) { + log.warn("Failed to schedule ping task: {}" + e.getMessage()); + } } } } @@ -115,30 +115,6 @@ public abstract class AbstractPingIndicator extends AbstractClientIndicator { public abstract ClientEventRecord updateLogEvent(final long now); - @Override - public double computeValueAt(final long timestamp) { - // TODO Auto-generated method stub - return 0; - } - - @Override - public void notifyValueChange(final ClientEvent event) { - // TODO Auto-generated method stub - - } - - @Override - public void notifyValueChange(final ClientEventRecord clientEventRecord) { - // TODO Auto-generated method stub - - } - - @Override - public IndicatorType getType() { - // TODO Auto-generated method stub - return null; - } - static final class PingUpdate implements Runnable { private final ClientEventLastPingMapper clientEventLastPingMapper; @@ -151,8 +127,12 @@ public abstract class AbstractPingIndicator extends AbstractClientIndicator { @Override public void run() { - this.clientEventLastPingMapper - .updatePingTime(this.pingRecord, Utils.getMillisecondsNow()); + try { + this.clientEventLastPingMapper + .updatePingTime(this.pingRecord, Utils.getMillisecondsNow()); + } catch (final Exception e) { + log.error("Failed to update ping: {}", e.getMessage()); + } } } diff --git a/src/main/java/ch/ethz/seb/sebserver/webservice/servicelayer/session/impl/indicator/DistributedPingCache.java b/src/main/java/ch/ethz/seb/sebserver/webservice/servicelayer/session/impl/indicator/DistributedPingCache.java index a80221ca..dd8de44b 100644 --- a/src/main/java/ch/ethz/seb/sebserver/webservice/servicelayer/session/impl/indicator/DistributedPingCache.java +++ b/src/main/java/ch/ethz/seb/sebserver/webservice/servicelayer/session/impl/indicator/DistributedPingCache.java @@ -176,6 +176,7 @@ public class DistributedPingCache implements DisposableBean { public Long getLastPing(final Long pingRecordId, final boolean missing) { try { + Long ping = this.pingCache.get(pingRecordId); if (ping == null && !missing) { @@ -199,7 +200,7 @@ public class DistributedPingCache implements DisposableBean { } private void updatePings() { - + if (this.pingCache.isEmpty()) { return; } diff --git a/src/main/java/ch/ethz/seb/sebserver/webservice/servicelayer/session/impl/indicator/PingIntervalClientIndicator.java b/src/main/java/ch/ethz/seb/sebserver/webservice/servicelayer/session/impl/indicator/PingIntervalClientIndicator.java index a5bb26fa..7f336f88 100644 --- a/src/main/java/ch/ethz/seb/sebserver/webservice/servicelayer/session/impl/indicator/PingIntervalClientIndicator.java +++ b/src/main/java/ch/ethz/seb/sebserver/webservice/servicelayer/session/impl/indicator/PingIntervalClientIndicator.java @@ -111,8 +111,8 @@ public final class PingIntervalClientIndicator extends AbstractPingIndicator { @Override public double getValue() { - final long now = DateTimeUtils.currentTimeMillis(); - return now - super.getValue(); + final double value = super.getValue(); + return DateTimeUtils.currentTimeMillis() - value; } @Override diff --git a/src/main/java/ch/ethz/seb/sebserver/webservice/weblayer/api/APIExceptionHandler.java b/src/main/java/ch/ethz/seb/sebserver/webservice/weblayer/api/APIExceptionHandler.java index 5bf6892e..19af5f4a 100644 --- a/src/main/java/ch/ethz/seb/sebserver/webservice/weblayer/api/APIExceptionHandler.java +++ b/src/main/java/ch/ethz/seb/sebserver/webservice/weblayer/api/APIExceptionHandler.java @@ -177,6 +177,16 @@ public class APIExceptionHandler extends ResponseEntityExceptionHandler { .createErrorResponse(ex.getMessage()); } + @ExceptionHandler(ExamNotRunningException.class) + public ResponseEntity handleExamNotRunning( + final ExamNotRunningException ex, + final WebRequest request) { + + log.warn("{}", ex.getMessage()); + return APIMessage.ErrorMessage.INTEGRITY_VALIDATION + .createErrorResponse(ex.getMessage()); + } + @ExceptionHandler(APIConstraintViolationException.class) public ResponseEntity handleIllegalAPIArgumentException( final APIConstraintViolationException ex, diff --git a/src/main/java/ch/ethz/seb/sebserver/webservice/weblayer/api/ExamMonitoringController.java b/src/main/java/ch/ethz/seb/sebserver/webservice/weblayer/api/ExamMonitoringController.java index 2302d637..5f8ef958 100644 --- a/src/main/java/ch/ethz/seb/sebserver/webservice/weblayer/api/ExamMonitoringController.java +++ b/src/main/java/ch/ethz/seb/sebserver/webservice/weblayer/api/ExamMonitoringController.java @@ -172,8 +172,14 @@ public class ExamMonitoringController { EntityType.EXAM, UserRole.EXAM_SUPPORTER); + // check exam running + final Exam runningExam = this.examSessionService.getRunningExam(examId).getOr(null); + if (runningExam == null) { + throw new ExamNotRunningException("Exam not currently running: " + examId); + } + // check running exam privilege for specified exam - if (!hasRunningExamPrivilege(examId, institutionId)) { + if (!hasRunningExamPrivilege(runningExam, institutionId)) { throw new PermissionDeniedException( EntityType.EXAM, PrivilegeType.READ, @@ -217,8 +223,14 @@ public class ExamMonitoringController { EntityType.EXAM, UserRole.EXAM_SUPPORTER); + // check exam running + final Exam runningExam = this.examSessionService.getRunningExam(examId).getOr(null); + if (runningExam == null) { + throw new ExamNotRunningException("Exam not currently running: " + examId); + } + // check running exam privilege for specified exam - if (!hasRunningExamPrivilege(examId, institutionId)) { + if (!hasRunningExamPrivilege(runningExam, institutionId)) { throw new PermissionDeniedException( EntityType.EXAM, PrivilegeType.READ, @@ -243,6 +255,11 @@ public class ExamMonitoringController { @PathVariable(name = API.PARAM_PARENT_MODEL_ID, required = true) final Long examId, @Valid @RequestBody final ClientInstruction clientInstruction) { + // check exam running + if (!this.examSessionService.isExamRunning(examId)) { + throw new ExamNotRunningException("Exam not currently running: " + examId); + } + this.sebClientInstructionService.registerInstruction(clientInstruction); } @@ -261,6 +278,11 @@ public class ExamMonitoringController { @PathVariable(name = API.PARAM_PARENT_MODEL_ID, required = true) final Long examId, @PathVariable(name = API.EXAM_API_SEB_CONNECTION_TOKEN, required = true) final String connectionToken) { + // check exam running + if (!this.examSessionService.isExamRunning(examId)) { + throw new ExamNotRunningException("Exam not currently running: " + examId); + } + final ClientConnectionData connection = getConnectionDataForSingleConnection( institutionId, examId, @@ -286,6 +308,11 @@ public class ExamMonitoringController { @PathVariable(name = API.PARAM_MODEL_ID, required = true) final Long notificationId, @PathVariable(name = API.EXAM_API_SEB_CONNECTION_TOKEN, required = true) final String connectionToken) { + // check exam running + if (!this.examSessionService.isExamRunning(examId)) { + throw new ExamNotRunningException("Exam not currently running: " + examId); + } + this.sebClientNotificationService.confirmPendingNotification( notificationId, examId, @@ -322,12 +349,6 @@ public class ExamMonitoringController { } } - private boolean hasRunningExamPrivilege(final Long examId, final Long institution) { - return hasRunningExamPrivilege( - this.examSessionService.getRunningExam(examId).getOr(null), - institution); - } - private boolean hasRunningExamPrivilege(final Exam exam, final Long institution) { if (exam == null) { return false; diff --git a/src/main/java/ch/ethz/seb/sebserver/webservice/weblayer/api/ExamNotRunningException.java b/src/main/java/ch/ethz/seb/sebserver/webservice/weblayer/api/ExamNotRunningException.java new file mode 100644 index 00000000..640e1311 --- /dev/null +++ b/src/main/java/ch/ethz/seb/sebserver/webservice/weblayer/api/ExamNotRunningException.java @@ -0,0 +1,27 @@ +/* + * Copyright (c) 2021 ETH Zürich, Educational Development and Technology (LET) + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ + +package ch.ethz.seb.sebserver.webservice.weblayer.api; + +public class ExamNotRunningException extends RuntimeException { + + private static final long serialVersionUID = -2931666431463176875L; + + public ExamNotRunningException() { + super(); + } + + public ExamNotRunningException(final String message, final Throwable cause) { + super(message, cause); + } + + public ExamNotRunningException(final String message) { + super(message); + } + +} diff --git a/src/main/resources/config/application-ws.properties b/src/main/resources/config/application-ws.properties index 88fef4ce..2f583436 100644 --- a/src/main/resources/config/application-ws.properties +++ b/src/main/resources/config/application-ws.properties @@ -38,7 +38,7 @@ sebserver.webservice.internalSecret=${sebserver.password} ### webservice networking sebserver.webservice.forceMaster=false -sebserver.webservice.distributed=false +sebserver.webservice.distributed=true sebserver.webservice.distributed.pingUpdate=3000 sebserver.webservice.http.external.scheme=https sebserver.webservice.http.external.servername= From 614f4430d26ae30c9aa1be207db3525237d07511 Mon Sep 17 00:00:00 2001 From: anhefti Date: Wed, 1 Dec 2021 13:49:48 +0100 Subject: [PATCH 5/7] fixed streaming download for SEB Log export --- .../remote/download/SEBClientLogExport.java | 38 +++++++++------- .../webservice/api/AbstractDownloadCall.java | 43 +++++++++++++++++++ .../webservice/api/AbstractExportCall.java | 1 + .../remote/webservice/api/RestCall.java | 11 +++++ .../api/exam/ExportSEBClientLogs.java | 18 ++------ .../impl/SEBClientEventAdminServiceImpl.java | 2 +- 6 files changed, 81 insertions(+), 32 deletions(-) create mode 100644 src/main/java/ch/ethz/seb/sebserver/gui/service/remote/webservice/api/AbstractDownloadCall.java diff --git a/src/main/java/ch/ethz/seb/sebserver/gui/service/remote/download/SEBClientLogExport.java b/src/main/java/ch/ethz/seb/sebserver/gui/service/remote/download/SEBClientLogExport.java index df46d6d4..ee36957c 100644 --- a/src/main/java/ch/ethz/seb/sebserver/gui/service/remote/download/SEBClientLogExport.java +++ b/src/main/java/ch/ethz/seb/sebserver/gui/service/remote/download/SEBClientLogExport.java @@ -55,26 +55,32 @@ public class SEBClientLogExport extends AbstractDownloadServiceHandler { queryParams.add(param, String.valueOf(request.getParameter(param))); } - final InputStream input = this.restService + this.restService .getBuilder(ExportSEBClientLogs.class) + .withResponseExtractor(response -> { + + try { + final InputStream input = response.getBody(); + IOUtils.copyLarge(input, downloadOut); + } catch (final IOException e) { + log.error( + "Unexpected error while streaming incoming config data from web-service to output-stream of download response: ", + e); + } finally { + try { + downloadOut.flush(); + downloadOut.close(); + } catch (final IOException e) { + log.error("Unexpected error while trying to close download output-stream"); + } + } + + return true; + }) .withQueryParams(queryParams) .call() - .getOrThrow(); + .onError(error -> log.error("Download failed: ", error)); - try { - IOUtils.copyLarge(input, downloadOut); - } catch (final IOException e) { - log.error( - "Unexpected error while streaming incoming config data from web-service to output-stream of download response: ", - e); - } finally { - try { - downloadOut.flush(); - downloadOut.close(); - } catch (final IOException e) { - log.error("Unexpected error while trying to close download output-stream"); - } - } } } diff --git a/src/main/java/ch/ethz/seb/sebserver/gui/service/remote/webservice/api/AbstractDownloadCall.java b/src/main/java/ch/ethz/seb/sebserver/gui/service/remote/webservice/api/AbstractDownloadCall.java new file mode 100644 index 00000000..3e2afb06 --- /dev/null +++ b/src/main/java/ch/ethz/seb/sebserver/gui/service/remote/webservice/api/AbstractDownloadCall.java @@ -0,0 +1,43 @@ +/* + * Copyright (c) 2021 ETH Zürich, Educational Development and Technology (LET) + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ + +package ch.ethz.seb.sebserver.gui.service.remote.webservice.api; + +import org.springframework.http.HttpMethod; +import org.springframework.http.MediaType; +import org.springframework.http.client.ClientHttpRequest; + +import com.fasterxml.jackson.core.type.TypeReference; + +import ch.ethz.seb.sebserver.gbl.util.Result; + +public class AbstractDownloadCall extends RestCall { + + protected AbstractDownloadCall( + final MediaType contentType, + final String path) { + + super(new RestCall.TypeKey<>(CallType.UNDEFINED, null, new TypeReference() { + }), HttpMethod.GET, contentType, path); + } + + @Override + protected Result exchange(final RestCallBuilder builder) { + + return Result.tryCatch(() -> builder + .getRestTemplate() + .execute( + builder.buildURI(), + this.httpMethod, + (final ClientHttpRequest requestCallback) -> { + }, + builder.getResponseExtractor(), + builder.getURIVariables())); + } + +} diff --git a/src/main/java/ch/ethz/seb/sebserver/gui/service/remote/webservice/api/AbstractExportCall.java b/src/main/java/ch/ethz/seb/sebserver/gui/service/remote/webservice/api/AbstractExportCall.java index 632529ac..7f20b564 100644 --- a/src/main/java/ch/ethz/seb/sebserver/gui/service/remote/webservice/api/AbstractExportCall.java +++ b/src/main/java/ch/ethz/seb/sebserver/gui/service/remote/webservice/api/AbstractExportCall.java @@ -17,6 +17,7 @@ import org.springframework.http.client.ClientHttpRequest; import ch.ethz.seb.sebserver.gbl.util.Result; +@Deprecated(since = "1.2. : This is not streaming correctly. Use AbstractDownloadCall instead") public abstract class AbstractExportCall extends RestCall { protected AbstractExportCall( diff --git a/src/main/java/ch/ethz/seb/sebserver/gui/service/remote/webservice/api/RestCall.java b/src/main/java/ch/ethz/seb/sebserver/gui/service/remote/webservice/api/RestCall.java index 790963fa..89e7ef7f 100644 --- a/src/main/java/ch/ethz/seb/sebserver/gui/service/remote/webservice/api/RestCall.java +++ b/src/main/java/ch/ethz/seb/sebserver/gui/service/remote/webservice/api/RestCall.java @@ -29,6 +29,7 @@ import org.springframework.http.ResponseEntity; import org.springframework.util.LinkedMultiValueMap; import org.springframework.util.MultiValueMap; import org.springframework.web.client.ResourceAccessException; +import org.springframework.web.client.ResponseExtractor; import org.springframework.web.client.RestClientResponseException; import org.springframework.web.client.RestTemplate; import org.springframework.web.util.UriComponentsBuilder; @@ -224,6 +225,7 @@ public abstract class RestCall { private final HttpHeaders httpHeaders; private String body = null; private InputStream streamingBody = null; + private ResponseExtractor responseExtractor = null; private final MultiValueMap queryParams; private final Map uriVariables; @@ -253,6 +255,15 @@ public abstract class RestCall { return this.restTemplate; } + public RestCallBuilder withResponseExtractor(final ResponseExtractor responseExtractor) { + this.responseExtractor = responseExtractor; + return this; + } + + public ResponseExtractor getResponseExtractor() { + return this.responseExtractor; + } + public RestCallBuilder withRestTemplate(final RestTemplate restTemplate) { this.restTemplate = restTemplate; return this; diff --git a/src/main/java/ch/ethz/seb/sebserver/gui/service/remote/webservice/api/exam/ExportSEBClientLogs.java b/src/main/java/ch/ethz/seb/sebserver/gui/service/remote/webservice/api/exam/ExportSEBClientLogs.java index 34aa0a68..8e76abba 100644 --- a/src/main/java/ch/ethz/seb/sebserver/gui/service/remote/webservice/api/exam/ExportSEBClientLogs.java +++ b/src/main/java/ch/ethz/seb/sebserver/gui/service/remote/webservice/api/exam/ExportSEBClientLogs.java @@ -8,33 +8,21 @@ package ch.ethz.seb.sebserver.gui.service.remote.webservice.api.exam; -import java.io.InputStream; - import org.springframework.context.annotation.Lazy; -import org.springframework.http.HttpMethod; import org.springframework.http.MediaType; import org.springframework.stereotype.Component; -import com.fasterxml.jackson.core.type.TypeReference; - import ch.ethz.seb.sebserver.gbl.api.API; -import ch.ethz.seb.sebserver.gbl.api.EntityType; import ch.ethz.seb.sebserver.gbl.profile.GuiProfile; -import ch.ethz.seb.sebserver.gui.service.remote.webservice.api.AbstractExportCall; +import ch.ethz.seb.sebserver.gui.service.remote.webservice.api.AbstractDownloadCall; @Lazy @Component @GuiProfile -public class ExportSEBClientLogs extends AbstractExportCall { +public class ExportSEBClientLogs extends AbstractDownloadCall { public ExportSEBClientLogs() { - super(new TypeKey<>( - CallType.UNDEFINED, - EntityType.CLIENT_EVENT, - new TypeReference() { - }), - HttpMethod.GET, - MediaType.APPLICATION_FORM_URLENCODED, + super(MediaType.APPLICATION_FORM_URLENCODED, API.SEB_CLIENT_EVENT_ENDPOINT + API.SEB_CLIENT_EVENT_EXPORT_PATH_SEGMENT); } diff --git a/src/main/java/ch/ethz/seb/sebserver/webservice/servicelayer/exam/impl/SEBClientEventAdminServiceImpl.java b/src/main/java/ch/ethz/seb/sebserver/webservice/servicelayer/exam/impl/SEBClientEventAdminServiceImpl.java index be4219f0..25ed7f4f 100644 --- a/src/main/java/ch/ethz/seb/sebserver/webservice/servicelayer/exam/impl/SEBClientEventAdminServiceImpl.java +++ b/src/main/java/ch/ethz/seb/sebserver/webservice/servicelayer/exam/impl/SEBClientEventAdminServiceImpl.java @@ -225,7 +225,7 @@ public class SEBClientEventAdminServiceImpl implements SEBClientEventAdminServic private final String sort; private int pageNumber = 1; - private final int pageSize = 1000; + private final int pageSize = 10000; private Collection nextRecords; From 3a65943bde018ff6a1639ae0b7b94ae2f62a7619 Mon Sep 17 00:00:00 2001 From: anhefti Date: Wed, 1 Dec 2021 14:37:44 +0100 Subject: [PATCH 6/7] fixed build --- .../gui/service/remote/webservice/api/AbstractExportCall.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/ch/ethz/seb/sebserver/gui/service/remote/webservice/api/AbstractExportCall.java b/src/main/java/ch/ethz/seb/sebserver/gui/service/remote/webservice/api/AbstractExportCall.java index 7f20b564..6690249e 100644 --- a/src/main/java/ch/ethz/seb/sebserver/gui/service/remote/webservice/api/AbstractExportCall.java +++ b/src/main/java/ch/ethz/seb/sebserver/gui/service/remote/webservice/api/AbstractExportCall.java @@ -17,7 +17,7 @@ import org.springframework.http.client.ClientHttpRequest; import ch.ethz.seb.sebserver.gbl.util.Result; -@Deprecated(since = "1.2. : This is not streaming correctly. Use AbstractDownloadCall instead") +@Deprecated // This is not streaming correctly. Use AbstractDownloadCall instead public abstract class AbstractExportCall extends RestCall { protected AbstractExportCall( From bd045b0d032d7042fb6bd0be6a6ff29d9ff540cb Mon Sep 17 00:00:00 2001 From: anhefti Date: Mon, 6 Dec 2021 10:57:42 +0100 Subject: [PATCH 7/7] SEBSERV-252 fixed CSV export escaping --- src/main/java/ch/ethz/seb/sebserver/gbl/util/Utils.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/main/java/ch/ethz/seb/sebserver/gbl/util/Utils.java b/src/main/java/ch/ethz/seb/sebserver/gbl/util/Utils.java index 4d0c4671..c818f145 100644 --- a/src/main/java/ch/ethz/seb/sebserver/gbl/util/Utils.java +++ b/src/main/java/ch/ethz/seb/sebserver/gbl/util/Utils.java @@ -678,7 +678,11 @@ public final class Utils { if (StringUtils.isBlank(text)) { return StringUtils.EMPTY; } - return Constants.DOUBLE_QUOTE + text.replace("\"", "\"\"") + Constants.DOUBLE_QUOTE; + return Constants.DOUBLE_QUOTE + + text + .replace("\r\n", "\n") + .replace("\"", "\"\"") + + Constants.DOUBLE_QUOTE; } }