From 3ce025c4b1ba9d79e0b10f9d9032c418199fa65e Mon Sep 17 00:00:00 2001 From: anhefti Date: Fri, 2 Jun 2023 10:52:52 +0200 Subject: [PATCH] SEBSERV-449 better timeouts and name search on moodle side --- .../ClientHttpRequestFactoryService.java | 2 +- .../lms/impl/QuizLookupServiceImpl.java | 28 +++++++++++++------ .../plugin/MoodlePluginCourseAccess.java | 18 +++++++++--- .../config/application-dev-ws.properties | 4 +-- .../config/application-ws.properties | 1 + 5 files changed, 38 insertions(+), 15 deletions(-) diff --git a/src/main/java/ch/ethz/seb/sebserver/ClientHttpRequestFactoryService.java b/src/main/java/ch/ethz/seb/sebserver/ClientHttpRequestFactoryService.java index 5272843e..3fd16e75 100644 --- a/src/main/java/ch/ethz/seb/sebserver/ClientHttpRequestFactoryService.java +++ b/src/main/java/ch/ethz/seb/sebserver/ClientHttpRequestFactoryService.java @@ -76,7 +76,7 @@ public class ClientHttpRequestFactoryService { final ClientCredentialService clientCredentialService, @Value("${sebserver.http.client.connect-timeout:15000}") final int connectTimeout, @Value("${sebserver.http.client.connection-request-timeout:20000}") final int connectionRequestTimeout, - @Value("${sebserver.http.client.read-timeout:20000}") final int readTimeout) { + @Value("${sebserver.http.client.read-timeout:30000}") final int readTimeout) { this.environment = environment; this.clientCredentialService = clientCredentialService; diff --git a/src/main/java/ch/ethz/seb/sebserver/webservice/servicelayer/lms/impl/QuizLookupServiceImpl.java b/src/main/java/ch/ethz/seb/sebserver/webservice/servicelayer/lms/impl/QuizLookupServiceImpl.java index e74deed9..cc83fe2e 100644 --- a/src/main/java/ch/ethz/seb/sebserver/webservice/servicelayer/lms/impl/QuizLookupServiceImpl.java +++ b/src/main/java/ch/ethz/seb/sebserver/webservice/servicelayer/lms/impl/QuizLookupServiceImpl.java @@ -22,6 +22,7 @@ import java.util.stream.Collectors; import org.joda.time.DateTime; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Value; import org.springframework.context.annotation.Lazy; import org.springframework.core.env.Environment; import org.springframework.stereotype.Service; @@ -53,16 +54,19 @@ public class QuizLookupServiceImpl implements QuizLookupService { private final UserService userService; private final LmsSetupDAO lmsSetupDAO; private final AsyncRunner asyncRunner; + private final long fetchedDataValiditySeconds; public QuizLookupServiceImpl( final UserService userService, final LmsSetupDAO lmsSetupDAO, final AsyncService asyncService, - final Environment environment) { + final Environment environment, + @Value("${sebserver.webservice.lms.datafetch.validity.seconds:600}") final long fetchedDataValiditySeconds) { this.userService = userService; this.lmsSetupDAO = lmsSetupDAO; this.asyncRunner = asyncService.getAsyncRunner(); + this.fetchedDataValiditySeconds = fetchedDataValiditySeconds; } @Override @@ -158,7 +162,10 @@ public class QuizLookupServiceImpl implements QuizLookupService { } if (!asyncLookup.isValid(filterMap)) { - this.lookups.remove(userId); + final AsyncLookup removed = this.lookups.remove(userId); + if (removed != null) { + removed.cancel(); + } this.createNewAsyncLookup(userId, filterMap, lmsAPITemplateSupplier); } @@ -198,7 +205,12 @@ public class QuizLookupServiceImpl implements QuizLookupService { } final LookupFilterCriteria criteria = new LookupFilterCriteria(filterMap); - final AsyncLookup asyncLookup = new AsyncLookup(userInstitutionId, userId, criteria, buffers); + final AsyncLookup asyncLookup = new AsyncLookup( + userInstitutionId, + userId, + criteria, + buffers, + this.fetchedDataValiditySeconds); if (log.isDebugEnabled()) { log.debug("Create new AsyncLookup: user={} criteria={}", userId, criteria); @@ -278,18 +290,21 @@ public class QuizLookupServiceImpl implements QuizLookupService { final Collection asyncBuffers; final long timeCreated; long timeCompleted = Long.MAX_VALUE; + private final long fetchedDataValiditySeconds; public AsyncLookup( final long institutionId, final String userId, final LookupFilterCriteria lookupFilterCriteria, - final Collection asyncBuffers) { + final Collection asyncBuffers, + final long fetchedDataValiditySeconds) { this.institutionId = institutionId; this.userId = userId; this.lookupFilterCriteria = lookupFilterCriteria; this.asyncBuffers = asyncBuffers; this.timeCreated = Utils.getMillisecondsNow(); + this.fetchedDataValiditySeconds = fetchedDataValiditySeconds; } LookupResult getAvailable() { @@ -307,10 +322,7 @@ public class QuizLookupServiceImpl implements QuizLookupService { boolean isUpToDate() { final long now = Utils.getMillisecondsNow(); - if (now - this.timeCreated > 5 * Constants.MINUTE_IN_MILLIS) { - return false; - } - if (now - this.timeCompleted > Constants.MINUTE_IN_MILLIS) { + if (now - this.timeCreated > this.fetchedDataValiditySeconds * Constants.SECOND_IN_MILLIS) { return false; } return true; diff --git a/src/main/java/ch/ethz/seb/sebserver/webservice/servicelayer/lms/impl/moodle/plugin/MoodlePluginCourseAccess.java b/src/main/java/ch/ethz/seb/sebserver/webservice/servicelayer/lms/impl/moodle/plugin/MoodlePluginCourseAccess.java index 2e3a464e..9148964e 100644 --- a/src/main/java/ch/ethz/seb/sebserver/webservice/servicelayer/lms/impl/moodle/plugin/MoodlePluginCourseAccess.java +++ b/src/main/java/ch/ethz/seb/sebserver/webservice/servicelayer/lms/impl/moodle/plugin/MoodlePluginCourseAccess.java @@ -61,6 +61,7 @@ import ch.ethz.seb.sebserver.webservice.servicelayer.lms.impl.moodle.MoodleUtils import ch.ethz.seb.sebserver.webservice.servicelayer.lms.impl.moodle.MoodleUtils.Courses; import ch.ethz.seb.sebserver.webservice.servicelayer.lms.impl.moodle.MoodleUtils.CoursesPlugin; import ch.ethz.seb.sebserver.webservice.servicelayer.lms.impl.moodle.MoodleUtils.MoodleUserDetails; +import io.micrometer.core.instrument.util.StringUtils; public class MoodlePluginCourseAccess extends AbstractCachedCourseAccess implements CourseAccessAPI { @@ -118,7 +119,7 @@ public class MoodlePluginCourseAccess extends AbstractCachedCourseAccess impleme environment.getProperty( "sebserver.webservice.circuitbreaker.moodleRestCall.blockingTime", Long.class, - Constants.SECOND_IN_MILLIS * 20), + Constants.SECOND_IN_MILLIS * 30), environment.getProperty( "sebserver.webservice.circuitbreaker.moodleRestCall.timeToRecover", Long.class, @@ -184,10 +185,11 @@ public class MoodlePluginCourseAccess extends AbstractCachedCourseAccess impleme quizFromTime = DateTime.now(DateTimeZone.UTC).minusYears(this.cutoffTimeOffset); } final Predicate quizFilter = LmsAPIService.quizFilterPredicate(filterMap); + final String quizName = filterMap.getQuizName(); while (!asyncQuizFetchBuffer.finished && !asyncQuizFetchBuffer.canceled) { try { - fetchQuizzesPage(page, quizFromTime, asyncQuizFetchBuffer, quizFilter); + fetchQuizzesPage(page, quizFromTime, quizName, asyncQuizFetchBuffer, quizFilter); page++; } catch (final Exception e) { log.error("Unexpected error while trying to fetch moodle quiz page: {}", page, e); @@ -371,6 +373,7 @@ public class MoodlePluginCourseAccess extends AbstractCachedCourseAccess impleme private void fetchQuizzesPage( final int page, final DateTime quizFromTime, + final String nameCondition, final AsyncQuizFetchBuffer asyncQuizFetchBuffer, final Predicate quizFilter) throws JsonParseException, JsonMappingException, IOException { @@ -382,7 +385,7 @@ public class MoodlePluginCourseAccess extends AbstractCachedCourseAccess impleme : lmsSetup.lmsApiUrl + Constants.URL_PATH_SEPARATOR + MOODLE_QUIZ_START_URL_PATH; final Collection fetchCoursesPage = - fetchCoursesPage(restTemplate, quizFromTime, page, this.pageSize); + fetchCoursesPage(restTemplate, quizFromTime, nameCondition, page, this.pageSize); // finish if page is empty (no courses left if (fetchCoursesPage.isEmpty()) { asyncQuizFetchBuffer.finish(); @@ -408,6 +411,7 @@ public class MoodlePluginCourseAccess extends AbstractCachedCourseAccess impleme private Collection fetchCoursesPage( final MoodleAPIRestTemplate restTemplate, final DateTime quizFromTime, + final String nameCondition, final int page, final int size) throws JsonParseException, JsonMappingException, IOException { @@ -422,13 +426,19 @@ public class MoodlePluginCourseAccess extends AbstractCachedCourseAccess impleme final long defaultCutOff = Utils.toUnixTimeInSeconds( DateTime.now(DateTimeZone.UTC).minusYears(this.cutoffTimeOffset)); final long cutoffDate = (filterDate < defaultCutOff) ? filterDate : defaultCutOff; - final String sqlCondition = String.format( + String sqlCondition = String.format( SQL_CONDITION_TEMPLATE, String.valueOf(cutoffDate), String.valueOf(filterDate)); final String fromElement = String.valueOf(page * size); final LinkedMultiValueMap attributes = new LinkedMultiValueMap<>(); + if (StringUtils.isNotBlank(nameCondition)) { + sqlCondition = sqlCondition + " AND (m.name LIKE '" + + Utils.toSQLWildcard(nameCondition) + + "')"; + } + // Note: courseid[]=0 means all courses. Moodle don't like empty parameter attributes.add(PARAM_COURSE_ID_ARRAY, "0"); attributes.add(PARAM_SQL_CONDITIONS, sqlCondition); diff --git a/src/main/resources/config/application-dev-ws.properties b/src/main/resources/config/application-dev-ws.properties index 25b871d9..b49bad3b 100644 --- a/src/main/resources/config/application-dev-ws.properties +++ b/src/main/resources/config/application-dev-ws.properties @@ -18,7 +18,7 @@ spring.datasource.hikari.leakDetectionThreshold=2000 sebserver.http.client.connect-timeout=15000 sebserver.http.client.connection-request-timeout=10000 -sebserver.http.client.read-timeout=20000 +sebserver.http.client.read-timeout=30000 sebserver.webservice.distributed.updateInterval=1000 sebserver.webservice.distributed.connectionUpdate=2000 sebserver.webservice.clean-db-on-startup=false @@ -52,7 +52,7 @@ sebserver.webservice.api.pagination.maxPageSize=500 sebserver.webservice.lms.openedx.api.token.request.paths=/oauth2/access_token sebserver.webservice.lms.moodle.api.token.request.paths= sebserver.webservice.lms.address.alias=lms.mockup.com=lms.address.alias -sebserver.webservice.cache.moodle.course.pageSize=10 +sebserver.webservice.cache.moodle.course.pageSize=250 springdoc.api-docs.enabled=true springdoc.swagger-ui.enabled=true diff --git a/src/main/resources/config/application-ws.properties b/src/main/resources/config/application-ws.properties index acafe27a..f215b762 100644 --- a/src/main/resources/config/application-ws.properties +++ b/src/main/resources/config/application-ws.properties @@ -83,6 +83,7 @@ sebserver.webservice.lms.moodle.prependShortCourseName=true sebserver.webservice.lms.moodle.fetch.cutoffdate.yearsBeforeNow=2 sebserver.webservice.lms.olat.sendAdditionalAttributesWithRestriction=false sebserver.webservice.lms.address.alias= +sebserver.webservice.lms.datafetch.validity.seconds=600 sebserver.webservice.proctoring.resetBroadcastOnLeav=true sebserver.webservice.proctoring.zoom.enableWaitingRoom=false