SEBSERV-449 better timeouts and name search on moodle side

src/main/java/ch/ethz/seb/sebserver/ClientHttpRequestFactoryService.java

@@ -76,7 +76,7 @@ public class ClientHttpRequestFactoryService {
             final ClientCredentialService clientCredentialService,
             @Value("${sebserver.http.client.connect-timeout:15000}") final int connectTimeout,
             @Value("${sebserver.http.client.connection-request-timeout:20000}") final int connectionRequestTimeout,
-            @Value("${sebserver.http.client.read-timeout:20000}") final int readTimeout) {
+            @Value("${sebserver.http.client.read-timeout:30000}") final int readTimeout) {
 
         this.environment = environment;
         this.clientCredentialService = clientCredentialService;

src/main/java/ch/ethz/seb/sebserver/webservice/servicelayer/lms/impl/QuizLookupServiceImpl.java

@@ -22,6 +22,7 @@ import java.util.stream.Collectors;
 import org.joda.time.DateTime;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.annotation.Value;
 import org.springframework.context.annotation.Lazy;
 import org.springframework.core.env.Environment;
 import org.springframework.stereotype.Service;
@@ -53,16 +54,19 @@ public class QuizLookupServiceImpl implements QuizLookupService {
     private final UserService userService;
     private final LmsSetupDAO lmsSetupDAO;
     private final AsyncRunner asyncRunner;
+    private final long fetchedDataValiditySeconds;
 
     public QuizLookupServiceImpl(
             final UserService userService,
             final LmsSetupDAO lmsSetupDAO,
             final AsyncService asyncService,
-            final Environment environment) {
+            final Environment environment,
+            @Value("${sebserver.webservice.lms.datafetch.validity.seconds:600}") final long fetchedDataValiditySeconds) {
 
         this.userService = userService;
         this.lmsSetupDAO = lmsSetupDAO;
         this.asyncRunner = asyncService.getAsyncRunner();
+        this.fetchedDataValiditySeconds = fetchedDataValiditySeconds;
     }
 
     @Override
@@ -158,7 +162,10 @@ public class QuizLookupServiceImpl implements QuizLookupService {
         }
 
         if (!asyncLookup.isValid(filterMap)) {
-            this.lookups.remove(userId);
+            final AsyncLookup removed = this.lookups.remove(userId);
+            if (removed != null) {
+                removed.cancel();
+            }
             this.createNewAsyncLookup(userId, filterMap, lmsAPITemplateSupplier);
         }
 
@@ -198,7 +205,12 @@ public class QuizLookupServiceImpl implements QuizLookupService {
             }
 
             final LookupFilterCriteria criteria = new LookupFilterCriteria(filterMap);
-            final AsyncLookup asyncLookup = new AsyncLookup(userInstitutionId, userId, criteria, buffers);
+            final AsyncLookup asyncLookup = new AsyncLookup(
+                    userInstitutionId,
+                    userId,
+                    criteria,
+                    buffers,
+                    this.fetchedDataValiditySeconds);
 
             if (log.isDebugEnabled()) {
                 log.debug("Create new AsyncLookup: user={} criteria={}", userId, criteria);
@@ -278,18 +290,21 @@ public class QuizLookupServiceImpl implements QuizLookupService {
         final Collection<AsyncQuizFetchBuffer> asyncBuffers;
         final long timeCreated;
         long timeCompleted = Long.MAX_VALUE;
+        private final long fetchedDataValiditySeconds;
 
         public AsyncLookup(
                 final long institutionId,
                 final String userId,
                 final LookupFilterCriteria lookupFilterCriteria,
-                final Collection<AsyncQuizFetchBuffer> asyncBuffers) {
+                final Collection<AsyncQuizFetchBuffer> asyncBuffers,
+                final long fetchedDataValiditySeconds) {
 
             this.institutionId = institutionId;
             this.userId = userId;
             this.lookupFilterCriteria = lookupFilterCriteria;
             this.asyncBuffers = asyncBuffers;
             this.timeCreated = Utils.getMillisecondsNow();
+            this.fetchedDataValiditySeconds = fetchedDataValiditySeconds;
         }
 
         LookupResult getAvailable() {
@@ -307,10 +322,7 @@ public class QuizLookupServiceImpl implements QuizLookupService {
 
         boolean isUpToDate() {
             final long now = Utils.getMillisecondsNow();
-            if (now - this.timeCreated > 5 * Constants.MINUTE_IN_MILLIS) {
+            if (now - this.timeCreated > this.fetchedDataValiditySeconds * Constants.SECOND_IN_MILLIS) {
-                return false;
-            }
-            if (now - this.timeCompleted > Constants.MINUTE_IN_MILLIS) {
                 return false;
             }
             return true;

src/main/java/ch/ethz/seb/sebserver/webservice/servicelayer/lms/impl/moodle/plugin/MoodlePluginCourseAccess.java

@@ -61,6 +61,7 @@ import ch.ethz.seb.sebserver.webservice.servicelayer.lms.impl.moodle.MoodleUtils
 import ch.ethz.seb.sebserver.webservice.servicelayer.lms.impl.moodle.MoodleUtils.Courses;
 import ch.ethz.seb.sebserver.webservice.servicelayer.lms.impl.moodle.MoodleUtils.CoursesPlugin;
 import ch.ethz.seb.sebserver.webservice.servicelayer.lms.impl.moodle.MoodleUtils.MoodleUserDetails;
+import io.micrometer.core.instrument.util.StringUtils;
 
 public class MoodlePluginCourseAccess extends AbstractCachedCourseAccess implements CourseAccessAPI {
 
@@ -118,7 +119,7 @@ public class MoodlePluginCourseAccess extends AbstractCachedCourseAccess impleme
                 environment.getProperty(
                         "sebserver.webservice.circuitbreaker.moodleRestCall.blockingTime",
                         Long.class,
-                        Constants.SECOND_IN_MILLIS * 20),
+                        Constants.SECOND_IN_MILLIS * 30),
                 environment.getProperty(
                         "sebserver.webservice.circuitbreaker.moodleRestCall.timeToRecover",
                         Long.class,
@@ -184,10 +185,11 @@ public class MoodlePluginCourseAccess extends AbstractCachedCourseAccess impleme
                 quizFromTime = DateTime.now(DateTimeZone.UTC).minusYears(this.cutoffTimeOffset);
             }
             final Predicate<QuizData> quizFilter = LmsAPIService.quizFilterPredicate(filterMap);
+            final String quizName = filterMap.getQuizName();
 
             while (!asyncQuizFetchBuffer.finished && !asyncQuizFetchBuffer.canceled) {
                 try {
-                    fetchQuizzesPage(page, quizFromTime, asyncQuizFetchBuffer, quizFilter);
+                    fetchQuizzesPage(page, quizFromTime, quizName, asyncQuizFetchBuffer, quizFilter);
                     page++;
                 } catch (final Exception e) {
                     log.error("Unexpected error while trying to fetch moodle quiz page: {}", page, e);
@@ -371,6 +373,7 @@ public class MoodlePluginCourseAccess extends AbstractCachedCourseAccess impleme
     private void fetchQuizzesPage(
             final int page,
             final DateTime quizFromTime,
+            final String nameCondition,
             final AsyncQuizFetchBuffer asyncQuizFetchBuffer,
             final Predicate<QuizData> quizFilter) throws JsonParseException, JsonMappingException, IOException {
 
@@ -382,7 +385,7 @@ public class MoodlePluginCourseAccess extends AbstractCachedCourseAccess impleme
                 : lmsSetup.lmsApiUrl + Constants.URL_PATH_SEPARATOR + MOODLE_QUIZ_START_URL_PATH;
 
         final Collection<CourseData> fetchCoursesPage =
-                fetchCoursesPage(restTemplate, quizFromTime, page, this.pageSize);
+                fetchCoursesPage(restTemplate, quizFromTime, nameCondition, page, this.pageSize);
         // finish if page is empty (no courses left
         if (fetchCoursesPage.isEmpty()) {
             asyncQuizFetchBuffer.finish();
@@ -408,6 +411,7 @@ public class MoodlePluginCourseAccess extends AbstractCachedCourseAccess impleme
     private Collection<CourseData> fetchCoursesPage(
             final MoodleAPIRestTemplate restTemplate,
             final DateTime quizFromTime,
+            final String nameCondition,
             final int page,
             final int size) throws JsonParseException, JsonMappingException, IOException {
 
@@ -422,13 +426,19 @@ public class MoodlePluginCourseAccess extends AbstractCachedCourseAccess impleme
             final long defaultCutOff = Utils.toUnixTimeInSeconds(
                     DateTime.now(DateTimeZone.UTC).minusYears(this.cutoffTimeOffset));
             final long cutoffDate = (filterDate < defaultCutOff) ? filterDate : defaultCutOff;
-            final String sqlCondition = String.format(
+            String sqlCondition = String.format(
                     SQL_CONDITION_TEMPLATE,
                     String.valueOf(cutoffDate),
                     String.valueOf(filterDate));
             final String fromElement = String.valueOf(page * size);
             final LinkedMultiValueMap<String, String> attributes = new LinkedMultiValueMap<>();
 
+            if (StringUtils.isNotBlank(nameCondition)) {
+                sqlCondition = sqlCondition + " AND (m.name LIKE '" +
+                        Utils.toSQLWildcard(nameCondition) +
+                        "')";
+            }
+
             // Note: courseid[]=0 means all courses. Moodle don't like empty parameter
             attributes.add(PARAM_COURSE_ID_ARRAY, "0");
             attributes.add(PARAM_SQL_CONDITIONS, sqlCondition);

src/main/resources/config/application-dev-ws.properties

@@ -18,7 +18,7 @@ spring.datasource.hikari.leakDetectionThreshold=2000
 
 sebserver.http.client.connect-timeout=15000
 sebserver.http.client.connection-request-timeout=10000
-sebserver.http.client.read-timeout=20000
+sebserver.http.client.read-timeout=30000
 sebserver.webservice.distributed.updateInterval=1000
 sebserver.webservice.distributed.connectionUpdate=2000
 sebserver.webservice.clean-db-on-startup=false
@@ -52,7 +52,7 @@ sebserver.webservice.api.pagination.maxPageSize=500
 sebserver.webservice.lms.openedx.api.token.request.paths=/oauth2/access_token
 sebserver.webservice.lms.moodle.api.token.request.paths=
 sebserver.webservice.lms.address.alias=lms.mockup.com=lms.address.alias
-sebserver.webservice.cache.moodle.course.pageSize=10
+sebserver.webservice.cache.moodle.course.pageSize=250
 
 springdoc.api-docs.enabled=true
 springdoc.swagger-ui.enabled=true

src/main/resources/config/application-ws.properties

@@ -83,6 +83,7 @@ sebserver.webservice.lms.moodle.prependShortCourseName=true
 sebserver.webservice.lms.moodle.fetch.cutoffdate.yearsBeforeNow=2
 sebserver.webservice.lms.olat.sendAdditionalAttributesWithRestriction=false
 sebserver.webservice.lms.address.alias=
+sebserver.webservice.lms.datafetch.validity.seconds=600
 
 sebserver.webservice.proctoring.resetBroadcastOnLeav=true
 sebserver.webservice.proctoring.zoom.enableWaitingRoom=false