better Moodle course access (background fetch and caching)

src/main/java/ch/ethz/seb/sebserver/gbl/util/Utils.java

@@ -516,6 +516,10 @@ public final class Utils {
         return DateTime.now(DateTimeZone.UTC).getMillis();
     }
 
+    public static long getSecondsNow() {
+        return DateTime.now(DateTimeZone.UTC).getMillis() / 1000;
+    }
+
     public static RGB toRGB(final String rgbString) {
         if (StringUtils.isNotBlank(rgbString)) {
             return new RGB(

src/main/java/ch/ethz/seb/sebserver/webservice/servicelayer/lms/impl/moodle/MoodleCourseAccess.java

@@ -8,7 +8,6 @@
 
 package ch.ethz.seb.sebserver.webservice.servicelayer.lms.impl.moodle;
 
-import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Collections;
@@ -17,13 +16,10 @@ import java.util.List;
 import java.util.Map;
 import java.util.Set;
 import java.util.function.Function;
-import java.util.function.Predicate;
 import java.util.function.Supplier;
 import java.util.stream.Collectors;
 
 import org.apache.commons.lang3.StringUtils;
-import org.joda.time.DateTime;
-import org.joda.time.DateTimeZone;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.util.LinkedMultiValueMap;
@@ -32,9 +28,7 @@ import org.springframework.util.MultiValueMap;
 import com.fasterxml.jackson.annotation.JsonCreator;
 import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
 import com.fasterxml.jackson.annotation.JsonProperty;
-import com.fasterxml.jackson.core.JsonParseException;
 import com.fasterxml.jackson.core.type.TypeReference;
-import com.fasterxml.jackson.databind.JsonMappingException;
 
 import ch.ethz.seb.sebserver.gbl.Constants;
 import ch.ethz.seb.sebserver.gbl.api.JSONMapper;
@@ -44,7 +38,6 @@ import ch.ethz.seb.sebserver.gbl.model.exam.QuizData;
 import ch.ethz.seb.sebserver.gbl.model.institution.LmsSetup;
 import ch.ethz.seb.sebserver.gbl.model.institution.LmsSetupTestResult;
 import ch.ethz.seb.sebserver.gbl.model.user.ExamineeAccountDetails;
-import ch.ethz.seb.sebserver.gbl.util.Pair;
 import ch.ethz.seb.sebserver.gbl.util.Result;
 import ch.ethz.seb.sebserver.gbl.util.Utils;
 import ch.ethz.seb.sebserver.webservice.servicelayer.lms.impl.CourseAccess;
@@ -59,23 +52,24 @@ public class MoodleCourseAccess extends CourseAccess {
 
     private static final String MOODLE_QUIZ_START_URL_PATH = "mod/quiz/view.php?id=";
 
-    private static final String MOODLE_COURSE_API_FUNCTION_NAME = "core_course_get_courses";
+    static final String MOODLE_COURSE_API_FUNCTION_NAME = "core_course_get_courses";
-    private static final String MOODLE_USER_PROFILE_API_FUNCTION_NAME = "core_user_get_users_by_field";
+    static final String MOODLE_USER_PROFILE_API_FUNCTION_NAME = "core_user_get_users_by_field";
-    private static final String MOODLE_QUIZ_API_FUNCTION_NAME = "mod_quiz_get_quizzes_by_courses";
+    static final String MOODLE_QUIZ_API_FUNCTION_NAME = "mod_quiz_get_quizzes_by_courses";
-    private static final String MOODLE_COURSE_API_COURSE_IDS = "courseids";
+    static final String MOODLE_COURSE_API_COURSE_IDS = "courseids";
-    private static final String MOODLE_COURSE_API_IDS = "ids";
+    static final String MOODLE_COURSE_API_IDS = "ids";
-    private static final String MOODLE_COURSE_SEARCH_API_FUNCTION_NAME = "core_course_search_courses";
+    static final String MOODLE_COURSE_SEARCH_API_FUNCTION_NAME = "core_course_search_courses";
-    private static final String MOODLE_COURSE_BY_FIELD_API_FUNCTION_NAME = "core_course_get_courses_by_field";
+    static final String MOODLE_COURSE_BY_FIELD_API_FUNCTION_NAME = "core_course_get_courses_by_field";
-    private static final String MOODLE_COURSE_API_FIELD_NAME = "field";
+    static final String MOODLE_COURSE_API_FIELD_NAME = "field";
-    private static final String MOODLE_COURSE_API_FIELD_VALUE = "value";
+    static final String MOODLE_COURSE_API_FIELD_VALUE = "value";
-    private static final String MOODLE_COURSE_API_SEARCH_CRITERIA_NAME = "criterianame";
+    static final String MOODLE_COURSE_API_SEARCH_CRITERIA_NAME = "criterianame";
-    private static final String MOODLE_COURSE_API_SEARCH_CRITERIA_VALUE = "criteriavalue";
+    static final String MOODLE_COURSE_API_SEARCH_CRITERIA_VALUE = "criteriavalue";
-    private static final String MOODLE_COURSE_API_SEARCH_PAGE = "page";
+    static final String MOODLE_COURSE_API_SEARCH_PAGE = "page";
-    private static final String MOODLE_COURSE_API_SEARCH_PAGE_SIZE = "perpage";
+    static final String MOODLE_COURSE_API_SEARCH_PAGE_SIZE = "perpage";
 
     private final JSONMapper jsonMapper;
     private final LmsSetup lmsSetup;
     private final MoodleRestTemplateFactory moodleRestTemplateFactory;
+    private final MoodleCourseDataLazyLoader moodleCourseDataLazyLoader;
 
     private MoodleAPIRestTemplate restTemplate;
 
@@ -83,11 +77,13 @@ public class MoodleCourseAccess extends CourseAccess {
             final JSONMapper jsonMapper,
             final LmsSetup lmsSetup,
             final MoodleRestTemplateFactory moodleRestTemplateFactory,
+            final MoodleCourseDataLazyLoader moodleCourseDataLazyLoader,
             final AsyncService asyncService) {
 
         super(asyncService);
         this.jsonMapper = jsonMapper;
         this.lmsSetup = lmsSetup;
+        this.moodleCourseDataLazyLoader = moodleCourseDataLazyLoader;
         this.moodleRestTemplateFactory = moodleRestTemplateFactory;
     }
 
@@ -187,12 +183,43 @@ public class MoodleCourseAccess extends CourseAccess {
         throw new UnsupportedOperationException("not available yet");
     }
 
-    private ArrayList<QuizData> collectAllQuizzes(final MoodleAPIRestTemplate restTemplate) {
+    private List<QuizData> collectAllQuizzes(final MoodleAPIRestTemplate restTemplate) {
 
         final String urlPrefix = (this.lmsSetup.lmsApiUrl.endsWith(Constants.URL_PATH_SEPARATOR))
                 ? this.lmsSetup.lmsApiUrl + MOODLE_QUIZ_START_URL_PATH
                 : this.lmsSetup.lmsApiUrl + Constants.URL_PATH_SEPARATOR + MOODLE_QUIZ_START_URL_PATH;
-        return getAllQuizzes(restTemplate)
+
+        Collection<CourseData> courseQuizData = Collections.emptyList();
+        if (this.moodleCourseDataLazyLoader.isRunning()) {
+            courseQuizData = this.moodleCourseDataLazyLoader.getPreFilteredCourseIds();
+        } else if (this.moodleCourseDataLazyLoader.getLastRunTime() <= 0) {
+            // first run async and wait some time, get what is there
+            this.moodleCourseDataLazyLoader.loadAsync(restTemplate);
+            try {
+                Thread.sleep(5 * Constants.SECOND_IN_MILLIS);
+                courseQuizData = this.moodleCourseDataLazyLoader.getPreFilteredCourseIds();
+            } catch (final Exception e) {
+                log.error("Failed to wait for first load run: ", e);
+                return Collections.emptyList();
+            }
+        } else if (this.moodleCourseDataLazyLoader.isLongRunningTask()) {
+            // kick off the task again when old asynchronously and take back what is there instantly
+            if (Utils.getMillisecondsNow() - this.moodleCourseDataLazyLoader.getLastRunTime() > 10
+                    * Constants.MINUTE_IN_MILLIS) {
+                this.moodleCourseDataLazyLoader.loadAsync(restTemplate);
+            }
+            courseQuizData = this.moodleCourseDataLazyLoader.getPreFilteredCourseIds();
+        } else {
+            // just run the task in sync
+            this.moodleCourseDataLazyLoader.loadSync(restTemplate);
+            courseQuizData = this.moodleCourseDataLazyLoader.getPreFilteredCourseIds();
+        }
+
+        if (courseQuizData.isEmpty()) {
+            return Collections.emptyList();
+        }
+
+        return courseQuizData
                 .stream()
                 .reduce(
                         new ArrayList<>(),
@@ -209,169 +236,6 @@ public class MoodleCourseAccess extends CourseAccess {
                         });
     }
 
-    private List<CourseData> getAllQuizzes(final MoodleAPIRestTemplate restTemplate) {
-
-        final List<CourseData> result = new ArrayList<>();
-
-        int page = 0;
-        Pair<List<CourseData>, Integer> quizzesBatch = getQuizzesBatch(restTemplate, page);
-        result.addAll(quizzesBatch.a);
-
-        log.info("Got quiz page batch for page {} of size {} with {} items",
-                page,
-                quizzesBatch.b,
-                quizzesBatch.a.size());
-
-        if (quizzesBatch.b != null && quizzesBatch.b > 0) {
-            page++;
-            quizzesBatch = getQuizzesBatch(restTemplate, page);
-            result.addAll(quizzesBatch.a);
-
-            log.info("Got quiz page batch for page {} of size {} with {} items",
-                    page,
-                    quizzesBatch.b,
-                    quizzesBatch.a.size());
-        }
-
-//        while (quizzesBatch.b != null && quizzesBatch.b > 0) {
-//            page++;
-//            quizzesBatch = getQuizzesBatch(restTemplate, page);
-//            result.addAll(quizzesBatch.a);
-//
-//            log.info("Got quiz page batch for page {} of size {} with {} items",
-//                    page,
-//                    quizzesBatch.b,
-//                    quizzesBatch.a.size());
-//        }
-        return result;
-    }
-
-    private Pair<List<CourseData>, Integer> getQuizzesBatch(
-            final MoodleAPIRestTemplate restTemplate,
-            final int page) {
-
-        try {
-
-            // first get courses from Moodle for page
-            final Map<String, CourseData> courseData = new HashMap<>();
-            final Collection<CourseData> coursesPage = getCoursesPage(restTemplate, page, 1000);
-
-            if (coursesPage.isEmpty()) {
-                return new Pair<>(Collections.emptyList(), 0);
-            }
-
-            courseData.putAll(coursesPage
-                    .stream()
-                    .collect(Collectors.toMap(cd -> cd.id, Function.identity())));
-
-            // then get all quizzes of courses and filter
-            final LinkedMultiValueMap<String, String> attributes = new LinkedMultiValueMap<>();
-            attributes.put(MOODLE_COURSE_API_COURSE_IDS, new ArrayList<>(courseData.keySet()));
-
-            final String quizzesJSON = restTemplate.callMoodleAPIFunction(
-                    MOODLE_QUIZ_API_FUNCTION_NAME,
-                    attributes);
-
-            final CourseQuizData courseQuizData = this.jsonMapper.readValue(
-                    quizzesJSON,
-                    CourseQuizData.class);
-
-            final Map<String, CourseData> finalCourseDataRef = courseData;
-            if (courseQuizData.quizzes != null) {
-                courseQuizData.quizzes
-                        .stream()
-                        .filter(getQuizFilter())
-                        .forEach(quiz -> {
-                            final CourseData course = finalCourseDataRef.get(quiz.course);
-                            if (course != null) {
-                                course.quizzes.add(quiz);
-                            }
-                        });
-            }
-
-            return new Pair<>(courseData.values()
-                    .stream()
-                    .filter(c -> !c.quizzes.isEmpty())
-                    .collect(Collectors.toList()),
-                    coursesPage.size());
-        } catch (final Exception e) {
-            log.error("Unexpected exception while trying to get course data: ", e);
-            return new Pair<>(Collections.emptyList(), 0);
-        }
-    }
-
-    private Predicate<CourseQuiz> getQuizFilter() {
-        final long now = DateTime.now(DateTimeZone.UTC).getMillis() / 1000;
-        return quiz -> {
-            if (quiz.time_close == null || quiz.time_close == 0 || quiz.time_close > now) {
-                return true;
-            }
-
-            log.info("remove quiz {} end_time {} now {}", quiz.name, quiz.time_close, now);
-            return false;
-        };
-    }
-
-    private Predicate<CourseData> getCourseFilter() {
-        final long now = DateTime.now(DateTimeZone.UTC).getMillis() / 1000;
-        return course -> {
-            if (course.end_date == null || course.end_date == 0 || course.end_date > now) {
-                return true;
-            }
-
-            log.info("remove course {} end_time {} now {}", course.short_name, course.end_date, now);
-            return false;
-        };
-    }
-
-    private Collection<CourseData> getCoursesPage(
-            final MoodleAPIRestTemplate restTemplate,
-            final int page,
-            final int size) throws JsonParseException, JsonMappingException, IOException {
-
-        try {
-            // get course ids per page
-            final LinkedMultiValueMap<String, String> attributes = new LinkedMultiValueMap<>();
-            attributes.add(MOODLE_COURSE_API_SEARCH_CRITERIA_NAME, "search");
-            attributes.add(MOODLE_COURSE_API_SEARCH_CRITERIA_VALUE, "");
-            attributes.add(MOODLE_COURSE_API_SEARCH_PAGE, String.valueOf(page));
-            attributes.add(MOODLE_COURSE_API_SEARCH_PAGE_SIZE, String.valueOf(size));
-
-            final String courseKeyPageJSON = restTemplate.callMoodleAPIFunction(
-                    MOODLE_COURSE_SEARCH_API_FUNCTION_NAME,
-                    attributes);
-
-            final CoursePage keysPage = this.jsonMapper.readValue(
-                    courseKeyPageJSON,
-                    CoursePage.class);
-
-            if (keysPage == null || keysPage.courseKeys == null || keysPage.courseKeys.isEmpty()) {
-                log.info("No courses found on page: {}", page);
-                return Collections.emptyList();
-            }
-
-            // get courses
-            final Set<String> ids = keysPage.courseKeys
-                    .stream()
-                    .map(key -> key.id)
-                    .collect(Collectors.toSet());
-
-//            return getCoursesForIds(restTemplate, ids);
-
-            final Collection<CourseData> result = getCoursesForIds(restTemplate, ids)
-                    .stream()
-                    .filter(getCourseFilter())
-                    .collect(Collectors.toList());
-
-//            log.info("course page with {} courses, after filtering {} left", keysPage.courseKeys, result.size());
-
-            return result;
-        } catch (final Exception e) {
-            log.error("Unexpected error while trying to get courses page: ", e);
-            return Collections.emptyList();
-        }
-    }
-
     private List<QuizData> getQuizzesForIds(
             final MoodleAPIRestTemplate restTemplate,
             final Set<String> quizIds) {
@@ -595,54 +459,6 @@ public class MoodleCourseAccess extends CourseAccess {
 
     // ---- Mapping Classes ---
 
-    @JsonIgnoreProperties(ignoreUnknown = true)
-    static final class CoursePage {
-        final Collection<CourseKey> courseKeys;
-
-        public CoursePage(
-                @JsonProperty(value = "courses") final Collection<CourseKey> courseKeys) {
-
-            this.courseKeys = courseKeys;
-        }
-    }
-
-    @JsonIgnoreProperties(ignoreUnknown = true)
-    static final class CourseKey {
-        final String id;
-        final String short_name;
-        final String category_name;
-        final String sort_order;
-
-        @JsonCreator
-        protected CourseKey(
-                @JsonProperty(value = "id") final String id,
-                @JsonProperty(value = "shortname") final String short_name,
-                @JsonProperty(value = "categoryname") final String category_name,
-                @JsonProperty(value = "sortorder") final String sort_order) {
-
-            this.id = id;
-            this.short_name = short_name;
-            this.category_name = category_name;
-            this.sort_order = sort_order;
-        }
-
-        @Override
-        public String toString() {
-            final StringBuilder builder = new StringBuilder();
-            builder.append("CourseKey [id=");
-            builder.append(this.id);
-            builder.append(", short_name=");
-            builder.append(this.short_name);
-            builder.append(", category_name=");
-            builder.append(this.category_name);
-            builder.append(", sort_order=");
-            builder.append(this.sort_order);
-            builder.append("]");
-            return builder.toString();
-        }
-
-    }
-
     /** Maps the Moodle course API course data */
     @JsonIgnoreProperties(ignoreUnknown = true)
     static final class CourseData {
@@ -680,6 +496,30 @@ public class MoodleCourseAccess extends CourseAccess {
             this.time_created = time_created;
         }
 
+        @Override
+        public int hashCode() {
+            final int prime = 31;
+            int result = 1;
+            result = prime * result + ((this.id == null) ? 0 : this.id.hashCode());
+            return result;
+        }
+
+        @Override
+        public boolean equals(final Object obj) {
+            if (this == obj)
+                return true;
+            if (obj == null)
+                return false;
+            if (getClass() != obj.getClass())
+                return false;
+            final CourseData other = (CourseData) obj;
+            if (this.id == null) {
+                if (other.id != null)
+                    return false;
+            } else if (!this.id.equals(other.id))
+                return false;
+            return true;
+        }
     }
 
     @JsonIgnoreProperties(ignoreUnknown = true)

src/main/java/ch/ethz/seb/sebserver/webservice/servicelayer/lms/impl/moodle/MoodleCourseDataLazyLoader.java

@@ -0,0 +1,423 @@
+/*
+ * Copyright (c) 2020 ETH Zürich, Educational Development and Technology (LET)
+ *
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/.
+ */
+
+package ch.ethz.seb.sebserver.webservice.servicelayer.lms.impl.moodle;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+import java.util.function.Function;
+import java.util.function.Predicate;
+import java.util.stream.Collectors;
+
+import org.apache.commons.lang3.StringUtils;
+import org.joda.time.DateTime;
+import org.joda.time.DateTimeZone;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.config.ConfigurableBeanFactory;
+import org.springframework.context.annotation.Lazy;
+import org.springframework.context.annotation.Scope;
+import org.springframework.stereotype.Component;
+import org.springframework.util.LinkedMultiValueMap;
+
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.core.JsonParseException;
+import com.fasterxml.jackson.databind.JsonMappingException;
+
+import ch.ethz.seb.sebserver.gbl.Constants;
+import ch.ethz.seb.sebserver.gbl.api.JSONMapper;
+import ch.ethz.seb.sebserver.gbl.async.AsyncRunner;
+import ch.ethz.seb.sebserver.gbl.profile.WebServiceProfile;
+import ch.ethz.seb.sebserver.gbl.util.Utils;
+import ch.ethz.seb.sebserver.webservice.servicelayer.lms.impl.moodle.MoodleCourseAccess.CourseData;
+import ch.ethz.seb.sebserver.webservice.servicelayer.lms.impl.moodle.MoodleCourseAccess.CourseQuiz;
+import ch.ethz.seb.sebserver.webservice.servicelayer.lms.impl.moodle.MoodleCourseAccess.CourseQuizData;
+import ch.ethz.seb.sebserver.webservice.servicelayer.lms.impl.moodle.MoodleCourseAccess.Courses;
+import ch.ethz.seb.sebserver.webservice.servicelayer.lms.impl.moodle.MoodleRestTemplateFactory.MoodleAPIRestTemplate;
+
+@Lazy
+@Component
+@WebServiceProfile
+@Scope(ConfigurableBeanFactory.SCOPE_PROTOTYPE)
+public class MoodleCourseDataLazyLoader {
+
+    private static final Logger log = LoggerFactory.getLogger(MoodleCourseDataLazyLoader.class);
+
+    private final JSONMapper jsonMapper;
+    private final AsyncRunner asyncRunner;
+
+    private final Set<CourseData> preFilteredCourseIds = new HashSet<>();
+
+    private long lastRunTime = 0;
+    private long lastLoadTime = 0;
+    private boolean running = false;
+
+    private final long fromCutTime = DateTime.now(DateTimeZone.UTC).minusYears(3).getMillis() / 1000;
+
+    public MoodleCourseDataLazyLoader(
+            final JSONMapper jsonMapper,
+            final AsyncRunner asyncRunner) {
+
+        this.jsonMapper = jsonMapper;
+        this.asyncRunner = asyncRunner;
+    }
+
+    public Set<CourseData> getPreFilteredCourseIds() {
+        return this.preFilteredCourseIds;
+    }
+
+    public long getLastRunTime() {
+        return this.lastRunTime;
+    }
+
+    public boolean isRunning() {
+        return this.running;
+    }
+
+    public boolean isLongRunningTask() {
+        return this.lastLoadTime > 30 * Constants.SECOND_IN_MILLIS;
+    }
+
+    public Set<CourseData> loadSync(final MoodleAPIRestTemplate restTemplate) {
+        if (this.running) {
+            throw new IllegalStateException("Is already running asynchronously");
+        }
+
+        this.running = true;
+        loadAndCache(restTemplate).run();
+        this.lastRunTime = Utils.getMillisecondsNow();
+
+        log.info("Loaded {} courses synchronously", this.preFilteredCourseIds.size());
+
+        return this.preFilteredCourseIds;
+    }
+
+    public void loadAsync(final MoodleAPIRestTemplate restTemplate) {
+        if (this.running) {
+            return;
+        }
+        this.running = true;
+        this.asyncRunner.runAsync(loadAndCache(restTemplate));
+        this.lastRunTime = Utils.getMillisecondsNow();
+
+    }
+
+    private Runnable loadAndCache(final MoodleAPIRestTemplate restTemplate) {
+        return () -> {
+            final long startTime = Utils.getMillisecondsNow();
+
+            loadAllQuizzes(restTemplate);
+
+            this.lastLoadTime = Utils.getMillisecondsNow() - startTime;
+            this.running = false;
+
+            log.info("Loaded {} courses asynchronously", this.preFilteredCourseIds.size());
+        };
+    }
+
+    private void loadAllQuizzes(final MoodleAPIRestTemplate restTemplate) {
+        int page = 0;
+        while (getQuizzesBatch(restTemplate, page)) {
+            page++;
+        }
+    }
+
+    private boolean getQuizzesBatch(
+            final MoodleAPIRestTemplate restTemplate,
+            final int page) {
+
+        try {
+
+            // first get courses from Moodle for page
+            final Map<String, CourseData> courseData = new HashMap<>();
+            final Collection<CourseData> coursesPage = getCoursesPage(restTemplate, page, 1000);
+
+            if (coursesPage == null || coursesPage.isEmpty()) {
+                return false;
+            }
+
+            courseData.putAll(coursesPage
+                    .stream()
+                    .collect(Collectors.toMap(cd -> cd.id, Function.identity())));
+
+            // then get all quizzes of courses and filter
+            final LinkedMultiValueMap<String, String> attributes = new LinkedMultiValueMap<>();
+            attributes.put(
+                    MoodleCourseAccess.MOODLE_COURSE_API_COURSE_IDS,
+                    new ArrayList<>(courseData.keySet()));
+
+            final String quizzesJSON = restTemplate.callMoodleAPIFunction(
+                    MoodleCourseAccess.MOODLE_QUIZ_API_FUNCTION_NAME,
+                    attributes);
+
+            final CourseQuizData courseQuizData = this.jsonMapper.readValue(
+                    quizzesJSON,
+                    CourseQuizData.class);
+
+            if (courseQuizData == null || courseQuizData.quizzes == null || courseQuizData.quizzes.isEmpty()) {
+                return false;
+            }
+
+            if (courseQuizData.quizzes != null) {
+                courseQuizData.quizzes
+                        .stream()
+                        .filter(getQuizFilter())
+                        .forEach(quiz -> {
+                            final CourseData data = courseData.get(quiz.course);
+                            if (data != null) {
+                                data.quizzes.add(quiz);
+                            }
+                        });
+
+                this.preFilteredCourseIds.addAll(
+                        courseData.values().stream()
+                                .filter(c -> !c.quizzes.isEmpty())
+                                .collect(Collectors.toList()));
+
+                return true;
+            } else {
+                return false;
+            }
+        } catch (final Exception e) {
+            log.error("Unexpected exception while trying to get course data: ", e);
+            return false;
+        }
+    }
+
+    private Collection<CourseData> getCoursesPage(
+            final MoodleAPIRestTemplate restTemplate,
+            final int page,
+            final int size) throws JsonParseException, JsonMappingException, IOException {
+
+        try {
+            // get course ids per page
+            final LinkedMultiValueMap<String, String> attributes = new LinkedMultiValueMap<>();
+            attributes.add(MoodleCourseAccess.MOODLE_COURSE_API_SEARCH_CRITERIA_NAME, "search");
+            attributes.add(MoodleCourseAccess.MOODLE_COURSE_API_SEARCH_CRITERIA_VALUE, "");
+            attributes.add(MoodleCourseAccess.MOODLE_COURSE_API_SEARCH_PAGE, String.valueOf(page));
+            attributes.add(MoodleCourseAccess.MOODLE_COURSE_API_SEARCH_PAGE_SIZE, String.valueOf(size));
+
+            final String courseKeyPageJSON = restTemplate.callMoodleAPIFunction(
+                    MoodleCourseAccess.MOODLE_COURSE_SEARCH_API_FUNCTION_NAME,
+                    attributes);
+
+            final CoursePage keysPage = this.jsonMapper.readValue(
+                    courseKeyPageJSON,
+                    CoursePage.class);
+
+            if (keysPage == null || keysPage.courseKeys == null || keysPage.courseKeys.isEmpty()) {
+                log.info("No courses found on page: {}", page);
+                return Collections.emptyList();
+            }
+
+            // get courses
+            final Set<String> ids = keysPage.courseKeys
+                    .stream()
+                    .map(key -> key.id)
+                    .collect(Collectors.toSet());
+
+            final Collection<CourseData> result = getCoursesForIds(restTemplate, ids)
+                    .stream()
+                    .filter(getCourseFilter())
+                    .collect(Collectors.toList());
+
+//            log.info("course page with {} courses, after filtering {} left", keysPage.courseKeys, result.size());
+
+            return result;
+        } catch (final Exception e) {
+            log.error("Unexpected error while trying to get courses page: ", e);
+            return Collections.emptyList();
+        }
+    }
+
+    private Collection<CourseData> getCoursesForIds(
+            final MoodleAPIRestTemplate restTemplate,
+            final Set<String> ids) {
+
+        try {
+
+            if (log.isDebugEnabled()) {
+                log.debug("Get courses for ids: {}", ids);
+            }
+
+            final String joinedIds = StringUtils.join(ids, Constants.COMMA);
+
+            final LinkedMultiValueMap<String, String> attributes = new LinkedMultiValueMap<>();
+            attributes.add(MoodleCourseAccess.MOODLE_COURSE_API_FIELD_NAME, MoodleCourseAccess.MOODLE_COURSE_API_IDS);
+            attributes.add(MoodleCourseAccess.MOODLE_COURSE_API_FIELD_VALUE, joinedIds);
+            final String coursePageJSON = restTemplate.callMoodleAPIFunction(
+                    MoodleCourseAccess.MOODLE_COURSE_BY_FIELD_API_FUNCTION_NAME,
+                    attributes);
+
+            return this.jsonMapper.<Courses> readValue(
+                    coursePageJSON,
+                    Courses.class).courses;
+        } catch (final Exception e) {
+            log.error("Unexpected error while trying to get courses for ids", e);
+            return Collections.emptyList();
+        }
+    }
+
+    private Predicate<CourseQuiz> getQuizFilter() {
+        final long now = Utils.getSecondsNow();
+        return quiz -> {
+            if (quiz.time_close == null || quiz.time_close == 0 || quiz.time_close > now) {
+                return true;
+            }
+
+            log.info("remove quiz {} end_time {} now {}", quiz.name, quiz.time_close, now);
+            return false;
+        };
+    }
+
+    private Predicate<CourseData> getCourseFilter() {
+        final long now = Utils.getSecondsNow();
+        return course -> {
+            if (course.start_date < this.fromCutTime) {
+                return false;
+            }
+
+            if (course.end_date == null || course.end_date == 0 || course.end_date > now) {
+                return true;
+            }
+
+            log.info("remove course {} end_time {} now {}", course.short_name, course.end_date, now);
+            return false;
+        };
+    }
+
+    @JsonIgnoreProperties(ignoreUnknown = true)
+    static final class CoursePage {
+        final Collection<CourseKey> courseKeys;
+
+        public CoursePage(
+                @JsonProperty(value = "courses") final Collection<CourseKey> courseKeys) {
+
+            this.courseKeys = courseKeys;
+        }
+    }
+
+    @JsonIgnoreProperties(ignoreUnknown = true)
+    static final class CourseKey {
+        final String id;
+        final String short_name;
+        final String category_name;
+        final String sort_order;
+
+        @JsonCreator
+        protected CourseKey(
+                @JsonProperty(value = "id") final String id,
+                @JsonProperty(value = "shortname") final String short_name,
+                @JsonProperty(value = "categoryname") final String category_name,
+                @JsonProperty(value = "sortorder") final String sort_order) {
+
+            this.id = id;
+            this.short_name = short_name;
+            this.category_name = category_name;
+            this.sort_order = sort_order;
+        }
+
+        @Override
+        public String toString() {
+            final StringBuilder builder = new StringBuilder();
+            builder.append("CourseKey [id=");
+            builder.append(this.id);
+            builder.append(", short_name=");
+            builder.append(this.short_name);
+            builder.append(", category_name=");
+            builder.append(this.category_name);
+            builder.append(", sort_order=");
+            builder.append(this.sort_order);
+            builder.append("]");
+            return builder.toString();
+        }
+
+    }
+
+//    @JsonIgnoreProperties(ignoreUnknown = true)
+//    static final class CourseKeys {
+//        final Collection<CourseDataKey> courses;
+//
+//        @JsonCreator
+//        protected CourseKeys(
+//                @JsonProperty(value = "courses") final Collection<CourseDataKey> courses) {
+//            this.courses = courses;
+//        }
+//    }
+
+//    /** Maps the Moodle course API course data */
+//    @JsonIgnoreProperties(ignoreUnknown = true)
+//    static final class CourseDataKey {
+//        final String id;
+//        final String short_name;
+//        final Long start_date; // unix-time seconds UTC
+//        final Long end_date; // unix-time seconds UTC
+//        final Long time_created; // unix-time seconds UTC
+//        final Collection<CourseQuizKey> quizzes = new ArrayList<>();
+//
+//        @JsonCreator
+//        protected CourseDataKey(
+//                @JsonProperty(value = "id") final String id,
+//                @JsonProperty(value = "shortname") final String short_name,
+//                @JsonProperty(value = "startdate") final Long start_date,
+//                @JsonProperty(value = "enddate") final Long end_date,
+//                @JsonProperty(value = "timecreated") final Long time_created) {
+//
+//            this.id = id;
+//            this.short_name = short_name;
+//            this.start_date = start_date;
+//            this.end_date = end_date;
+//            this.time_created = time_created;
+//        }
+//
+//    }
+
+//    @JsonIgnoreProperties(ignoreUnknown = true)
+//    static final class CourseQuizKeys {
+//        final Collection<CourseQuizKey> quizzes;
+//
+//        @JsonCreator
+//        protected CourseQuizKeys(
+//                @JsonProperty(value = "quizzes") final Collection<CourseQuizKey> quizzes) {
+//            this.quizzes = quizzes;
+//        }
+//    }
+//
+//    @JsonIgnoreProperties(ignoreUnknown = true)
+//    static final class CourseQuizKey {
+//        final String id;
+//        final String course;
+//        final String name;
+//        final Long time_open; // unix-time seconds UTC
+//        final Long time_close; // unix-time seconds UTC
+//
+//        @JsonCreator
+//        protected CourseQuizKey(
+//                @JsonProperty(value = "id") final String id,
+//                @JsonProperty(value = "course") final String course,
+//                @JsonProperty(value = "name") final String name,
+//                @JsonProperty(value = "timeopen") final Long time_open,
+//                @JsonProperty(value = "timeclose") final Long time_close) {
+//
+//            this.id = id;
+//            this.course = course;
+//            this.name = name;
+//            this.time_open = time_open;
+//            this.time_close = time_close;
+//        }
+//    }
+
+}

src/main/java/ch/ethz/seb/sebserver/webservice/servicelayer/lms/impl/moodle/MoodleLmsAPITemplateFactory.java

@@ -33,6 +33,7 @@ public class MoodleLmsAPITemplateFactory {
     private final AsyncService asyncService;
     private final ClientCredentialService clientCredentialService;
     private final ClientHttpRequestFactoryService clientHttpRequestFactoryService;
+    private final MoodleCourseDataLazyLoader moodleCourseDataLazyLoader;
     private final String[] alternativeTokenRequestPaths;
 
     protected MoodleLmsAPITemplateFactory(
@@ -40,12 +41,14 @@ public class MoodleLmsAPITemplateFactory {
             final AsyncService asyncService,
             final ClientCredentialService clientCredentialService,
             final ClientHttpRequestFactoryService clientHttpRequestFactoryService,
+            final MoodleCourseDataLazyLoader moodleCourseDataLazyLoader,
             @Value("${sebserver.webservice.lms.moodle.api.token.request.paths:}") final String alternativeTokenRequestPaths) {
 
         this.jsonMapper = jsonMapper;
         this.asyncService = asyncService;
         this.clientCredentialService = clientCredentialService;
         this.clientHttpRequestFactoryService = clientHttpRequestFactoryService;
+        this.moodleCourseDataLazyLoader = moodleCourseDataLazyLoader;
         this.alternativeTokenRequestPaths = (alternativeTokenRequestPaths != null)
                 ? StringUtils.split(alternativeTokenRequestPaths, Constants.LIST_SEPARATOR)
                 : null;
@@ -71,6 +74,7 @@ public class MoodleLmsAPITemplateFactory {
                     this.jsonMapper,
                     lmsSetup,
                     moodleRestTemplateFactory,
+                    this.moodleCourseDataLazyLoader,
                     this.asyncService);
 
             final MoodleCourseRestriction moodleCourseRestriction = new MoodleCourseRestriction(

src/main/resources/config/application-dev.properties

@@ -7,3 +7,7 @@ server.tomcat.uri-encoding=UTF-8
 
 logging.level.ch=INFO
 
+sebserver.http.client.connect-timeout=150000
+sebserver.http.client.connection-request-timeout=100000
+sebserver.http.client.read-timeout=200000
+

src/test/java/ch/ethz/seb/sebserver/webservice/servicelayer/lms/impl/moodle/MoodleCourseAccessTest.java

@@ -67,6 +67,7 @@ public class MoodleCourseAccessTest {
                 new JSONMapper(),
                 null,
                 moodleRestTemplateFactory,
+                null,
                 mock(AsyncService.class));
 
         final String examId = "123";
@@ -114,6 +115,7 @@ public class MoodleCourseAccessTest {
                 new JSONMapper(),
                 null,
                 moodleRestTemplateFactory,
+                null,
                 mock(AsyncService.class));
 
         final LmsSetupTestResult initAPIAccess = moodleCourseAccess.initAPIAccess();
@@ -135,6 +137,7 @@ public class MoodleCourseAccessTest {
                 new JSONMapper(),
                 null,
                 moodleRestTemplateFactory,
+                null,
                 mock(AsyncService.class));
 
         final LmsSetupTestResult initAPIAccess = moodleCourseAccess.initAPIAccess();
@@ -155,6 +158,7 @@ public class MoodleCourseAccessTest {
                 new JSONMapper(),
                 null,
                 moodleRestTemplateFactory,
+                null,
                 mock(AsyncService.class));
 
         final LmsSetupTestResult initAPIAccess = moodleCourseAccess.initAPIAccess();