zervo/seb-server
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

SEBSERV-413 removed old LMS Setup code (async loader Moodle)

Browse source
This commit is contained in:
anhefti 2023-05-15 09:49:39 +02:00
parent bf6aa19a82
commit 7c42974838
11 changed files with 1 additions and 895 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

15
src/main/java/ch/ethz/seb/sebserver/webservice/servicelayer/lms/CourseAccessAPI.java
View file

@ -10,7 +10,6 @@ package ch.ethz.seb.sebserver.webservice.servicelayer.lms;
import java.util.Collection;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import org.slf4j.Logger;
@ -52,20 +51,6 @@ public interface CourseAccessAPI {
}
}
/** Get an unsorted List of filtered {@link QuizData } from the LMS course/quiz API
*
* @param filterMap the {@link FilterMap } to get a filtered result. Possible filter attributes are:
*
* <pre>
* {@link QuizData.FILTER_ATTR_QUIZ_NAME } The quiz name filter text (exclude all names that do not contain the given text)
* {@link QuizData.FILTER_ATTR_START_TIME } The quiz start time (exclude all quizzes that starts before)
* </pre>
*
* @return Result of an unsorted List of filtered {@link QuizData } from the LMS course/quiz API
* or refer to an error when happened */
@Deprecated
Result<List<QuizData>> getQuizzes(FilterMap filterMap);
void fetchQuizzes(FilterMap filterMap, AsyncQuizFetchBuffer asyncQuizFetchBuffer);
/** Get all {@link QuizData } for the set of {@link QuizData } identifiers from LMS API in a collection

21
src/main/java/ch/ethz/seb/sebserver/webservice/servicelayer/lms/impl/LmsAPITemplateAdapter.java
View file

@ -9,7 +9,6 @@
package ch.ethz.seb.sebserver.webservice.servicelayer.lms.impl;
import java.util.Collection;
import java.util.List;
import java.util.Set;
import org.slf4j.Logger;
@ -221,26 +220,6 @@ public class LmsAPITemplateAdapter implements LmsAPITemplate {
return LmsSetupTestResult.ofAPINotSupported(getType());
}
@Override
@Deprecated
public Result<List<QuizData>> getQuizzes(final FilterMap filterMap) {
if (this.courseAccessAPI == null) {
return Result
.ofError(new UnsupportedOperationException("Course API Not Supported For: " + getType().name()));
}
if (log.isDebugEnabled()) {
log.debug("Get quizzes for LMSSetup: {}", lmsSetup());
}
return this.courseAccessAPI
.getQuizzes(filterMap)
.onError(error -> log.error(
"Failed to run protectedQuizzesRequest: {}",
error.getMessage()));
}
@Override
public void fetchQuizzes(final FilterMap filterMap, final AsyncQuizFetchBuffer asyncQuizFetchBuffer) {
if (this.courseAccessAPI == null) {

9
src/main/java/ch/ethz/seb/sebserver/webservice/servicelayer/lms/impl/ans/AnsLmsAPITemplate.java
View file

@ -165,15 +165,6 @@ public class AnsLmsAPITemplate extends AbstractCachedCourseAccess implements Lms
return LmsSetupTestResult.ofOkay(LmsType.ANS_DELFT);
}
@Override
public Result<List<QuizData>> getQuizzes(final FilterMap filterMap) {
return this
.allQuizzesRequest(filterMap)
.map(quizzes -> quizzes.stream()
.filter(LmsAPIService.quizFilterPredicate(filterMap))
.collect(Collectors.toList()));
}
@Override
public void fetchQuizzes(final FilterMap filterMap, final AsyncQuizFetchBuffer asyncQuizFetchBuffer) {
this.allQuizzesRequest(filterMap)

40
src/main/java/ch/ethz/seb/sebserver/webservice/servicelayer/lms/impl/edx/OpenEdxCourseAccess.java
View file

@ -140,11 +140,6 @@ final class OpenEdxCourseAccess extends AbstractCachedCourseAccess implements Co
return LmsSetupTestResult.ofOkay(LmsType.OPEN_EDX);
}
@Override
public Result<List<QuizData>> getQuizzes(final FilterMap filterMap) {
return getRestTemplate().map(this::collectAllQuizzes);
}
@Override
public void fetchQuizzes(final FilterMap filterMap, final AsyncQuizFetchBuffer asyncQuizFetchBuffer) {
try {
@ -360,25 +355,6 @@ final class OpenEdxCourseAccess extends AbstractCachedCourseAccess implements Co
});
}
private ArrayList<QuizData> collectAllQuizzes(final OAuth2RestTemplate restTemplate) {
final LmsSetup lmsSetup = getApiTemplateDataSupplier().getLmsSetup();
final String externalStartURI = getExternalLMSServerAddress(lmsSetup);
return collectAllCourses(
lmsSetup.lmsApiUrl + OPEN_EDX_DEFAULT_COURSE_ENDPOINT,
restTemplate)
.stream()
.reduce(
new ArrayList<>(),
(list, courseData) -> {
list.add(quizDataOf(lmsSetup, courseData, externalStartURI));
return list;
},
(list1, list2) -> {
list1.addAll(list2);
return list1;
});
}
private String getExternalLMSServerAddress(final LmsSetup lmsSetup) {
final String externalAddressAlias = this.webserviceInfo.getLmsExternalAddressAlias(lmsSetup.lmsApiUrl);
String _externalStartURI = lmsSetup.lmsApiUrl + OPEN_EDX_DEFAULT_COURSE_START_URL_PREFIX;
@ -466,22 +442,6 @@ final class OpenEdxCourseAccess extends AbstractCachedCourseAccess implements Co
}
}
private List<CourseData> collectAllCourses(final String pageURI, final OAuth2RestTemplate restTemplate) {
final List<CourseData> collector = new ArrayList<>();
EdXPage page = getEdxPage(pageURI, restTemplate).getBody();
if (page != null) {
collector.addAll(page.results);
while (page != null && StringUtils.isNotBlank(page.next)) {
page = getEdxPage(page.next, restTemplate).getBody();
if (page != null) {
collector.addAll(page.results);
}
}
}
return collector;
}
private ResponseEntity<EdXPage> getEdxPage(final String pageURI, final OAuth2RestTemplate restTemplate) {
final HttpHeaders httpHeaders = new HttpHeaders();
return restTemplate.exchange(

21
src/main/java/ch/ethz/seb/sebserver/webservice/servicelayer/lms/impl/mockup/MockCourseAccessAPI.java
View file

@ -160,27 +160,6 @@ public class MockCourseAccessAPI implements CourseAccessAPI {
return missingAttrs;
}
@Override
public Result<List<QuizData>> getQuizzes(final FilterMap filterMap) {
return Result.tryCatch(() -> {
if (!authenticate()) {
throw new IllegalArgumentException("Wrong clientId or secret");
}
if (this.simulateLatency) {
final int seconds = this.random.nextInt(20);
System.out.println("************ Mockup LMS wait for " + seconds + " seconds before respond");
Thread.sleep(seconds * 1000);
}
return this.mockups
.stream()
.map(this::getExternalAddressAlias)
.filter(LmsAPIService.quizFilterPredicate(filterMap))
.collect(Collectors.toList());
});
}
@Override
public void fetchQuizzes(final FilterMap filterMap, final AsyncQuizFetchBuffer asyncQuizFetchBuffer) {
if (!authenticate()) {

162
src/main/java/ch/ethz/seb/sebserver/webservice/servicelayer/lms/impl/moodle/legacy/MoodleCourseAccess.java
View file

@ -22,7 +22,6 @@ import java.util.stream.Stream;
import org.apache.commons.lang3.BooleanUtils;
import org.apache.commons.lang3.StringUtils;
import org.joda.time.DateTime;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.core.env.Environment;
@ -59,8 +58,6 @@ import ch.ethz.seb.sebserver.webservice.servicelayer.lms.impl.moodle.MoodleUtils
import ch.ethz.seb.sebserver.webservice.servicelayer.lms.impl.moodle.MoodleUtils.CourseQuizData;
import ch.ethz.seb.sebserver.webservice.servicelayer.lms.impl.moodle.MoodleUtils.Courses;
import ch.ethz.seb.sebserver.webservice.servicelayer.lms.impl.moodle.MoodleUtils.MoodleUserDetails;
import ch.ethz.seb.sebserver.webservice.servicelayer.lms.impl.moodle.legacy.MoodleCourseDataAsyncLoader.CourseDataShort;
import ch.ethz.seb.sebserver.webservice.servicelayer.lms.impl.moodle.legacy.MoodleCourseDataAsyncLoader.CourseQuizShort;
/** Implements the LmsAPITemplate for Open edX LMS Course API access.
*
@ -78,7 +75,7 @@ import ch.ethz.seb.sebserver.webservice.servicelayer.lms.impl.moodle.legacy.Mood
* possibly make this synchronous fetch strategy obsolete in the future. */
public class MoodleCourseAccess implements CourseAccessAPI {
private static final long INITIAL_WAIT_TIME = 3 * Constants.SECOND_IN_MILLIS;
//private static final long INITIAL_WAIT_TIME = 3 * Constants.SECOND_IN_MILLIS;
private static final Logger log = LoggerFactory.getLogger(MoodleCourseAccess.class);
@ -101,7 +98,6 @@ public class MoodleCourseAccess implements CourseAccessAPI {
private final JSONMapper jsonMapper;
private final MoodleRestTemplateFactory restTemplateFactory;
private final MoodleCourseDataAsyncLoader moodleCourseDataAsyncLoader;
private final boolean prependShortCourseName;
private final CircuitBreaker<String> protectedMoodlePageCall;
private final int pageSize;
@ -113,11 +109,9 @@ public class MoodleCourseAccess implements CourseAccessAPI {
final JSONMapper jsonMapper,
final AsyncService asyncService,
final MoodleRestTemplateFactory restTemplateFactory,
final MoodleCourseDataAsyncLoader moodleCourseDataAsyncLoader,
final Environment environment) {
this.jsonMapper = jsonMapper;
this.moodleCourseDataAsyncLoader = moodleCourseDataAsyncLoader;
this.restTemplateFactory = restTemplateFactory;
this.prependShortCourseName = BooleanUtils.toBoolean(environment.getProperty(
@ -176,16 +170,6 @@ public class MoodleCourseAccess implements CourseAccessAPI {
return LmsSetupTestResult.ofOkay(LmsType.MOODLE);
}
@Override
public Result<List<QuizData>> getQuizzes(final FilterMap filterMap) {
return Result.tryCatch(() -> getRestTemplate()
.map(template -> collectAllQuizzes(template, filterMap))
.map(quizzes -> quizzes.stream()
.filter(LmsAPIService.quizFilterPredicate(filterMap))
.collect(Collectors.toList()))
.getOr(Collections.emptyList()));
}
@Override
public void fetchQuizzes(final FilterMap filterMap, final AsyncQuizFetchBuffer asyncQuizFetchBuffer) {
try {
@ -297,35 +281,6 @@ public class MoodleCourseAccess implements CourseAccessAPI {
@Override
public Result<QuizData> getQuiz(final String id) {
return Result.tryCatch(() -> {
final Map<String, CourseDataShort> cachedCourseData = this.moodleCourseDataAsyncLoader
.getCachedCourseData();
final String courseId = MoodleUtils.getCourseId(id);
final String quizId = MoodleUtils.getQuizId(id);
if (cachedCourseData.containsKey(courseId)) {
final CourseDataShort courseData = cachedCourseData.get(courseId);
final CourseQuizShort quiz = courseData.quizzes
.stream()
.filter(q -> q.id.equals(quizId))
.findFirst()
.orElse(null);
if (quiz != null) {
final Map<String, String> additionalAttrs = new HashMap<>();
additionalAttrs.put(QuizData.ATTR_ADDITIONAL_CREATION_TIME,
String.valueOf(courseData.time_created));
additionalAttrs.put(QuizData.ATTR_ADDITIONAL_SHORT_NAME, courseData.short_name);
additionalAttrs.put(QuizData.ATTR_ADDITIONAL_ID_NUMBER, courseData.idnumber);
final LmsSetup lmsSetup = getApiTemplateDataSupplier().getLmsSetup();
final String urlPrefix = (lmsSetup.lmsApiUrl.endsWith(Constants.URL_PATH_SEPARATOR))
? lmsSetup.lmsApiUrl + MOODLE_QUIZ_START_URL_PATH
: lmsSetup.lmsApiUrl + Constants.URL_PATH_SEPARATOR + MOODLE_QUIZ_START_URL_PATH;
return createQuizData(lmsSetup, courseData, urlPrefix, additionalAttrs, quiz);
}
}
// get from LMS in protected request
final Set<String> ids = Stream.of(id).collect(Collectors.toSet());
return getRestTemplate()
@ -475,72 +430,6 @@ public class MoodleCourseAccess implements CourseAccessAPI {
return Result.ofError(new UnsupportedOperationException("not available yet"));
}
public void clearCache() {
this.moodleCourseDataAsyncLoader.clearCache();
}
private List<QuizData> collectAllQuizzes(
final MoodleAPIRestTemplate restTemplate,
final FilterMap filterMap) {
final LmsSetup lmsSetup = getApiTemplateDataSupplier().getLmsSetup();
final String urlPrefix = (lmsSetup.lmsApiUrl.endsWith(Constants.URL_PATH_SEPARATOR))
? lmsSetup.lmsApiUrl + MOODLE_QUIZ_START_URL_PATH
: lmsSetup.lmsApiUrl + Constants.URL_PATH_SEPARATOR + MOODLE_QUIZ_START_URL_PATH;
final DateTime quizFromTime = (filterMap != null) ? filterMap.getQuizFromTime() : null;
final long fromCutTime = (quizFromTime != null) ? Utils.toUnixTimeInSeconds(quizFromTime) : -1;
// Verify and call the proper strategy to get the course and quiz data
Collection<CourseDataShort> courseQuizData = Collections.emptyList();
if (this.moodleCourseDataAsyncLoader.isRunning()) {
courseQuizData = this.moodleCourseDataAsyncLoader.getCachedCourseData().values();
} else if (this.moodleCourseDataAsyncLoader.getLastRunTime() <= 0) {
// set cut time if available
if (fromCutTime >= 0) {
this.moodleCourseDataAsyncLoader.setFromCutTime(fromCutTime);
}
// first run async and wait some time, get what is there
this.moodleCourseDataAsyncLoader.loadAsync(restTemplate);
try {
Thread.sleep(INITIAL_WAIT_TIME);
courseQuizData = this.moodleCourseDataAsyncLoader.getCachedCourseData().values();
} catch (final Exception e) {
log.error("Failed to wait for first load run: ", e);
return Collections.emptyList();
}
} else if (this.moodleCourseDataAsyncLoader.isLongRunningTask()) {
// on long running tasks if we have a different fromCutTime as before
// kick off the lazy loading task immediately with the new time filter
courseQuizData = this.moodleCourseDataAsyncLoader.getCachedCourseData().values();
if (fromCutTime > 0 && fromCutTime != this.moodleCourseDataAsyncLoader.getFromCutTime()) {
this.moodleCourseDataAsyncLoader.setFromCutTime(fromCutTime);
this.moodleCourseDataAsyncLoader.loadAsync(restTemplate);
// otherwise kick off only if the last fetch task was then minutes ago
} else if (Utils.getMillisecondsNow() - this.moodleCourseDataAsyncLoader.getLastRunTime() > 10
* Constants.MINUTE_IN_MILLIS) {
this.moodleCourseDataAsyncLoader.loadAsync(restTemplate);
}
} else {
// just run the task in sync
if (fromCutTime >= 0) {
this.moodleCourseDataAsyncLoader.setFromCutTime(fromCutTime);
}
this.moodleCourseDataAsyncLoader.loadSync(restTemplate);
courseQuizData = this.moodleCourseDataAsyncLoader.getCachedCourseData().values();
}
if (courseQuizData.isEmpty()) {
return Collections.emptyList();
}
return courseQuizData
.stream()
.flatMap(courseData -> quizDataOf(lmsSetup, courseData, urlPrefix).stream())
.collect(Collectors.toList());
}
private List<QuizData> getQuizzesForIds(
final MoodleAPIRestTemplate restTemplate,
final Set<String> quizIds) {
@ -672,55 +561,6 @@ public class MoodleCourseAccess implements CourseAccessAPI {
}
}
private List<QuizData> quizDataOf(
final LmsSetup lmsSetup,
final CourseDataShort courseData,
final String uriPrefix) {
final Map<String, String> additionalAttrs = new HashMap<>();
additionalAttrs.put(QuizData.ATTR_ADDITIONAL_CREATION_TIME, String.valueOf(courseData.time_created));
additionalAttrs.put(QuizData.ATTR_ADDITIONAL_SHORT_NAME, courseData.short_name);
additionalAttrs.put(QuizData.ATTR_ADDITIONAL_ID_NUMBER, courseData.idnumber);
final List<QuizData> courseAndQuiz = courseData.quizzes
.stream()
.map(courseQuizData -> createQuizData(lmsSetup, courseData, uriPrefix, additionalAttrs, courseQuizData))
.collect(Collectors.toList());
return courseAndQuiz;
}
private QuizData createQuizData(
final LmsSetup lmsSetup,
final CourseDataShort courseData,
final String uriPrefix,
final Map<String, String> additionalAttrs,
final CourseQuizShort courseQuizData) {
final String startURI = uriPrefix + courseQuizData.course_module;
return new QuizData(
MoodleUtils.getInternalQuizId(
courseQuizData.course_module, // TODO this is wrong should be id. Create recovery task
courseData.id,
courseData.short_name,
courseData.idnumber),
lmsSetup.getInstitutionId(),
lmsSetup.id,
lmsSetup.getLmsType(),
(this.prependShortCourseName)
? courseData.short_name + " : " + courseQuizData.name
: courseQuizData.name,
Constants.EMPTY_NOTE,
(courseQuizData.time_open != null && courseQuizData.time_open > 0)
? Utils.toDateTimeUTCUnix(courseQuizData.time_open)
: Utils.toDateTimeUTCUnix(courseData.start_date),
(courseQuizData.time_close != null && courseQuizData.time_close > 0)
? Utils.toDateTimeUTCUnix(courseQuizData.time_close)
: Utils.toDateTimeUTCUnix(courseData.end_date),
startURI,
additionalAttrs);
}
private static final void fillSelectedQuizzes(
final Set<String> quizIds,
final Map<String, CourseData> finalCourseDataRef,

601
src/main/java/ch/ethz/seb/sebserver/webservice/servicelayer/lms/impl/moodle/legacy/MoodleCourseDataAsyncLoader.java
View file

@ -1,601 +0,0 @@
/*
* Copyright (c) 2020 ETH Zürich, Educational Development and Technology (LET)
*
* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/.
*/
package ch.ethz.seb.sebserver.webservice.servicelayer.lms.impl.moodle.legacy;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.function.Function;
import java.util.function.Predicate;
import java.util.stream.Collectors;
import org.apache.commons.lang3.StringUtils;
import org.joda.time.DateTime;
import org.joda.time.DateTimeZone;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.config.ConfigurableBeanFactory;
import org.springframework.context.annotation.Lazy;
import org.springframework.context.annotation.Scope;
import org.springframework.core.env.Environment;
import org.springframework.stereotype.Component;
import org.springframework.util.LinkedMultiValueMap;
import org.springframework.util.MultiValueMap;
import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.fasterxml.jackson.core.JsonParseException;
import com.fasterxml.jackson.databind.JsonMappingException;
import ch.ethz.seb.sebserver.gbl.Constants;
import ch.ethz.seb.sebserver.gbl.api.JSONMapper;
import ch.ethz.seb.sebserver.gbl.async.AsyncRunner;
import ch.ethz.seb.sebserver.gbl.async.AsyncService;
import ch.ethz.seb.sebserver.gbl.async.CircuitBreaker;
import ch.ethz.seb.sebserver.gbl.profile.WebServiceProfile;
import ch.ethz.seb.sebserver.gbl.util.Utils;
import ch.ethz.seb.sebserver.webservice.servicelayer.lms.impl.moodle.MoodleAPIRestTemplate;
import ch.ethz.seb.sebserver.webservice.servicelayer.lms.impl.moodle.MoodleAPIRestTemplate.Warning;
import ch.ethz.seb.sebserver.webservice.servicelayer.lms.impl.moodle.MoodleUtils.CoursePage;
@Lazy
@Component
@WebServiceProfile
@Scope(ConfigurableBeanFactory.SCOPE_PROTOTYPE)
/** This implements the (temporary) asynchronous fetch strategy to fetch
* course and quiz data within a background task and fill up a shared cache. */
@Deprecated
public class MoodleCourseDataAsyncLoader {
private static final Logger log = LoggerFactory.getLogger(MoodleCourseDataAsyncLoader.class);
private final JSONMapper jsonMapper;
private final AsyncRunner asyncRunner;
private final CircuitBreaker<String> moodleRestCall;
private final int maxSize;
private final int pageSize;
private final Map<String, CourseDataShort> cachedCourseData = new HashMap<>();
private final Set<String> newIds = new HashSet<>();
private String lmsSetup = Constants.EMPTY_NOTE;
private long lastRunTime = 0;
private long lastLoadTime = 0;
private boolean running = false;
private long fromCutTime;
public MoodleCourseDataAsyncLoader(
final JSONMapper jsonMapper,
final AsyncService asyncService,
final AsyncRunner asyncRunner,
final Environment environment) {
this.jsonMapper = jsonMapper;
final int yearsBeforeNow = environment.getProperty(
"sebserver.webservice.lms.moodle.fetch.cutoffdate.yearsBeforeNow",
Integer.class, 3);
this.fromCutTime = Utils.toUnixTimeInSeconds(DateTime.now(DateTimeZone.UTC).minusYears(yearsBeforeNow));
this.asyncRunner = asyncRunner;
this.moodleRestCall = asyncService.createCircuitBreaker(
environment.getProperty(
"sebserver.webservice.circuitbreaker.moodleRestCall.attempts",
Integer.class,
2),
environment.getProperty(
"sebserver.webservice.circuitbreaker.moodleRestCall.blockingTime",
Long.class,
Constants.SECOND_IN_MILLIS * 20),
environment.getProperty(
"sebserver.webservice.circuitbreaker.moodleRestCall.timeToRecover",
Long.class,
Constants.MINUTE_IN_MILLIS));
this.maxSize =
environment.getProperty("sebserver.webservice.cache.moodle.course.maxSize", Integer.class, 10000);
this.pageSize =
environment.getProperty("sebserver.webservice.cache.moodle.course.pageSize", Integer.class, 500);
}
public void init(final String lmsSetupName) {
if (Constants.EMPTY_NOTE.equals(this.lmsSetup)) {
this.lmsSetup = lmsSetupName;
} else {
throw new IllegalStateException(
"Invalid initialization of MoodleCourseDataAsyncLoader. It has already been initialized yet");
}
}
public long getFromCutTime() {
return this.fromCutTime;
}
public void setFromCutTime(final long fromCutTime) {
this.fromCutTime = fromCutTime;
}
public Map<String, CourseDataShort> getCachedCourseData() {
return new HashMap<>(this.cachedCourseData);
}
public long getLastRunTime() {
return this.lastRunTime;
}
public boolean isRunning() {
return this.running;
}
public boolean isLongRunningTask() {
return this.lastLoadTime > 3 * Constants.SECOND_IN_MILLIS;
}
public Map<String, CourseDataShort> loadSync(final MoodleAPIRestTemplate restTemplate) {
if (this.running) {
throw new IllegalStateException("Is already running asynchronously");
}
this.running = true;
loadAndCache(restTemplate).run();
this.lastRunTime = Utils.getMillisecondsNow();
log.info("LMS Setup: {} loaded {} courses synchronously",
this.lmsSetup,
this.cachedCourseData.size());
return this.cachedCourseData;
}
public void loadAsync(final MoodleAPIRestTemplate restTemplate) {
if (this.running) {
return;
}
this.running = true;
this.asyncRunner.runAsync(loadAndCache(restTemplate));
this.lastRunTime = Utils.getMillisecondsNow();
log.info("LMS Setup: {} loaded {} courses asynchronously",
this.lmsSetup,
this.cachedCourseData.size());
}
public void clearCache() {
if (!isRunning()) {
this.cachedCourseData.clear();
this.newIds.clear();
}
}
private Runnable loadAndCache(final MoodleAPIRestTemplate restTemplate) {
return () -> {
this.newIds.clear();
final long startTime = Utils.getMillisecondsNow();
loadAllQuizzes(restTemplate);
this.syncCache();
this.lastLoadTime = Utils.getMillisecondsNow() - startTime;
this.running = false;
};
}
private void loadAllQuizzes(final MoodleAPIRestTemplate restTemplate) {
int page = 0;
while (getQuizzesBatch(restTemplate, page)) {
page++;
}
}
private boolean getQuizzesBatch(
final MoodleAPIRestTemplate restTemplate,
final int page) {
try {
// first get courses from Moodle for page
final Map<String, CourseDataShort> courseData = new HashMap<>();
final Collection<CourseDataShort> coursesPage = getCoursesPage(restTemplate, page, this.pageSize);
if (coursesPage == null || coursesPage.isEmpty()) {
return false;
}
courseData.putAll(coursesPage
.stream()
.collect(Collectors.toMap(cd -> cd.id, Function.identity())));
// then get all quizzes of courses and filter
final LinkedMultiValueMap<String, String> attributes = new LinkedMultiValueMap<>();
final List<String> courseIds = new ArrayList<>(courseData.keySet());
if (courseIds.size() == 1) {
// NOTE: This is a workaround because the Moodle API do not support lists with only one element.
courseIds.add("0");
}
attributes.put(
MoodleCourseAccess.MOODLE_COURSE_API_COURSE_IDS,
courseIds);
final String quizzesJSON = callMoodleRestAPI(
restTemplate,
MoodleCourseAccess.MOODLE_QUIZ_API_FUNCTION_NAME,
attributes);
final CourseQuizData courseQuizData = this.jsonMapper.readValue(
quizzesJSON,
CourseQuizData.class);
if (courseQuizData == null) {
// return false; SEBSERV-361
return true;
}
if (courseQuizData.warnings != null && !courseQuizData.warnings.isEmpty()) {
log.warn(
"There are warnings from Moodle response: Moodle: {} request: {} warnings: {} warning sample: {}",
this.lmsSetup,
MoodleCourseAccess.MOODLE_QUIZ_API_FUNCTION_NAME,
courseQuizData.warnings.size(),
courseQuizData.warnings.iterator().next().toString());
if (log.isTraceEnabled()) {
log.trace("All warnings from Moodle: {}", courseQuizData.warnings.toString());
}
}
if (courseQuizData.quizzes == null || courseQuizData.quizzes.isEmpty()) {
// no quizzes on this page
return true;
}
courseQuizData.quizzes
.stream()
.filter(getQuizFilter())
.forEach(quiz -> {
final CourseDataShort data = courseData.get(quiz.course);
if (data != null) {
data.quizzes.add(quiz);
}
});
courseData.values().stream()
.filter(c -> !c.quizzes.isEmpty())
.forEach(c -> {
if (this.cachedCourseData.size() >= this.maxSize) {
log.error(
"LMS Setup: {} Cache is full and has reached its maximal size. Skip data: -> {}",
this.lmsSetup,
c);
} else {
this.cachedCourseData.put(c.id, c);
this.newIds.add(c.id);
}
});
return true;
} catch (final Exception e) {
log.error("LMS Setup: {} Unexpected exception while trying to get course data: ", this.lmsSetup, e);
return false;
}
}
private Collection<CourseDataShort> getCoursesPage(
final MoodleAPIRestTemplate restTemplate,
final int page,
final int size) throws JsonParseException, JsonMappingException, IOException {
try {
// get course ids per page
final LinkedMultiValueMap<String, String> attributes = new LinkedMultiValueMap<>();
attributes.add(MoodleCourseAccess.MOODLE_COURSE_API_SEARCH_CRITERIA_NAME, "search");
attributes.add(MoodleCourseAccess.MOODLE_COURSE_API_SEARCH_CRITERIA_VALUE, "");
attributes.add(MoodleCourseAccess.MOODLE_COURSE_API_SEARCH_PAGE, String.valueOf(page));
attributes.add(MoodleCourseAccess.MOODLE_COURSE_API_SEARCH_PAGE_SIZE, String.valueOf(size));
final String courseKeyPageJSON = callMoodleRestAPI(
restTemplate,
MoodleCourseAccess.MOODLE_COURSE_SEARCH_API_FUNCTION_NAME,
attributes);
final CoursePage keysPage = this.jsonMapper.readValue(
courseKeyPageJSON,
CoursePage.class);
if (keysPage == null) {
log.error("No CoursePage Response");
return Collections.emptyList();
}
if (keysPage.warnings != null && !keysPage.warnings.isEmpty()) {
log.warn(
"There are warnings from Moodle response: Moodle: {} request: {} warnings: {} warning sample: {}",
this.lmsSetup,
MoodleCourseAccess.MOODLE_COURSE_SEARCH_API_FUNCTION_NAME,
keysPage.warnings.size(),
keysPage.warnings.iterator().next().toString());
if (log.isTraceEnabled()) {
log.trace("All warnings from Moodle: {}", keysPage.warnings.toString());
}
}
if (keysPage.courseKeys == null || keysPage.courseKeys.isEmpty()) {
if (log.isDebugEnabled()) {
log.debug("LMS Setup: {} No courses found on page: {}", this.lmsSetup, page);
if (log.isTraceEnabled()) {
log.trace("Moodle response: {}", courseKeyPageJSON);
}
}
return Collections.emptyList();
}
// get courses
final Set<String> ids = keysPage.courseKeys
.stream()
.map(key -> key.id)
.collect(Collectors.toSet());
final Collection<CourseDataShort> result = getCoursesForIds(restTemplate, ids)
.stream()
.filter(getCourseFilter())
.collect(Collectors.toList());
if (log.isDebugEnabled()) {
log.debug("course page with {} courses, after filtering {} left",
keysPage.courseKeys.size(),
result.size());
}
return result;
} catch (final Exception e) {
log.error("LMS Setup: {} Unexpected error while trying to get courses page: ", this.lmsSetup, e);
return Collections.emptyList();
}
}
private Collection<CourseDataShort> getCoursesForIds(
final MoodleAPIRestTemplate restTemplate,
final Set<String> ids) {
try {
if (log.isDebugEnabled()) {
log.debug("LMS Setup: {} Get courses for ids: {}", this.lmsSetup, ids);
}
final String joinedIds = StringUtils.join(ids, Constants.COMMA);
final LinkedMultiValueMap<String, String> attributes = new LinkedMultiValueMap<>();
attributes.add(MoodleCourseAccess.MOODLE_COURSE_API_FIELD_NAME, MoodleCourseAccess.MOODLE_COURSE_API_IDS);
attributes.add(MoodleCourseAccess.MOODLE_COURSE_API_FIELD_VALUE, joinedIds);
final String coursePageJSON = callMoodleRestAPI(
restTemplate,
MoodleCourseAccess.MOODLE_COURSE_BY_FIELD_API_FUNCTION_NAME,
attributes);
final Courses courses = this.jsonMapper.readValue(
coursePageJSON,
Courses.class);
if (courses == null) {
log.error("No Courses response: LMS: {} API call: {}", this.lmsSetup,
MoodleCourseAccess.MOODLE_COURSE_BY_FIELD_API_FUNCTION_NAME);
return Collections.emptyList();
}
if (courses.warnings != null && !courses.warnings.isEmpty()) {
log.warn(
"There are warnings from Moodle response: Moodle: {} request: {} warnings: {} warning sample: {}",
this.lmsSetup,
MoodleCourseAccess.MOODLE_COURSE_BY_FIELD_API_FUNCTION_NAME,
courses.warnings.size(),
courses.warnings.iterator().next().toString());
if (log.isTraceEnabled()) {
log.trace("All warnings from Moodle: {}", courses.warnings.toString());
}
}
if (courses.courses == null || courses.courses.isEmpty()) {
log.warn("No courses found for ids: {} on LMS {}", ids, this.lmsSetup);
return Collections.emptyList();
}
return courses.courses;
} catch (final Exception e) {
log.error("LMS Setup: {} Unexpected error while trying to get courses for ids", this.lmsSetup, e);
return Collections.emptyList();
}
}
private String callMoodleRestAPI(
final MoodleAPIRestTemplate restTemplate,
final String function,
final MultiValueMap<String, String> queryAttributes) {
return this.moodleRestCall
.protectedRun(() -> restTemplate.callMoodleAPIFunction(
function,
queryAttributes))
.getOrThrow();
}
private Predicate<CourseQuizShort> getQuizFilter() {
final long now = Utils.getSecondsNow();
return quiz -> {
if (quiz.time_close == null || quiz.time_close == 0 || quiz.time_close > now) {
return true;
}
if (log.isDebugEnabled()) {
log.debug("LMS Setup: {} remove quiz {} end_time {} now {}",
this.lmsSetup,
quiz.name,
quiz.time_close,
now);
}
return false;
};
}
private Predicate<CourseDataShort> getCourseFilter() {
final long now = Utils.getSecondsNow();
return course -> {
if (course.start_date != null && course.start_date < this.fromCutTime) {
return false;
}
if (course.end_date == null || course.end_date == 0 || course.end_date > now) {
return true;
}
if (log.isDebugEnabled()) {
log.info("LMS Setup: {} remove course {} end_time {} now {}",
this.lmsSetup,
course.short_name,
course.end_date,
now);
}
return false;
};
}
private void syncCache() {
if (!this.cachedCourseData.isEmpty()) {
final Set<String> oldData = this.cachedCourseData
.keySet()
.stream()
.filter(id -> !this.newIds.contains(id))
.collect(Collectors.toSet());
synchronized (this.cachedCourseData) {
oldData.stream().forEach(this.cachedCourseData::remove);
}
}
this.newIds.clear();
}
/** Maps the Moodle course API course data */
@JsonIgnoreProperties(ignoreUnknown = true)
static final class CourseDataShort {
final String id;
final String short_name;
final String idnumber;
final Long start_date; // unix-time seconds UTC
final Long end_date; // unix-time seconds UTC
final Long time_created; // unix-time seconds UTC
final Collection<CourseQuizShort> quizzes = new ArrayList<>();
@JsonCreator
protected CourseDataShort(
@JsonProperty(value = "id") final String id,
@JsonProperty(value = "shortname") final String short_name,
@JsonProperty(value = "idnumber") final String idnumber,
@JsonProperty(value = "startdate") final Long start_date,
@JsonProperty(value = "enddate") final Long end_date,
@JsonProperty(value = "timecreated") final Long time_created) {
this.id = id;
this.short_name = short_name;
this.idnumber = idnumber;
this.start_date = start_date;
this.end_date = end_date;
this.time_created = time_created;
}
@Override
public int hashCode() {
final int prime = 31;
int result = 1;
result = prime * result + ((this.id == null) ? 0 : this.id.hashCode());
return result;
}
@Override
public boolean equals(final Object obj) {
if (this == obj)
return true;
if (obj == null)
return false;
if (getClass() != obj.getClass())
return false;
final CourseDataShort other = (CourseDataShort) obj;
if (this.id == null) {
if (other.id != null)
return false;
} else if (!this.id.equals(other.id))
return false;
return true;
}
}
@JsonIgnoreProperties(ignoreUnknown = true)
private static final class Courses {
final Collection<CourseDataShort> courses;
final Collection<Warning> warnings;
@JsonCreator
protected Courses(
@JsonProperty(value = "courses") final Collection<CourseDataShort> courses,
@JsonProperty(value = "warnings") final Collection<Warning> warnings) {
this.courses = courses;
this.warnings = warnings;
}
}
@JsonIgnoreProperties(ignoreUnknown = true)
static final class CourseQuizData {
final Collection<CourseQuizShort> quizzes;
final Collection<Warning> warnings;
@JsonCreator
protected CourseQuizData(
@JsonProperty(value = "quizzes") final Collection<CourseQuizShort> quizzes,
@JsonProperty(value = "warnings") final Collection<Warning> warnings) {
this.quizzes = quizzes;
this.warnings = warnings;
}
}
@JsonIgnoreProperties(ignoreUnknown = true)
static final class CourseQuizShort {
final String id;
final String course;
final String course_module;
final String name;
final Long time_open; // unix-time seconds UTC
final Long time_close; // unix-time seconds UTC
@JsonCreator
protected CourseQuizShort(
@JsonProperty(value = "id") final String id,
@JsonProperty(value = "course") final String course,
@JsonProperty(value = "coursemodule") final String course_module,
@JsonProperty(value = "name") final String name,
@JsonProperty(value = "timeopen") final Long time_open,
@JsonProperty(value = "timeclose") final Long time_close) {
this.id = id;
this.course = course;
this.course_module = course_module;
this.name = name;
this.time_open = time_open;
this.time_close = time_close;
}
}
}

9
src/main/java/ch/ethz/seb/sebserver/webservice/servicelayer/lms/impl/moodle/legacy/MoodleLmsAPITemplateFactory.java
View file

@ -20,7 +20,6 @@ import ch.ethz.seb.sebserver.gbl.Constants;
import ch.ethz.seb.sebserver.gbl.api.JSONMapper;
import ch.ethz.seb.sebserver.gbl.async.AsyncService;
import ch.ethz.seb.sebserver.gbl.client.ClientCredentialService;
import ch.ethz.seb.sebserver.gbl.model.institution.LmsSetup;
import ch.ethz.seb.sebserver.gbl.model.institution.LmsSetup.LmsType;
import ch.ethz.seb.sebserver.gbl.profile.WebServiceProfile;
import ch.ethz.seb.sebserver.gbl.util.Result;
@ -43,7 +42,6 @@ public class MoodleLmsAPITemplateFactory implements LmsAPITemplateFactory {
private final Environment environment;
private final ClientCredentialService clientCredentialService;
private final ClientHttpRequestFactoryService clientHttpRequestFactoryService;
private final ApplicationContext applicationContext;
private final String[] alternativeTokenRequestPaths;
protected MoodleLmsAPITemplateFactory(
@ -60,7 +58,6 @@ public class MoodleLmsAPITemplateFactory implements LmsAPITemplateFactory {
this.environment = environment;
this.clientCredentialService = clientCredentialService;
this.clientHttpRequestFactoryService = clientHttpRequestFactoryService;
this.applicationContext = applicationContext;
this.alternativeTokenRequestPaths = (alternativeTokenRequestPaths != null)
? StringUtils.split(alternativeTokenRequestPaths, Constants.LIST_SEPARATOR)
: null;
@ -76,11 +73,6 @@ public class MoodleLmsAPITemplateFactory implements LmsAPITemplateFactory {
return Result.tryCatch(() -> {
final LmsSetup lmsSetup = apiTemplateDataSupplier.getLmsSetup();
final MoodleCourseDataAsyncLoader asyncLoaderPrototype = this.applicationContext
.getBean(MoodleCourseDataAsyncLoader.class);
asyncLoaderPrototype.init(lmsSetup.getModelId());
final MoodleRestTemplateFactory restTemplateFactory = new MoodleRestTemplateFactoryImpl(
this.jsonMapper,
apiTemplateDataSupplier,
@ -92,7 +84,6 @@ public class MoodleLmsAPITemplateFactory implements LmsAPITemplateFactory {
this.jsonMapper,
this.asyncService,
restTemplateFactory,
asyncLoaderPrototype,
this.environment);
return new LmsAPITemplateAdapter(

5
src/main/java/ch/ethz/seb/sebserver/webservice/servicelayer/lms/impl/moodle/plugin/MoodlePluginCourseAccess.java
View file

@ -168,11 +168,6 @@ public class MoodlePluginCourseAccess extends AbstractCachedCourseAccess impleme
return LmsSetupTestResult.ofOkay(LmsType.MOODLE_PLUGIN);
}
@Override
public Result<List<QuizData>> getQuizzes(final FilterMap filterMap) {
return Result.ofError(new UnsupportedOperationException());
}
@Override
public void fetchQuizzes(final FilterMap filterMap, final AsyncQuizFetchBuffer asyncQuizFetchBuffer) {
try {

9
src/main/java/ch/ethz/seb/sebserver/webservice/servicelayer/lms/impl/olat/OlatLmsAPITemplate.java
View file

@ -170,15 +170,6 @@ public class OlatLmsAPITemplate extends AbstractCachedCourseAccess implements Lm
return LmsSetupTestResult.ofOkay(LmsType.OPEN_OLAT);
}
@Override
public Result<List<QuizData>> getQuizzes(final FilterMap filterMap) {
return this
.allQuizzesRequest(filterMap)
.map(quizzes -> quizzes.stream()
.filter(LmsAPIService.quizFilterPredicate(filterMap))
.collect(Collectors.toList()));
}
@Override
public void fetchQuizzes(final FilterMap filterMap, final AsyncQuizFetchBuffer asyncQuizFetchBuffer) {
this.allQuizzesRequest(filterMap)

4
src/test/java/ch/ethz/seb/sebserver/webservice/servicelayer/lms/impl/moodle/legacy/MoodleCourseAccessTest.java
View file

@ -80,7 +80,6 @@ public class MoodleCourseAccessTest {
new JSONMapper(),
this.asyncService,
moodleRestTemplateFactory,
null,
this.env);
final String examId = "123";
@ -129,7 +128,6 @@ public class MoodleCourseAccessTest {
new JSONMapper(),
this.asyncService,
moodleRestTemplateFactory,
null,
this.env);
final LmsSetupTestResult initAPIAccess = moodleCourseAccess.testCourseAccessAPI();
@ -152,7 +150,6 @@ public class MoodleCourseAccessTest {
new JSONMapper(),
this.asyncService,
moodleRestTemplateFactory,
null,
this.env);
final LmsSetupTestResult initAPIAccess = moodleCourseAccess.testCourseAccessAPI();
@ -174,7 +171,6 @@ public class MoodleCourseAccessTest {
new JSONMapper(),
this.asyncService,
moodleRestTemplateFactory,
null,
this.env);
final LmsSetupTestResult initAPIAccess = moodleCourseAccess.testCourseAccessAPI();