more resilient implementation of Moodle course API access

This commit is contained in:
anhefti 2020-12-09 08:07:08 +01:00
parent b74778a67d
commit bbf241b08e
7 changed files with 394 additions and 88 deletions

View file

@ -10,11 +10,9 @@ package ch.ethz.seb.sebserver.webservice.servicelayer.lms;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import java.util.stream.Collectors;
import org.apache.commons.lang3.StringUtils;
@ -72,14 +70,7 @@ public interface LmsAPITemplate {
*
* @param ids the Set of Quiz identifiers to get the QuizData for
* @return Collection of all QuizData from the given id set */
default Collection<Result<QuizData>> getQuizzes(final Set<String> ids) {
return getQuizzes(new FilterMap())
.getOrElse(Collections::emptyList)
.stream()
.filter(quiz -> ids.contains(quiz.id))
.map(Result::of)
.collect(Collectors.toList());
}
Collection<Result<QuizData>> getQuizzes(Set<String> ids);
/** Get all QuizData for the set of QuizData identifiers from LMS API in a collection
* of Result. If particular Quiz cannot be loaded because of errors or deletion,

View file

@ -37,6 +37,7 @@ public abstract class CourseAccess {
private static final Logger log = LoggerFactory.getLogger(CourseAccess.class);
protected final MemoizingCircuitBreaker<List<QuizData>> allQuizzesRequest;
protected final CircuitBreaker<List<QuizData>> quizzesRequest;
protected final CircuitBreaker<Chapters> chaptersRequest;
protected final CircuitBreaker<ExamineeAccountDetails> accountDetailRequest;
@ -49,6 +50,11 @@ public abstract class CourseAccess {
true,
Constants.HOUR_IN_MILLIS);
this.quizzesRequest = asyncService.createCircuitBreaker(
3,
Constants.MINUTE_IN_MILLIS,
Constants.MINUTE_IN_MILLIS);
this.chaptersRequest = asyncService.createCircuitBreaker(
3,
Constants.SECOND_IN_MILLIS * 10,
@ -72,22 +78,29 @@ public abstract class CourseAccess {
public Result<Collection<Result<QuizData>>> getQuizzesFromCache(final Set<String> ids) {
return Result.tryCatch(() -> {
final List<QuizData> cached = this.allQuizzesRequest.getCached();
if (cached == null) {
throw new RuntimeException("No cached quizzes");
}
final List<QuizData> available = (cached != null)
? cached
: quizzesSupplier(ids).get();
final Map<String, QuizData> cacheMapping = cached
final Map<String, QuizData> quizMapping = available
.stream()
.collect(Collectors.toMap(q -> q.id, Function.identity()));
if (!cacheMapping.keySet().containsAll(ids)) {
throw new RuntimeException("Not all requested quizzes cached");
if (!quizMapping.keySet().containsAll(ids)) {
final Map<String, QuizData> collect = quizzesSupplier(ids).get()
.stream()
.collect(Collectors.toMap(qd -> qd.id, Function.identity()));
if (collect != null) {
quizMapping.clear();
quizMapping.putAll(collect);
}
}
return ids
.stream()
.map(id -> {
final QuizData q = cacheMapping.get(id);
final QuizData q = quizMapping.get(id);
return (q == null)
? Result.<QuizData> ofError(new NoSuchElementException("Quiz with id: " + id))
: Result.of(q);
@ -96,6 +109,11 @@ public abstract class CourseAccess {
});
}
/* Note: this can be overwritten to load missing requested quiz data from specified LMS by id */
protected Map<String, QuizData> loadMissingData(final Set<String> ids) {
throw new RuntimeException("Not all requested quizzes cached");
}
public Result<List<QuizData>> getQuizzes(final FilterMap filterMap) {
return this.allQuizzesRequest.get()
.map(LmsAPIService.quizzesFilterFunction(filterMap));
@ -130,6 +148,8 @@ public abstract class CourseAccess {
Collections.emptyMap());
}
protected abstract Supplier<List<QuizData>> quizzesSupplier(final Set<String> ids);
protected abstract Supplier<List<QuizData>> allQuizzesSupplier();
protected abstract Supplier<Chapters> getCourseChaptersSupplier(final String courseId);

View file

@ -13,6 +13,7 @@ import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.function.Supplier;
import java.util.stream.Collectors;
@ -164,6 +165,13 @@ final class OpenEdxCourseAccess extends CourseAccess {
});
}
@Override
protected Supplier<List<QuizData>> quizzesSupplier(final Set<String> ids) {
return () -> getRestTemplate()
.map(template -> this.collectQuizzes(template, ids))
.getOrThrow();
}
@Override
protected Supplier<List<QuizData>> allQuizzesSupplier() {
return () -> getRestTemplate()
@ -187,6 +195,25 @@ final class OpenEdxCourseAccess extends CourseAccess {
};
}
private ArrayList<QuizData> collectQuizzes(final OAuth2RestTemplate restTemplate, final Set<String> ids) {
final String externalStartURI = getExternalLMSServerAddress(this.lmsSetup);
return collectCourses(
this.lmsSetup.lmsApiUrl + OPEN_EDX_DEFAULT_COURSE_ENDPOINT,
restTemplate,
ids)
.stream()
.reduce(
new ArrayList<>(),
(list, courseData) -> {
list.add(quizDataOf(this.lmsSetup, courseData, externalStartURI));
return list;
},
(list1, list2) -> {
list1.addAll(list2);
return list1;
});
}
private ArrayList<QuizData> collectAllQuizzes(final OAuth2RestTemplate restTemplate) {
final String externalStartURI = getExternalLMSServerAddress(this.lmsSetup);
return collectAllCourses(
@ -229,6 +256,29 @@ final class OpenEdxCourseAccess extends CourseAccess {
return _externalStartURI;
}
private List<CourseData> collectCourses(
final String pageURI,
final OAuth2RestTemplate restTemplate,
final Set<String> ids) {
final List<CourseData> collector = new ArrayList<>();
EdXPage page = getEdxPage(pageURI, restTemplate).getBody();
if (page != null) {
collector.addAll(page.results);
while (page != null && StringUtils.isNotBlank(page.next)) {
page = getEdxPage(page.next, restTemplate).getBody();
if (page != null) {
page.results
.stream()
.filter(cd -> ids.contains(cd.id))
.forEach(collector::add);
}
}
}
return collector;
}
private List<CourseData> collectAllCourses(final String pageURI, final OAuth2RestTemplate restTemplate) {
final List<CourseData> collector = new ArrayList<>();
EdXPage page = getEdxPage(pageURI, restTemplate).getBody();

View file

@ -12,7 +12,10 @@ import java.util.Arrays;
import java.util.Collection;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.function.Function;
import java.util.stream.Collectors;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -67,6 +70,22 @@ final class OpenEdxLmsAPITemplate implements LmsAPITemplate {
return this.openEdxCourseAccess.getQuizzes(filterMap);
}
@Override
public Collection<Result<QuizData>> getQuizzes(final Set<String> ids) {
final Map<String, QuizData> mapping = this.openEdxCourseAccess
.quizzesSupplier(ids)
.get()
.stream()
.collect(Collectors.toMap(qd -> qd.id, Function.identity()));
return ids.stream()
.map(id -> {
final QuizData data = mapping.get(id);
return (data == null) ? Result.<QuizData> ofRuntimeError("Missing id: " + id) : Result.of(data);
})
.collect(Collectors.toList());
}
@Override
public Result<QuizData> getQuizFromCache(final String id) {
return getQuizzesFromCache(new HashSet<>(Arrays.asList(id)))

View file

@ -8,11 +8,14 @@
package ch.ethz.seb.sebserver.webservice.servicelayer.lms.impl.moodle;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.function.Function;
import java.util.function.Supplier;
import java.util.stream.Collectors;
@ -28,7 +31,9 @@ import org.springframework.util.MultiValueMap;
import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.fasterxml.jackson.core.JsonParseException;
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.JsonMappingException;
import ch.ethz.seb.sebserver.gbl.Constants;
import ch.ethz.seb.sebserver.gbl.api.JSONMapper;
@ -51,10 +56,20 @@ public class MoodleCourseAccess extends CourseAccess {
private static final Logger log = LoggerFactory.getLogger(MoodleCourseAccess.class);
private static final String MOODLE_QUIZ_START_URL_PATH = "mod/quiz/view.php?id=";
private static final String MOODLE_COURSE_API_FUNCTION_NAME = "core_course_get_courses";
private static final String MOODLE_USER_PROFILE_API_FUNCTION_NAME = "core_user_get_users_by_field";
private static final String MOODLE_QUIZ_API_FUNCTION_NAME = "mod_quiz_get_quizzes_by_courses";
private static final String MOODLE_COURSE_API_COURSE_IDS = "courseids";
private static final String MOODLE_COURSE_API_IDS = "ids";
private static final String MOODLE_COURSE_SEARCH_API_FUNCTION_NAME = "core_course_search_courses";
private static final String MOODLE_COURSE_BY_FIELD_API_FUNCTION_NAME = "core_course_get_courses_by_field";
private static final String MOODLE_COURSE_API_FIELD_NAME = "field";
private static final String MOODLE_COURSE_API_FIELD_VALUE = "value";
private static final String MOODLE_COURSE_API_SEARCH_CRITERIA_NAME = "criterianame";
private static final String MOODLE_COURSE_API_SEARCH_CRITERIA_VALUE = "criteriavalue";
private static final String MOODLE_COURSE_API_SEARCH_PAGE = "page";
private static final String MOODLE_COURSE_API_SEARCH_PAGE_SIZE = "perpage";
private final JSONMapper jsonMapper;
private final LmsSetup lmsSetup;
@ -150,6 +165,14 @@ public class MoodleCourseAccess extends CourseAccess {
return LmsSetupTestResult.ofOkay();
}
@Override
protected Supplier<List<QuizData>> quizzesSupplier(final Set<String> ids) {
return () -> getRestTemplate()
.map(template -> getQuizzesForIds(template, ids))
.getOrThrow();
}
@Override
protected Supplier<List<QuizData>> allQuizzesSupplier() {
return () -> getRestTemplate()
@ -166,7 +189,7 @@ public class MoodleCourseAccess extends CourseAccess {
final String urlPrefix = (this.lmsSetup.lmsApiUrl.endsWith(Constants.URL_PATH_SEPARATOR))
? this.lmsSetup.lmsApiUrl + MOODLE_QUIZ_START_URL_PATH
: this.lmsSetup.lmsApiUrl + Constants.URL_PATH_SEPARATOR + MOODLE_QUIZ_START_URL_PATH;
return collectAllCourses(restTemplate)
return getAllQuizzes(restTemplate)
.stream()
.reduce(
new ArrayList<>(),
@ -183,30 +206,13 @@ public class MoodleCourseAccess extends CourseAccess {
});
}
private List<CourseData> collectAllCourses(final MoodleAPIRestTemplate restTemplate) {
private List<CourseData> getAllQuizzes(final MoodleAPIRestTemplate restTemplate) {
try {
// first get courses from Moodle per page
final Map<String, CourseData> courseData = new HashMap<>();
// first get courses from Moodle...
final String coursesJSON = restTemplate.callMoodleAPIFunction(MOODLE_COURSE_API_FUNCTION_NAME);
Map<String, CourseData> courseData = this.jsonMapper.<Collection<CourseData>> readValue(
coursesJSON,
new TypeReference<Collection<CourseData>>() {
})
.stream()
.collect(Collectors.toMap(d -> d.id, Function.identity()));
if (courseData.size() > 100) {
log.warn(
"Got more then 100 courses form Moodle: size {}. Trim it to the latest 100 courses",
courseData.size());
final long nowInMillis = DateTime.now(DateTimeZone.UTC).getMillis();
courseData = courseData.values().stream()
.filter(cd -> cd.end_date != null && cd.end_date.longValue() < nowInMillis)
.sorted((cd1, cd2) -> cd1.start_date.compareTo(cd2.start_date))
.limit(100)
.collect(Collectors.toMap(cd -> cd.id, Function.identity()));
}
final Collection<CourseData> coursesPage = getCoursesPage(restTemplate, 0, 1000);
courseData.putAll(coursesPage.stream().collect(Collectors.toMap(cd -> cd.id, Function.identity())));
// then get all quizzes of courses and filter
final LinkedMultiValueMap<String, String> attributes = new LinkedMultiValueMap<>();
@ -234,7 +240,154 @@ public class MoodleCourseAccess extends CourseAccess {
.filter(c -> !c.quizzes.isEmpty())
.collect(Collectors.toList());
} catch (final Exception e) {
throw new RuntimeException("Unexpected exception while trying to get course data: ", e);
log.error("Unexpected exception while trying to get course data: ", e);
return Collections.emptyList();
}
}
private Collection<CourseData> getCoursesPage(
final MoodleAPIRestTemplate restTemplate,
final int page,
final int size) throws JsonParseException, JsonMappingException, IOException {
try {
final long aYearAgo = DateTime.now(DateTimeZone.UTC).minusYears(1).getMillis();
// get course ids per page
final LinkedMultiValueMap<String, String> attributes = new LinkedMultiValueMap<>();
attributes.add(MOODLE_COURSE_API_SEARCH_CRITERIA_NAME, "search");
attributes.add(MOODLE_COURSE_API_SEARCH_CRITERIA_VALUE, "");
attributes.add(MOODLE_COURSE_API_SEARCH_PAGE, String.valueOf(page));
attributes.add(MOODLE_COURSE_API_SEARCH_PAGE_SIZE, String.valueOf(size));
final String courseKeyPageJSON = restTemplate.callMoodleAPIFunction(
MOODLE_COURSE_SEARCH_API_FUNCTION_NAME,
attributes);
final CoursePage keysPage = this.jsonMapper.readValue(
courseKeyPageJSON,
CoursePage.class);
// get courses
final Set<String> ids = keysPage.courseKeys
.stream()
.map(key -> key.id)
.collect(Collectors.toSet());
final long now = DateTime.now(DateTimeZone.UTC).getMillis();
return getCoursesForIds(restTemplate, ids)
.stream()
.filter(course -> course.time_created == null
|| course.time_created.longValue() > aYearAgo
|| (course.end_date == null
|| (course.end_date <= 0
|| course.end_date > now)))
.collect(Collectors.toList());
} catch (final Exception e) {
log.error("Unexpected error while trying to get courses page: ", e);
return Collections.emptyList();
}
}
private List<QuizData> getQuizzesForIds(
final MoodleAPIRestTemplate restTemplate,
final Set<String> quizIds) {
try {
if (log.isDebugEnabled()) {
log.debug("Get quizzes for ids: {}", quizIds);
}
final Map<String, CourseData> courseData = getCoursesForIds(
restTemplate,
quizIds.stream()
.map(MoodleCourseAccess::getCourseId)
.collect(Collectors.toSet()))
.stream()
.collect(Collectors.toMap(cd -> cd.id, Function.identity()));
final List<String> courseIds = new ArrayList<>(courseData.keySet());
if (courseIds.isEmpty()) {
return Collections.emptyList();
}
if (courseIds.size() == 1) {
// NOTE: This is a workaround because the Moodle API do not support lists with only one element.
courseIds.add("0");
}
// then get all quizzes of courses and filter
final LinkedMultiValueMap<String, String> attributes = new LinkedMultiValueMap<>();
attributes.put(MOODLE_COURSE_API_COURSE_IDS, courseIds);
final String quizzesJSON = restTemplate.callMoodleAPIFunction(
MOODLE_QUIZ_API_FUNCTION_NAME,
attributes);
final CourseQuizData courseQuizData = this.jsonMapper.readValue(
quizzesJSON,
CourseQuizData.class);
final Map<String, CourseData> finalCourseDataRef = courseData;
courseQuizData.quizzes
.forEach(quiz -> {
final CourseData course = finalCourseDataRef.get(quiz.course);
if (course != null) {
course.quizzes.add(quiz);
}
});
final String urlPrefix = (this.lmsSetup.lmsApiUrl.endsWith(Constants.URL_PATH_SEPARATOR))
? this.lmsSetup.lmsApiUrl + MOODLE_QUIZ_START_URL_PATH
: this.lmsSetup.lmsApiUrl + Constants.URL_PATH_SEPARATOR + MOODLE_QUIZ_START_URL_PATH;
return courseData.values()
.stream()
.filter(c -> !c.quizzes.isEmpty())
.reduce(
new ArrayList<>(),
(list, cd) -> {
list.addAll(quizDataOf(
this.lmsSetup,
cd,
urlPrefix));
return list;
},
(list1, list2) -> {
list1.addAll(list2);
return list1;
});
} catch (final Exception e) {
log.error("Unexpected error while trying to get quizzes for ids", e);
return Collections.emptyList();
}
}
private Collection<CourseData> getCoursesForIds(
final MoodleAPIRestTemplate restTemplate,
final Set<String> ids) {
try {
if (log.isDebugEnabled()) {
log.debug("Get courses for ids: {}", ids);
}
final String joinedIds = StringUtils.join(ids, Constants.COMMA);
final LinkedMultiValueMap<String, String> attributes = new LinkedMultiValueMap<>();
attributes.add(MOODLE_COURSE_API_FIELD_NAME, MOODLE_COURSE_API_IDS);
attributes.add(MOODLE_COURSE_API_FIELD_VALUE, joinedIds);
final String coursePageJSON = restTemplate.callMoodleAPIFunction(
MOODLE_COURSE_BY_FIELD_API_FUNCTION_NAME,
attributes);
return this.jsonMapper.<Courses> readValue(
coursePageJSON,
Courses.class).courses;
} catch (final Exception e) {
log.error("Unexpected error while trying to get courses for ids", e);
return Collections.emptyList();
}
}
@ -259,7 +412,11 @@ public class MoodleCourseAccess extends CourseAccess {
final String startURI = uriPrefix + courseQuizData.course_module;
additionalAttrs.put(QuizData.ATTR_ADDITIONAL_TIME_LIMIT, String.valueOf(courseQuizData.time_limit));
return new QuizData(
getInternalQuizId(courseQuizData.course_module, courseData.short_name, courseData.idnumber),
getInternalQuizId(
courseQuizData.course_module,
courseData.id,
courseData.short_name,
courseData.idnumber),
lmsSetup.getInstitutionId(),
lmsSetup.id,
lmsSetup.getLmsType(),
@ -289,6 +446,93 @@ public class MoodleCourseAccess extends CourseAccess {
return Result.of(this.restTemplate);
}
public static final String getInternalQuizId(
final String quizId,
final String courseId,
final String shortname,
final String idnumber) {
return StringUtils.join(
new String[] {
quizId,
courseId,
StringUtils.isNotBlank(shortname) ? shortname : Constants.EMPTY_NOTE,
StringUtils.isNotBlank(idnumber) ? idnumber : Constants.EMPTY_NOTE
},
Constants.COLON);
}
public static final String getQuizId(final String internalQuizId) {
if (StringUtils.isBlank(internalQuizId)) {
return null;
}
return StringUtils.split(internalQuizId, Constants.COLON)[0];
}
public static final String getCourseId(final String internalQuizId) {
if (StringUtils.isBlank(internalQuizId)) {
return null;
}
return StringUtils.split(internalQuizId, Constants.COLON)[1];
}
public static final String getShortname(final String internalQuizId) {
if (StringUtils.isBlank(internalQuizId)) {
return null;
}
final String[] split = StringUtils.split(internalQuizId, Constants.COLON);
if (split.length < 3) {
return null;
}
final String shortName = split[2];
return shortName.equals(Constants.EMPTY_NOTE) ? null : shortName;
}
public static final String getIdnumber(final String internalQuizId) {
if (StringUtils.isBlank(internalQuizId)) {
return null;
}
final String[] split = StringUtils.split(internalQuizId, Constants.COLON);
if (split.length < 4) {
return null;
}
final String idNumber = split[3];
return idNumber.equals(Constants.EMPTY_NOTE) ? null : idNumber;
}
// ---- Mapping Classes ---
@JsonIgnoreProperties(ignoreUnknown = true)
static final class CoursePage {
final Collection<CourseKey> courseKeys;
public CoursePage(
@JsonProperty(value = "courses") final Collection<CourseKey> courseKeys) {
this.courseKeys = courseKeys;
}
}
@JsonIgnoreProperties(ignoreUnknown = true)
static final class CourseKey {
final String id;
final String short_name;
@JsonCreator
protected CourseKey(
@JsonProperty(value = "id") final String id,
@JsonProperty(value = "shortname") final String short_name) {
this.id = id;
this.short_name = short_name;
}
}
/** Maps the Moodle course API course data */
@JsonIgnoreProperties(ignoreUnknown = true)
static final class CourseData {
@ -328,51 +572,14 @@ public class MoodleCourseAccess extends CourseAccess {
}
public static final String getInternalQuizId(final String quizId, final String shortname, final String idnumber) {
final StringBuilder sb = new StringBuilder(quizId);
if (StringUtils.isNotEmpty(shortname)) {
sb.insert(0, ":").insert(0, shortname);
}
if (StringUtils.isNotEmpty(idnumber)) {
sb.insert(0, ":").insert(0, idnumber);
}
return sb.toString();
}
@JsonIgnoreProperties(ignoreUnknown = true)
static final class Courses {
final Collection<CourseData> courses;
public static final String getQuizId(final String internalQuizId) {
if (StringUtils.isBlank(internalQuizId)) {
return null;
}
final String[] ids = StringUtils.split(internalQuizId, Constants.COLON);
return ids[ids.length - 1];
}
public static final String getShortname(final String internalQuizId) {
if (StringUtils.isBlank(internalQuizId)) {
return null;
}
final String[] ids = StringUtils.split(internalQuizId, Constants.COLON);
if (ids.length == 3) {
return ids[1];
} else if (ids.length == 2) {
return ids[0];
} else {
return null;
}
}
public static final String getIdnumber(final String internalQuizId) {
if (StringUtils.isBlank(internalQuizId)) {
return null;
}
final String[] ids = StringUtils.split(internalQuizId, Constants.COLON);
if (ids.length == 3) {
return ids[0];
} else {
return null;
@JsonCreator
protected Courses(
@JsonProperty(value = "courses") final Collection<CourseData> courses) {
this.courses = courses;
}
}

View file

@ -10,7 +10,10 @@ package ch.ethz.seb.sebserver.webservice.servicelayer.lms.impl.moodle;
import java.util.Collection;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.function.Function;
import java.util.stream.Collectors;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -65,6 +68,22 @@ public class MoodleLmsAPITemplate implements LmsAPITemplate {
return this.moodleCourseAccess.getQuizzes(filterMap);
}
@Override
public Collection<Result<QuizData>> getQuizzes(final Set<String> ids) {
final Map<String, QuizData> mapping = this.moodleCourseAccess
.quizzesSupplier(ids)
.get()
.stream()
.collect(Collectors.toMap(qd -> qd.id, Function.identity()));
return ids.stream()
.map(id -> {
final QuizData data = mapping.get(id);
return (data == null) ? Result.<QuizData> ofRuntimeError("Missing id: " + id) : Result.of(data);
})
.collect(Collectors.toList());
}
@Override
public Collection<Result<QuizData>> getQuizzesFromCache(final Set<String> ids) {
return this.moodleCourseAccess.getQuizzesFromCache(ids)

View file

@ -10,9 +10,9 @@ sebserver.gui.webservice.apipath=/admin-api/v1
# defines the polling interval that is used to poll the webservice for client connection data on a monitored exam page
sebserver.gui.webservice.poll-interval=1000
sebserver.gui.webservice.mock-lms-enabled=false
sebserver.gui.webservice.mock-lms-enabled=true
sebserver.gui.webservice.edx-lms-enabled=true
sebserver.gui.webservice.moodle-lms-enabled=false
sebserver.gui.webservice.moodle-lms-enabled=true
sebserver.gui.theme=css/sebserver.css