zervo/seb-server
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

simplified LMS API

Browse source
This commit is contained in:
anhefti 2021-05-17 21:19:01 +02:00
parent 0b00995aa7
commit 213cf443e1
8 changed files with 170 additions and 157 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

18
src/main/java/ch/ethz/seb/sebserver/webservice/servicelayer/dao/impl/ExamDAOImpl.java
View file

@ -806,11 +806,9 @@ public class ExamDAOImpl implements ExamDAO {
// get and map quizzes
final Map<String, QuizData> quizzes = this.lmsAPIService
.getLmsAPITemplate(lmsSetupId)
.map(template -> getQuizzesFromLMS(template, recordMapping.keySet()))
.onError(error -> log.error("Failed to get quizzes for exams: ", error))
.getOr(Collections.emptyList())
.flatMap(template -> template.getQuizzes(recordMapping.keySet()))
.getOrElse(() -> Collections.emptyList())
.stream()
.flatMap(Result::skipOnError)
.collect(Collectors.toMap(q -> q.id, Function.identity()));
if (records.size() != quizzes.size()) {
@ -858,18 +856,6 @@ public class ExamDAOImpl implements ExamDAO {
});
}
private Collection<Result<QuizData>> getQuizzesFromLMS(
final LmsAPITemplate template,
final Set<String> ids) {
try {
return template.getQuizzes(ids);
} catch (final Exception e) {
log.error("Unexpected error while using LmsAPITemplate to get quizzes: ", e);
return Collections.emptyList();
}
}
private QuizData getQuizData(
final Map<String, QuizData> quizzes,
final String externalId,

42
src/main/java/ch/ethz/seb/sebserver/webservice/servicelayer/lms/LmsAPITemplate.java
View file

@ -12,7 +12,7 @@ import java.util.Collection;
import java.util.List;
import java.util.Set;
import ch.ethz.seb.sebserver.gbl.async.MemoizingCircuitBreaker;
import ch.ethz.seb.sebserver.gbl.async.CircuitBreaker;
import ch.ethz.seb.sebserver.gbl.model.exam.Chapters;
import ch.ethz.seb.sebserver.gbl.model.exam.Exam;
import ch.ethz.seb.sebserver.gbl.model.exam.QuizData;
@ -22,6 +22,7 @@ import ch.ethz.seb.sebserver.gbl.model.institution.LmsSetupTestResult;
import ch.ethz.seb.sebserver.gbl.model.user.ExamineeAccountDetails;
import ch.ethz.seb.sebserver.gbl.util.Result;
import ch.ethz.seb.sebserver.webservice.servicelayer.dao.FilterMap;
import ch.ethz.seb.sebserver.webservice.servicelayer.lms.impl.AbstractCachedCourseAccess;
import ch.ethz.seb.sebserver.webservice.servicelayer.lms.impl.AbstractCourseAccess;
/** Defines an LMS API access template to build SEB Server LMS integration.
@ -41,28 +42,18 @@ import ch.ethz.seb.sebserver.webservice.servicelayer.lms.impl.AbstractCourseAcce
* All course API requests of this template shall not block and return as fast as possible
* with the best result it can provide for the time on that the request was made.
* </p>
* Each request to a remote LMS shall be executed within a protected call such that the
* request don't block the API call as well as do not attack the remote LMS with endless
* requests on failure.</br>
* Therefore the abstract class {@link AbstractCourseAccess} defines protected calls
* for different API calls by using {@link CircuitBreaker}. documentation on the class for
* more information.
* </p>
* Since the course API requests course data from potentially thousands of existing and
* active courses, the course API can implement some caches if needed.</br>
* A cache in the course API has manly two purposes; The first and prior purpose is to
* be able to provide course data as fast as possible even if the LMS is not available or
* busy at the time. The second purpose is to guarantee fast data access for the system
* if this is needed and data actuality has second priority.</br>
* Therefore usual get quiz data functions like {@link #getQuizzes(FilterMap filterMap) },
* {@link #getQuizzes(Set<String> ids) } and {@link #getQuiz(final String id) }
* shall always first try to connect to the LMS and request the specified data from the LMS.
* If this succeeds the cache shall be updated with the received quizzes data and return them.
* If this is not possible within a certain time, the implementation shall get as much of the
* requested data from the cache and return them to the caller to not block the call too long
* and allow the caller to return fast and present as much data as possible.</br>
* This can be done with a {@link MemoizingCircuitBreaker} or a simple {@link CircuitBreaker}
* with a separated cache, for example. The abstract implementation; {@link AbstractCourseAccess}
* provides already defined wrapped circuit breaker for each call. To use it, just extend the
* abstract class and implement the needed suppliers.</br>
* On the other hand, dedicated cache access functions like {@link #getQuizzesFromCache(Set<String> ids) }
* shall always first look into the cache to geht the requested data and if not
* available, call the LMS and request the data from the LMS. If partial data is needed to get
* be requested from the LMS, this functions shall also update the catch with the requested
* and cache missed data afterwards.
* active courses, the course API can implement some short time caches if needed.</br>
* The abstract class {@link AbstractCachedCourseAccess} defines such a short time
* cache for all implementing classes using EH-Cache. See documentation on the class for
* more information.
* </p>
* <b>SEB restriction API</b></br>
* For this API we need no caching since this is mostly about pushing data to the LMS for the LMS
@ -120,15 +111,14 @@ public interface LmsAPITemplate {
Result<List<QuizData>> getQuizzes(FilterMap filterMap);
/** Get all {@link QuizData } for the set of {@link QuizData } identifiers from LMS API in a collection
* of Result. If particular quiz cannot be loaded because of errors or deletion,
* the Result will have an error reference.
* of Result. If particular quizzes cannot be loaded because of errors or deletion,
* the the referencing QuizData will not be in the resulting list and an error is logged.
*
* @param ids the Set of Quiz identifiers to get the {@link QuizData } for
* @return Collection of all {@link QuizData } from the given id set */
Collection<Result<QuizData>> getQuizzes(Set<String> ids);
Result<Collection<QuizData>> getQuizzes(Set<String> ids);
/** Get the quiz data with specified identifier.
*
*
* @param id the quiz data identifier
* @return Result refer to the quiz data or to an error when happened */

60
src/main/java/ch/ethz/seb/sebserver/webservice/servicelayer/lms/impl/AbstractCourseAccess.java
View file

@ -8,6 +8,7 @@
package ch.ethz.seb.sebserver.webservice.servicelayer.lms.impl;
import java.util.Collection;
import java.util.Collections;
import java.util.List;
import java.util.Set;
@ -44,7 +45,11 @@ public abstract class AbstractCourseAccess {
}
/** CircuitBreaker for protected quiz and course data requests */
protected final CircuitBreaker<List<QuizData>> quizzesRequest;
protected final CircuitBreaker<List<QuizData>> allQuizzesRequest;
/** CircuitBreaker for protected quiz and course data requests */
protected final CircuitBreaker<Collection<QuizData>> quizzesRequest;
/** CircuitBreaker for protected quiz and course data requests */
protected final CircuitBreaker<QuizData> quizRequest;
/** CircuitBreaker for protected chapter data requests */
protected final CircuitBreaker<Chapters> chaptersRequest;
/** CircuitBreaker for protected examinee account details requests */
@ -54,6 +59,20 @@ public abstract class AbstractCourseAccess {
final AsyncService asyncService,
final Environment environment) {
this.allQuizzesRequest = asyncService.createCircuitBreaker(
environment.getProperty(
"sebserver.webservice.circuitbreaker.quizzesRequest.attempts",
Integer.class,
3),
environment.getProperty(
"sebserver.webservice.circuitbreaker.quizzesRequest.blockingTime",
Long.class,
Constants.MINUTE_IN_MILLIS),
environment.getProperty(
"sebserver.webservice.circuitbreaker.quizzesRequest.timeToRecover",
Long.class,
Constants.MINUTE_IN_MILLIS));
this.quizzesRequest = asyncService.createCircuitBreaker(
environment.getProperty(
"sebserver.webservice.circuitbreaker.quizzesRequest.attempts",
@ -68,6 +87,20 @@ public abstract class AbstractCourseAccess {
Long.class,
Constants.MINUTE_IN_MILLIS));
this.quizRequest = asyncService.createCircuitBreaker(
environment.getProperty(
"sebserver.webservice.circuitbreaker.quizzesRequest.attempts",
Integer.class,
3),
environment.getProperty(
"sebserver.webservice.circuitbreaker.quizzesRequest.blockingTime",
Long.class,
Constants.MINUTE_IN_MILLIS),
environment.getProperty(
"sebserver.webservice.circuitbreaker.quizzesRequest.timeToRecover",
Long.class,
Constants.MINUTE_IN_MILLIS));
this.chaptersRequest = asyncService.createCircuitBreaker(
environment.getProperty(
"sebserver.webservice.circuitbreaker.chaptersRequest.attempts",
@ -97,8 +130,18 @@ public abstract class AbstractCourseAccess {
Constants.SECOND_IN_MILLIS * 10));
}
public Result<List<QuizData>> getQuizzes(final FilterMap filterMap) {
return this.quizzesRequest.protectedRun(allQuizzesSupplier(filterMap));
public Result<List<QuizData>> protectedQuizzesRequest(final FilterMap filterMap) {
return this.allQuizzesRequest.protectedRun(allQuizzesSupplier(filterMap));
}
public Collection<QuizData> protectedQuizzesRequest(final Set<String> ids) {
return this.quizzesRequest.protectedRun(quizzesSupplier(ids))
.onError(error -> log.error("Failed to get QuizData for ids: ", error))
.getOrElse(() -> Collections.emptyList());
}
public Result<QuizData> protectedQuizRequest(final String id) {
return this.quizRequest.protectedRun(quizSupplier(id));
}
public Result<ExamineeAccountDetails> getExamineeAccountDetails(final String examineeSessionId) {
@ -116,7 +159,7 @@ public abstract class AbstractCourseAccess {
.getOr(examineeSessionId);
}
protected Result<Chapters> getCourseChapters(final String courseId) {
public Result<Chapters> getCourseChapters(final String courseId) {
return this.chaptersRequest.protectedRun(getCourseChaptersSupplier(courseId));
}
@ -134,12 +177,15 @@ public abstract class AbstractCourseAccess {
Collections.emptyMap());
}
/** Provides a supplier for the quiz data request to use within the circuit breaker */
protected abstract Supplier<List<QuizData>> quizzesSupplier(final Set<String> ids);
/** Provides a supplier to supply request to use within the circuit breaker */
protected abstract Supplier<List<QuizData>> allQuizzesSupplier(final FilterMap filterMap);
/** Provides a supplier for the quiz data request to use within the circuit breaker */
protected abstract Supplier<Collection<QuizData>> quizzesSupplier(final Set<String> ids);
/** Provides a supplier for the quiz data request to use within the circuit breaker */
protected abstract Supplier<QuizData> quizSupplier(final String id);
/** Provides a supplier for the course chapter data request to use within the circuit breaker */
protected abstract Supplier<Chapters> getCourseChaptersSupplier(final String courseId);

89
src/main/java/ch/ethz/seb/sebserver/webservice/servicelayer/lms/impl/edx/OpenEdxCourseAccess.java
View file

@ -12,12 +12,10 @@ import java.net.URL;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.NoSuchElementException;
import java.util.Set;
import java.util.function.Supplier;
import java.util.stream.Collectors;
@ -198,57 +196,25 @@ final class OpenEdxCourseAccess extends AbstractCachedCourseAccess {
.getOrThrow();
}
public Collection<Result<QuizData>> getQuizzesFromCache(final Set<String> ids) {
final HashSet<String> leftIds = new HashSet<>(ids);
final Collection<Result<QuizData>> result = new ArrayList<>();
ids.stream()
.map(this::getQuizFromCache)
.forEach(q -> {
if (q != null) {
leftIds.remove(q.id);
result.add(Result.of(q));
}
});
@Override
protected Supplier<QuizData> quizSupplier(final String id) {
return () -> {
final LmsSetup lmsSetup = getApiTemplateDataSupplier().getLmsSetup();
final String externalStartURI = getExternalLMSServerAddress(lmsSetup);
final QuizData quizData = quizDataOf(
lmsSetup,
this.getOneCourse(id, this.restTemplate, id),
externalStartURI);
if (!leftIds.isEmpty()) {
super.quizzesRequest.protectedRun(this.quizzesSupplier(leftIds))
.onError(error -> log.error("Failed to get quizzes by ids: ", error))
.getOrElse(() -> Collections.emptyList())
.stream()
.forEach(q -> {
leftIds.remove(q.id);
result.add(Result.of(q));
});
}
if (!leftIds.isEmpty()) {
leftIds.forEach(q -> result.add(Result.ofError(new NoSuchElementException())));
}
return result;
}
public QuizData getQuizFromCache(final String id) {
return super.getFromCache(id);
}
public QuizData getQuizFromLMS(final String id) {
final LmsSetup lmsSetup = getApiTemplateDataSupplier().getLmsSetup();
final String externalStartURI = getExternalLMSServerAddress(lmsSetup);
final QuizData quizData = quizDataOf(
lmsSetup,
this.getOneCourse(id, this.restTemplate, id),
externalStartURI);
if (quizData != null) {
super.putToCache(quizData);
}
return quizData;
if (quizData != null) {
super.putToCache(quizData);
}
return quizData;
};
}
@Override
protected Supplier<List<QuizData>> quizzesSupplier(final Set<String> ids) {
protected Supplier<Collection<QuizData>> quizzesSupplier(final Set<String> ids) {
if (ids.size() == 1) {
return () -> {
@ -295,6 +261,31 @@ final class OpenEdxCourseAccess extends AbstractCachedCourseAccess {
};
}
public Result<Collection<QuizData>> getQuizzesFromCache(final Set<String> ids) {
return Result.tryCatch(() -> {
final HashSet<String> leftIds = new HashSet<>(ids);
final Collection<QuizData> result = new ArrayList<>();
ids.stream()
.map(this::getQuizFromCache)
.forEach(q -> {
if (q != null) {
leftIds.remove(q.id);
result.add(q);
}
});
if (!leftIds.isEmpty()) {
result.addAll(super.protectedQuizzesRequest(leftIds));
}
return result;
});
}
public QuizData getQuizFromCache(final String id) {
return super.getFromCache(id);
}
private ArrayList<QuizData> collectQuizzes(final OAuth2RestTemplate restTemplate, final Set<String> ids) {
final LmsSetup lmsSetup = getApiTemplateDataSupplier().getLmsSetup();
final String externalStartURI = getExternalLMSServerAddress(lmsSetup);

16
src/main/java/ch/ethz/seb/sebserver/webservice/servicelayer/lms/impl/edx/OpenEdxLmsAPITemplate.java
View file

@ -74,7 +74,7 @@ final class OpenEdxLmsAPITemplate implements LmsAPITemplate {
@Override
public Result<List<QuizData>> getQuizzes(final FilterMap filterMap) {
return this.openEdxCourseAccess
.getQuizzes(filterMap)
.protectedQuizzesRequest(filterMap)
.map(quizzes -> quizzes.stream()
.filter(LmsAPIService.quizFilterPredicate(filterMap))
.collect(Collectors.toList()));
@ -82,18 +82,16 @@ final class OpenEdxLmsAPITemplate implements LmsAPITemplate {
@Override
public Result<QuizData> getQuiz(final String id) {
return Result.tryCatch(() -> {
final QuizData quizFromCache = this.openEdxCourseAccess.getQuizFromCache(id);
if (quizFromCache != null) {
return quizFromCache;
}
final QuizData quizFromCache = this.openEdxCourseAccess.getQuizFromCache(id);
if (quizFromCache != null) {
return Result.of(quizFromCache);
}
return this.openEdxCourseAccess.getQuizFromLMS(id);
});
return this.openEdxCourseAccess.protectedQuizRequest(id);
}
@Override
public Collection<Result<QuizData>> getQuizzes(final Set<String> ids) {
public Result<Collection<QuizData>> getQuizzes(final Set<String> ids) {
return this.openEdxCourseAccess.getQuizzesFromCache(ids);
}

21
src/main/java/ch/ethz/seb/sebserver/webservice/servicelayer/lms/impl/mockup/MockupLmsAPITemplate.java
View file

@ -172,17 +172,18 @@ public class MockupLmsAPITemplate implements LmsAPITemplate {
}
@Override
public Collection<Result<QuizData>> getQuizzes(final Set<String> ids) {
if (!authenticate()) {
throw new IllegalArgumentException("Wrong clientId or secret");
}
public Result<Collection<QuizData>> getQuizzes(final Set<String> ids) {
return Result.tryCatch(() -> {
if (!authenticate()) {
throw new IllegalArgumentException("Wrong clientId or secret");
}
return this.mockups
.stream()
.map(this::getExternalAddressAlias)
.filter(mock -> ids.contains(mock.id))
.map(Result::of)
.collect(Collectors.toList());
return this.mockups
.stream()
.map(this::getExternalAddressAlias)
.filter(mock -> ids.contains(mock.id))
.collect(Collectors.toList());
});
}
@Override

77
src/main/java/ch/ethz/seb/sebserver/webservice/servicelayer/lms/impl/moodle/MoodleCourseAccess.java
View file

@ -14,7 +14,6 @@ import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.NoSuchElementException;
import java.util.Set;
import java.util.function.Function;
import java.util.function.Supplier;
@ -245,52 +244,54 @@ public class MoodleCourseAccess extends AbstractCourseAccess {
}
}
// get from LMS
final Set<String> ids = Stream.of(id).collect(Collectors.toSet());
return super.quizzesRequest
.protectedRun(quizzesSupplier(ids))
.getOrThrow()
.get(0);
// get from LMS in protected request
return super.protectedQuizRequest(id).getOrThrow();
});
}
public Collection<Result<QuizData>> getQuizzesFromCache(final Set<String> ids) {
final List<QuizData> cached = getCached();
final List<QuizData> available = (cached != null)
? cached
: Collections.emptyList();
public Result<Collection<QuizData>> getQuizzesFromCache(final Set<String> ids) {
return Result.tryCatch(() -> {
final List<QuizData> cached = getCached();
final List<QuizData> available = (cached != null)
? cached
: Collections.emptyList();
final Map<String, QuizData> quizMapping = available
.stream()
.collect(Collectors.toMap(q -> q.id, Function.identity()));
if (!quizMapping.keySet().containsAll(ids)) {
final Map<String, QuizData> collect = super.quizzesRequest
.protectedRun(quizzesSupplier(ids))
.onError(error -> log.error("Failed to get quizzes by ids: ", error))
.getOrElse(() -> Collections.emptyList())
final Map<String, QuizData> quizMapping = available
.stream()
.collect(Collectors.toMap(qd -> qd.id, Function.identity()));
if (collect != null) {
quizMapping.clear();
quizMapping.putAll(collect);
}
}
.collect(Collectors.toMap(q -> q.id, Function.identity()));
return ids
.stream()
.map(id -> {
final QuizData q = quizMapping.get(id);
return (q == null)
? Result.<QuizData> ofError(new NoSuchElementException("Quiz with id: " + id))
: Result.of(q);
})
.collect(Collectors.toList());
if (!quizMapping.keySet().containsAll(ids)) {
final Map<String, QuizData> collect = super.quizzesRequest
.protectedRun(quizzesSupplier(ids))
.onError(error -> log.error("Failed to get quizzes by ids: ", error))
.getOrElse(() -> Collections.emptyList())
.stream()
.collect(Collectors.toMap(qd -> qd.id, Function.identity()));
if (collect != null) {
quizMapping.clear();
quizMapping.putAll(collect);
}
}
return quizMapping.values();
});
}
@Override
protected Supplier<List<QuizData>> quizzesSupplier(final Set<String> ids) {
protected Supplier<QuizData> quizSupplier(final String id) {
return () -> {
final Set<String> ids = Stream.of(id).collect(Collectors.toSet());
return getRestTemplate()
.map(template -> getQuizzesForIds(template, ids))
.getOr(Collections.emptyList())
.get(0);
};
}
@Override
protected Supplier<Collection<QuizData>> quizzesSupplier(final Set<String> ids) {
return () -> getRestTemplate()
.map(template -> getQuizzesForIds(template, ids))
.getOr(Collections.emptyList());

4
src/main/java/ch/ethz/seb/sebserver/webservice/servicelayer/lms/impl/moodle/MoodleLmsAPITemplate.java
View file

@ -86,7 +86,7 @@ public class MoodleLmsAPITemplate implements LmsAPITemplate {
@Override
public Result<List<QuizData>> getQuizzes(final FilterMap filterMap) {
return this.moodleCourseAccess
.getQuizzes(filterMap)
.protectedQuizzesRequest(filterMap)
.map(quizzes -> quizzes.stream()
.filter(LmsAPIService.quizFilterPredicate(filterMap))
.collect(Collectors.toList()));
@ -98,7 +98,7 @@ public class MoodleLmsAPITemplate implements LmsAPITemplate {
}
@Override
public Collection<Result<QuizData>> getQuizzes(final Set<String> ids) {
public Result<Collection<QuizData>> getQuizzes(final Set<String> ids) {
return this.moodleCourseAccess.getQuizzesFromCache(ids);
}