zervo/seb-server
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

refactoring of LMS API service with overall eh-caching

Browse source
This commit is contained in:
anhefti 2021-05-15 17:43:26 +02:00
parent dd6150ec2a
commit 9214719642
19 changed files with 406 additions and 210 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
src/main/java/ch/ethz/seb/sebserver/gbl/Constants.java
View file

@ -59,6 +59,7 @@ public final class Constants {
public static final Character DOUBLE_QUOTE = '"';
public static final Character COMMA = ',';
public static final Character PIPE = '|';
public static final Character UNDERLINE = '_';
public static final Character AMPERSAND = '&';
public static final Character EQUALITY_SIGN = '=';
public static final Character LIST_SEPARATOR_CHAR = COMMA;

2
src/main/java/ch/ethz/seb/sebserver/webservice/servicelayer/dao/ExamDAO.java
View file

@ -37,6 +37,8 @@ public interface ExamDAO extends ActivatableEntityDAO<Exam, Exam>, BulkActionSup
* happened */
Result<GrantEntity> examGrantEntityByClientConnection(Long connectionId);
Result<Exam> getWithQuizDataFromCache(Long id);
/** Get all active Exams for a given institution.
*
* @param institutionId the identifier of the institution

4
src/main/java/ch/ethz/seb/sebserver/webservice/servicelayer/dao/impl/ExamConfigurationMapDAOImpl.java
View file

@ -377,6 +377,7 @@ public class ExamConfigurationMapDAOImpl implements ExamConfigurationMapDAO {
private Result<Collection<ExamConfigurationMap>> toDomainModel(
final Collection<ExamConfigurationMapRecord> records) {
return Result.tryCatch(() -> records
.stream()
.map(model -> this.toDomainModel(model).getOrThrow())
@ -390,7 +391,8 @@ public class ExamConfigurationMapDAOImpl implements ExamConfigurationMapDAO {
.selectByPrimaryKey(record.getConfigurationNodeId());
final String status = config.getStatus();
final Exam exam = this.examDAO.byPK(record.getExamId())
final Exam exam = this.examDAO
.getWithQuizDataFromCache(record.getExamId())
.getOr(null);
return new ExamConfigurationMap(

18
src/main/java/ch/ethz/seb/sebserver/webservice/servicelayer/dao/impl/ExamDAOImpl.java
View file

@ -119,6 +119,13 @@ public class ExamDAOImpl implements ExamDAO {
.map(record -> toDomainModel(record, null, null).getOrThrow());
}
@Override
@Transactional(readOnly = true)
public Result<Exam> getWithQuizDataFromCache(final Long id) {
return recordById(id)
.flatMap(this::toDomainModelFromCache);
}
@Override
@Transactional(readOnly = true)
public Result<Collection<Exam>> all(final Long institutionId, final Boolean active) {
@ -761,6 +768,17 @@ public class ExamDAOImpl implements ExamDAO {
exam.getDescription());
}
private Result<Exam> toDomainModelFromCache(final ExamRecord record) {
return this.lmsAPIService
.getLmsAPITemplate(record.getLmsSetupId())
.flatMap(template -> this.toDomainModel(
record,
template.getQuizFromCache(record.getExternalId())
.getOrThrow(),
null));
}
private Result<Exam> toDomainModel(final ExamRecord record) {
return toDomainModel(
record.getLmsSetupId(),

9
src/main/java/ch/ethz/seb/sebserver/webservice/servicelayer/lms/LmsAPITemplate.java
View file

@ -165,6 +165,15 @@ public interface LmsAPITemplate {
* @return Collection of all {@link QuizData } from the given id set */
Collection<Result<QuizData>> getQuizzesFromCache(Set<String> ids);
/** Get a particular quiz data from cache if available. If not, tries to get it from the LMS.
*
* @param id the quiz identifier, external identifier of the exam.
* @return Result refer to the {@link QuizData } or to an error when happended */
Result<QuizData> getQuizFromCache(String id);
/** Clears the underling caches if there are some for a particular implementation. */
void clearCache();
/** Convert an anonymous or temporary examineeUserId, sent by the SEB Client on LMS login,
* to LMS examinee account details by requesting them on the LMS API with the given examineeUserId
*

82
src/main/java/ch/ethz/seb/sebserver/webservice/servicelayer/lms/impl/AbstractCachedCourseAccess.java Normal file
View file

@ -0,0 +1,82 @@
/*
* Copyright (c) 2021 ETH Zürich, Educational Development and Technology (LET)
*
* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/.
*/
package ch.ethz.seb.sebserver.webservice.servicelayer.lms.impl;
import java.util.Collection;
import java.util.Set;
import java.util.stream.Collectors;
import org.springframework.cache.Cache;
import org.springframework.cache.CacheManager;
import org.springframework.core.env.Environment;
import ch.ethz.seb.sebserver.gbl.Constants;
import ch.ethz.seb.sebserver.gbl.async.AsyncService;
import ch.ethz.seb.sebserver.gbl.model.exam.QuizData;
import ch.ethz.seb.sebserver.gbl.util.Result;
public abstract class AbstractCachedCourseAccess extends AbstractCourseAccess {
public static final String CACHE_NAME_QUIZ_DATA = "QUIZ_DATA_CACHE";
private static final String NO_QUIZ_DATA_FOUND_ERROR = "NO_QUIZ_DATA_FOUND_ERROR";
private final Cache cache;
protected AbstractCachedCourseAccess(
final AsyncService asyncService,
final Environment environment,
final CacheManager cacheManager) {
super(asyncService, environment);
this.cache = cacheManager.getCache(CACHE_NAME_QUIZ_DATA);
}
public void clearCache() {
this.cache.clear();
}
protected QuizData getFromCache(final String id) {
return this.cache.get(createCacheKey(id), QuizData.class);
}
protected void putToCache(final QuizData quizData) {
this.cache.put(createCacheKey(quizData.id), quizData);
}
protected void putToCache(final Collection<QuizData> quizData) {
quizData.stream().forEach(q -> this.cache.put(createCacheKey(q.id), q));
}
protected void evict(final String id) {
this.cache.evict(createCacheKey(id));
}
@Override
public Result<QuizData> getQuizFromCache(final String id) {
final QuizData fromCache = getFromCache(id);
if (fromCache != null) {
return Result.of(fromCache);
} else {
return Result.ofRuntimeError(NO_QUIZ_DATA_FOUND_ERROR);
}
}
@Override
public Result<Collection<Result<QuizData>>> getQuizzesFromCache(final Set<String> ids) {
return Result.of(ids.stream().map(this::getQuizFromCache).collect(Collectors.toList()));
}
protected abstract Long getLmsSetupId();
private final String createCacheKey(final String id) {
return id + Constants.UNDERLINE + getLmsSetupId();
}
}

88
src/main/java/ch/ethz/seb/sebserver/webservice/servicelayer/lms/impl/AbstractCourseAccess.java
View file

@ -11,12 +11,8 @@ package ch.ethz.seb.sebserver.webservice.servicelayer.lms.impl;
import java.util.Collection;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.NoSuchElementException;
import java.util.Set;
import java.util.function.Function;
import java.util.function.Supplier;
import java.util.stream.Collectors;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -25,6 +21,7 @@ import org.springframework.core.env.Environment;
import ch.ethz.seb.sebserver.gbl.Constants;
import ch.ethz.seb.sebserver.gbl.async.AsyncService;
import ch.ethz.seb.sebserver.gbl.async.CircuitBreaker;
import ch.ethz.seb.sebserver.gbl.async.CircuitBreaker.State;
import ch.ethz.seb.sebserver.gbl.model.exam.Chapters;
import ch.ethz.seb.sebserver.gbl.model.exam.QuizData;
import ch.ethz.seb.sebserver.gbl.model.user.ExamineeAccountDetails;
@ -101,42 +98,8 @@ public abstract class AbstractCourseAccess {
Constants.SECOND_IN_MILLIS * 10));
}
public Result<Collection<Result<QuizData>>> getQuizzesFromCache(final Set<String> ids) {
return Result.tryCatch(() -> {
final List<QuizData> cached = allQuizzesSupplier().getAllCached();
final List<QuizData> available = (cached != null)
? cached
: Collections.emptyList();
final Map<String, QuizData> quizMapping = available
.stream()
.collect(Collectors.toMap(q -> q.id, Function.identity()));
if (!quizMapping.keySet().containsAll(ids)) {
final Map<String, QuizData> collect = quizzesSupplier(ids).get()
.stream()
.collect(Collectors.toMap(qd -> qd.id, Function.identity()));
if (collect != null) {
quizMapping.clear();
quizMapping.putAll(collect);
}
}
return ids
.stream()
.map(id -> {
final QuizData q = quizMapping.get(id);
return (q == null)
? Result.<QuizData> ofError(new NoSuchElementException("Quiz with id: " + id))
: Result.of(q);
})
.collect(Collectors.toList());
});
}
public Result<List<QuizData>> getQuizzes(final FilterMap filterMap) {
return allQuizzesSupplier().getAll(filterMap);
return this.quizzesRequest.protectedRun(allQuizzesSupplier(filterMap));
}
public Result<ExamineeAccountDetails> getExamineeAccountDetails(final String examineeSessionId) {
@ -172,30 +135,41 @@ public abstract class AbstractCourseAccess {
Collections.emptyMap());
}
/** This abstraction has no cache implementation and therefore this returns a Result
* with an "No cache supported error.
* </p>
* To implement and use caching, this must be overridden and implemented
*
* @param id The identifier of the QuizData to get from cache
* @return Result with an "No cache supported error */
public Result<QuizData> getQuizFromCache(final String id) {
return Result.ofRuntimeError("No cache supported");
}
/** This abstraction has no cache implementation and therefore this returns a Result
* with an "No cache supported error.
* </p>
* To implement and use caching, this must be overridden and implemented
*
* @param ids Collection of quiz data identifier to get from the cache
* @return Result with an "No cache supported error */
public Result<Collection<Result<QuizData>>> getQuizzesFromCache(final Set<String> ids) {
return Result.ofRuntimeError("No cache supported");
}
/** Provides a supplier for the quiz data request to use within the circuit breaker */
protected abstract Supplier<List<QuizData>> quizzesSupplier(final Set<String> ids);
/** Provides a AllQuizzesSupplier to supply quiz data either form cache or from LMS */
protected abstract AllQuizzesSupplier allQuizzesSupplier();
/** Provides a supplier to supply request to use within the circuit breaker */
protected abstract Supplier<List<QuizData>> allQuizzesSupplier(final FilterMap filterMap);
/** Provides a supplier for the course chapter data request to use within the circuit breaker */
protected abstract Supplier<Chapters> getCourseChaptersSupplier(final String courseId);
/** Gives a fetch status if asynchronous quiz data fetching is part of the underling implementation */
protected abstract FetchStatus getFetchStatus();
/** Uses to supply quiz data */
protected interface AllQuizzesSupplier {
/** Get all currently cached quiz data if supported by the underling implementation
*
* @return List containing all cached quiz data objects */
List<QuizData> getAllCached();
/** Get a list of all quiz data filtered by the given filter map from LMS.
*
* @param filterMap Map containing the filter criteria
* @return Result refer to the list of filtered quiz data or to an error when happened */
Result<List<QuizData>> getAll(final FilterMap filterMap);
protected FetchStatus getFetchStatus() {
if (this.quizzesRequest.getState() != State.CLOSED) {
return FetchStatus.FETCH_ERROR;
}
return FetchStatus.ALL_FETCHED;
}
}

21
src/main/java/ch/ethz/seb/sebserver/webservice/servicelayer/lms/impl/LmsAPIServiceImpl.java
View file

@ -98,17 +98,20 @@ public class LmsAPIServiceImpl implements LmsAPIService {
@Override
public Result<LmsAPITemplate> getLmsAPITemplate(final String lmsSetupId) {
return Result.tryCatch(() -> {
LmsAPITemplate lmsAPITemplate = getFromCache(lmsSetupId);
if (lmsAPITemplate == null) {
lmsAPITemplate = createLmsSetupTemplate(lmsSetupId);
if (lmsAPITemplate != null) {
this.cache.put(new CacheKey(lmsSetupId, System.currentTimeMillis()), lmsAPITemplate);
synchronized (this) {
LmsAPITemplate lmsAPITemplate = getFromCache(lmsSetupId);
if (lmsAPITemplate == null) {
lmsAPITemplate = createLmsSetupTemplate(lmsSetupId);
if (lmsAPITemplate != null) {
this.cache.put(new CacheKey(lmsSetupId, System.currentTimeMillis()), lmsAPITemplate);
}
}
if (lmsAPITemplate == null) {
throw new ResourceNotFoundException(EntityType.LMS_SETUP, lmsSetupId);
}
return lmsAPITemplate;
}
if (lmsAPITemplate == null) {
throw new ResourceNotFoundException(EntityType.LMS_SETUP, lmsSetupId);
}
return lmsAPITemplate;
});
}

2
src/main/java/ch/ethz/seb/sebserver/webservice/servicelayer/lms/impl/SEBRestrictionServiceImpl.java
View file

@ -91,12 +91,10 @@ public class SEBRestrictionServiceImpl implements SEBRestrictionService {
public Result<SEBRestriction> getSEBRestrictionFromExam(final Exam exam) {
return Result.tryCatch(() -> {
// load the config keys from restriction and merge with new generated config keys
final long currentTimeMillis = System.currentTimeMillis();
final Set<String> configKeys = new HashSet<>();
final Collection<String> generatedKeys = this.examConfigService
.generateConfigKeys(exam.institutionId, exam.id)
.getOrThrow();
System.out.println("******* " + (System.currentTimeMillis() - currentTimeMillis));
configKeys.addAll(generatedKeys);
if (generatedKeys != null && !generatedKeys.isEmpty()) {

132
src/main/java/ch/ethz/seb/sebserver/webservice/servicelayer/lms/impl/edx/OpenEdxCourseAccess.java
View file

@ -21,6 +21,7 @@ import java.util.stream.Collectors;
import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.cache.CacheManager;
import org.springframework.core.env.Environment;
import org.springframework.http.HttpEntity;
import org.springframework.http.HttpHeaders;
@ -43,8 +44,6 @@ import com.fasterxml.jackson.core.type.TypeReference;
import ch.ethz.seb.sebserver.gbl.Constants;
import ch.ethz.seb.sebserver.gbl.api.JSONMapper;
import ch.ethz.seb.sebserver.gbl.async.AsyncService;
import ch.ethz.seb.sebserver.gbl.async.CircuitBreaker.State;
import ch.ethz.seb.sebserver.gbl.async.MemoizingCircuitBreaker;
import ch.ethz.seb.sebserver.gbl.model.exam.Chapters;
import ch.ethz.seb.sebserver.gbl.model.exam.QuizData;
import ch.ethz.seb.sebserver.gbl.model.institution.LmsSetup;
@ -55,12 +54,12 @@ import ch.ethz.seb.sebserver.gbl.util.Utils;
import ch.ethz.seb.sebserver.webservice.WebserviceInfo;
import ch.ethz.seb.sebserver.webservice.servicelayer.dao.FilterMap;
import ch.ethz.seb.sebserver.webservice.servicelayer.lms.APITemplateDataSupplier;
import ch.ethz.seb.sebserver.webservice.servicelayer.lms.impl.AbstractCourseAccess;
import ch.ethz.seb.sebserver.webservice.servicelayer.lms.impl.AbstractCachedCourseAccess;
/** Implements the LmsAPITemplate for Open edX LMS Course API access.
*
* See also: https://course-catalog-api-guide.readthedocs.io */
final class OpenEdxCourseAccess extends AbstractCourseAccess {
final class OpenEdxCourseAccess extends AbstractCachedCourseAccess {
private static final Logger log = LoggerFactory.getLogger(OpenEdxCourseAccess.class);
@ -74,65 +73,34 @@ final class OpenEdxCourseAccess extends AbstractCourseAccess {
private final JSONMapper jsonMapper;
private final OpenEdxRestTemplateFactory openEdxRestTemplateFactory;
private final WebserviceInfo webserviceInfo;
private final MemoizingCircuitBreaker<List<QuizData>> allQuizzesRequest;
private final AllQuizzesSupplier allQuizzesSupplier;
private OAuth2RestTemplate restTemplate;
private final Long lmsSetupId;
public OpenEdxCourseAccess(
final JSONMapper jsonMapper,
final OpenEdxRestTemplateFactory openEdxRestTemplateFactory,
final WebserviceInfo webserviceInfo,
final AsyncService asyncService,
final Environment environment) {
final Environment environment,
final CacheManager cacheManager) {
super(asyncService, environment);
super(asyncService, environment, cacheManager);
this.jsonMapper = jsonMapper;
this.openEdxRestTemplateFactory = openEdxRestTemplateFactory;
this.webserviceInfo = webserviceInfo;
this.allQuizzesRequest = asyncService.createMemoizingCircuitBreaker(
quizzesSupplier(),
environment.getProperty(
"sebserver.webservice.circuitbreaker.allQuizzesRequest.attempts",
Integer.class,
3),
environment.getProperty(
"sebserver.webservice.circuitbreaker.allQuizzesRequest.blockingTime",
Long.class,
Constants.MINUTE_IN_MILLIS),
environment.getProperty(
"sebserver.webservice.circuitbreaker.allQuizzesRequest.timeToRecover",
Long.class,
Constants.MINUTE_IN_MILLIS),
environment.getProperty(
"sebserver.webservice.circuitbreaker.allQuizzesRequest.memoize",
Boolean.class,
true),
environment.getProperty(
"sebserver.webservice.circuitbreaker.allQuizzesRequest.memoizingTime",
Long.class,
Constants.HOUR_IN_MILLIS));
this.allQuizzesSupplier = new AllQuizzesSupplier() {
@Override
public List<QuizData> getAllCached() {
return OpenEdxCourseAccess.this.allQuizzesRequest.getCached();
}
@Override
public Result<List<QuizData>> getAll(final FilterMap filterMap) {
return OpenEdxCourseAccess.this.allQuizzesRequest.get();
}
};
this.lmsSetupId = openEdxRestTemplateFactory.apiTemplateDataSupplier.getLmsSetup().id;
}
APITemplateDataSupplier getApiTemplateDataSupplier() {
return this.openEdxRestTemplateFactory.apiTemplateDataSupplier;
}
@Override
protected Long getLmsSetupId() {
return this.lmsSetupId;
}
LmsSetupTestResult initAPIAccess() {
final LmsSetup lmsSetup = getApiTemplateDataSupplier().getLmsSetup();
@ -219,6 +187,13 @@ final class OpenEdxCourseAccess extends AbstractCourseAccess {
});
}
@Override
protected Supplier<List<QuizData>> allQuizzesSupplier(final FilterMap filterMap) {
return () -> getRestTemplate()
.map(this::collectAllQuizzes)
.getOrThrow();
}
@Override
protected Supplier<List<QuizData>> quizzesSupplier(final Set<String> ids) {
@ -231,7 +206,7 @@ final class OpenEdxCourseAccess extends AbstractCourseAccess {
getOneCourses(
lmsSetup.lmsApiUrl + OPEN_EDX_DEFAULT_COURSE_ENDPOINT,
getRestTemplate().getOrThrow(),
ids.iterator().next()),
ids),
externalStartURI));
};
} else {
@ -241,12 +216,6 @@ final class OpenEdxCourseAccess extends AbstractCourseAccess {
}
}
private Supplier<List<QuizData>> quizzesSupplier() {
return () -> getRestTemplate()
.map(this::collectAllQuizzes)
.getOrThrow();
}
@Override
protected Supplier<Chapters> getCourseChaptersSupplier(final String courseId) {
return () -> {
@ -337,7 +306,10 @@ final class OpenEdxCourseAccess extends AbstractCourseAccess {
final List<CourseData> collector = new ArrayList<>();
EdXPage page = getEdxPage(pageURI, restTemplate).getBody();
if (page != null) {
collector.addAll(page.results);
page.results
.stream()
.filter(cd -> ids.contains(cd.id))
.forEach(collector::add);
while (page != null && StringUtils.isNotBlank(page.next)) {
page = getEdxPage(page.next, restTemplate).getBody();
if (page != null) {
@ -355,16 +327,33 @@ final class OpenEdxCourseAccess extends AbstractCourseAccess {
private CourseData getOneCourses(
final String pageURI,
final OAuth2RestTemplate restTemplate,
final String id) {
final Set<String> ids) {
final HttpHeaders httpHeaders = new HttpHeaders();
final ResponseEntity<CourseData> exchange = restTemplate.exchange(
pageURI + "/" + id,
HttpMethod.GET,
new HttpEntity<>(httpHeaders),
CourseData.class);
System.out.println("********************");
return exchange.getBody();
// NOTE: try first to get the course data by id. This seems to be possible
// when the SEB restriction is not set. Once the SEB restriction is set,
// this gives a 403 response.
// We haven't found another way to get course data by id in this case so far
// Workaround is to search the course by paging (slow)
try {
final HttpHeaders httpHeaders = new HttpHeaders();
final String uri = pageURI + ids.iterator().next();
final ResponseEntity<CourseData> exchange = restTemplate.exchange(
uri,
HttpMethod.GET,
new HttpEntity<>(httpHeaders),
CourseData.class);
return exchange.getBody();
} catch (final Exception e) {
// try with paging
final List<CourseData> collectCourses = collectCourses(pageURI, restTemplate, ids);
if (collectCourses.isEmpty()) {
return null;
}
return collectCourses.get(0);
}
}
private List<CourseData> collectAllCourses(final String pageURI, final OAuth2RestTemplate restTemplate) {
@ -403,7 +392,7 @@ final class OpenEdxCourseAccess extends AbstractCourseAccess {
Blocks.class);
}
private static QuizData quizDataOf(
private QuizData quizDataOf(
final LmsSetup lmsSetup,
final CourseData courseData,
final String uriPrefix) {
@ -411,7 +400,7 @@ final class OpenEdxCourseAccess extends AbstractCourseAccess {
final String startURI = uriPrefix + courseData.id;
final Map<String, String> additionalAttrs = new HashMap<>();
additionalAttrs.put("blocks_url", courseData.blocks_url);
return new QuizData(
final QuizData quizData = new QuizData(
courseData.id,
lmsSetup.getInstitutionId(),
lmsSetup.id,
@ -421,6 +410,10 @@ final class OpenEdxCourseAccess extends AbstractCourseAccess {
courseData.start,
courseData.end,
startURI);
super.putToCache(quizData);
return quizData;
}
/** Maps a OpenEdX course API course page */
@ -538,17 +531,4 @@ final class OpenEdxCourseAccess extends AbstractCourseAccess {
return Result.of(this.restTemplate);
}
@Override
protected FetchStatus getFetchStatus() {
if (this.allQuizzesRequest.getState() != State.CLOSED) {
return FetchStatus.FETCH_ERROR;
}
return FetchStatus.ALL_FETCHED;
}
@Override
protected AllQuizzesSupplier allQuizzesSupplier() {
return this.allQuizzesSupplier;
}
}

12
src/main/java/ch/ethz/seb/sebserver/webservice/servicelayer/lms/impl/edx/OpenEdxLmsAPITemplate.java
View file

@ -98,12 +98,24 @@ final class OpenEdxLmsAPITemplate implements LmsAPITemplate {
.collect(Collectors.toList());
}
@Override
public Result<QuizData> getQuizFromCache(final String id) {
return this.openEdxCourseAccess
.getQuizFromCache(id)
.orElse(() -> getQuiz(id));
}
@Override
public Collection<Result<QuizData>> getQuizzesFromCache(final Set<String> ids) {
return this.openEdxCourseAccess.getQuizzesFromCache(ids)
.getOrElse(() -> getQuizzes(ids));
}
@Override
public void clearCache() {
this.openEdxCourseAccess.clearCache();
}
@Override
public Result<Chapters> getCourseChapters(final String courseId) {
return Result.tryCatch(() -> this.openEdxCourseAccess

7
src/main/java/ch/ethz/seb/sebserver/webservice/servicelayer/lms/impl/edx/OpenEdxLmsAPITemplateFactory.java
View file

@ -10,6 +10,7 @@ package ch.ethz.seb.sebserver.webservice.servicelayer.lms.impl.edx;
import org.apache.commons.lang3.StringUtils;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.cache.CacheManager;
import org.springframework.context.annotation.Lazy;
import org.springframework.core.env.Environment;
import org.springframework.stereotype.Service;
@ -36,6 +37,7 @@ public class OpenEdxLmsAPITemplateFactory implements LmsAPITemplateFactory {
private final WebserviceInfo webserviceInfo;
private final AsyncService asyncService;
private final Environment environment;
private final CacheManager cacheManager;
private final ClientCredentialService clientCredentialService;
private final ClientHttpRequestFactoryService clientHttpRequestFactoryService;
private final String[] alternativeTokenRequestPaths;
@ -46,6 +48,7 @@ public class OpenEdxLmsAPITemplateFactory implements LmsAPITemplateFactory {
final WebserviceInfo webserviceInfo,
final AsyncService asyncService,
final Environment environment,
final CacheManager cacheManager,
final ClientCredentialService clientCredentialService,
final ClientHttpRequestFactoryService clientHttpRequestFactoryService,
@Value("${sebserver.webservice.lms.openedx.api.token.request.paths}") final String alternativeTokenRequestPaths,
@ -55,6 +58,7 @@ public class OpenEdxLmsAPITemplateFactory implements LmsAPITemplateFactory {
this.webserviceInfo = webserviceInfo;
this.asyncService = asyncService;
this.environment = environment;
this.cacheManager = cacheManager;
this.clientCredentialService = clientCredentialService;
this.clientHttpRequestFactoryService = clientHttpRequestFactoryService;
this.alternativeTokenRequestPaths = (alternativeTokenRequestPaths != null)
@ -84,7 +88,8 @@ public class OpenEdxLmsAPITemplateFactory implements LmsAPITemplateFactory {
openEdxRestTemplateFactory,
this.webserviceInfo,
this.asyncService,
this.environment);
this.environment,
this.cacheManager);
final OpenEdxCourseRestriction openEdxCourseRestriction = new OpenEdxCourseRestriction(
this.jsonMapper,

12
src/main/java/ch/ethz/seb/sebserver/webservice/servicelayer/lms/impl/mockup/MockupLmsAPITemplate.java
View file

@ -9,7 +9,9 @@
package ch.ethz.seb.sebserver.webservice.servicelayer.lms.impl.mockup;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import java.util.stream.Collectors;
@ -181,6 +183,16 @@ public class MockupLmsAPITemplate implements LmsAPITemplate {
return getQuizzes(ids);
}
@Override
public Result<QuizData> getQuizFromCache(final String id) {
return getQuizzes(new HashSet<>(Arrays.asList(id))).iterator().next();
}
@Override
public void clearCache() {
}
@Override
public Result<Chapters> getCourseChapters(final String courseId) {
return Result.ofError(new UnsupportedOperationException());

162
src/main/java/ch/ethz/seb/sebserver/webservice/servicelayer/lms/impl/moodle/MoodleCourseAccess.java
View file

@ -14,6 +14,7 @@ import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.NoSuchElementException;
import java.util.Set;
import java.util.function.Function;
import java.util.function.Supplier;
@ -37,6 +38,7 @@ import ch.ethz.seb.sebserver.gbl.Constants;
import ch.ethz.seb.sebserver.gbl.api.JSONMapper;
import ch.ethz.seb.sebserver.gbl.async.AsyncService;
import ch.ethz.seb.sebserver.gbl.async.CircuitBreaker;
import ch.ethz.seb.sebserver.gbl.async.CircuitBreaker.State;
import ch.ethz.seb.sebserver.gbl.model.exam.Chapters;
import ch.ethz.seb.sebserver.gbl.model.exam.QuizData;
import ch.ethz.seb.sebserver.gbl.model.institution.LmsSetup;
@ -48,6 +50,7 @@ import ch.ethz.seb.sebserver.webservice.servicelayer.dao.FilterMap;
import ch.ethz.seb.sebserver.webservice.servicelayer.lms.APITemplateDataSupplier;
import ch.ethz.seb.sebserver.webservice.servicelayer.lms.impl.AbstractCourseAccess;
import ch.ethz.seb.sebserver.webservice.servicelayer.lms.impl.moodle.MoodleCourseDataAsyncLoader.CourseDataShort;
import ch.ethz.seb.sebserver.webservice.servicelayer.lms.impl.moodle.MoodleCourseDataAsyncLoader.CourseQuizShort;
import ch.ethz.seb.sebserver.webservice.servicelayer.lms.impl.moodle.MoodleRestTemplateFactory.MoodleAPIRestTemplate;
/** Implements the LmsAPITemplate for Open edX LMS Course API access.
@ -90,7 +93,6 @@ public class MoodleCourseAccess extends AbstractCourseAccess {
private final MoodleRestTemplateFactory moodleRestTemplateFactory;
private final MoodleCourseDataAsyncLoader moodleCourseDataAsyncLoader;
private final CircuitBreaker<List<QuizData>> allQuizzesRequest;
private final AllQuizzesSupplier allQuizzesSupplier;
private MoodleAPIRestTemplate restTemplate;
@ -119,19 +121,6 @@ public class MoodleCourseAccess extends AbstractCourseAccess {
"sebserver.webservice.circuitbreaker.allQuizzesRequest.timeToRecover",
Long.class,
Constants.MINUTE_IN_MILLIS));
this.allQuizzesSupplier = new AllQuizzesSupplier() {
@Override
public List<QuizData> getAllCached() {
return getCached();
}
@Override
public Result<List<QuizData>> getAll(final FilterMap filterMap) {
return MoodleCourseAccess.this.allQuizzesRequest.protectedRun(allQuizzesSupplier(filterMap));
}
};
}
APITemplateDataSupplier getApiTemplateDataSupplier() {
@ -223,6 +212,77 @@ public class MoodleCourseAccess extends AbstractCourseAccess {
return LmsSetupTestResult.ofOkay();
}
@Override
public Result<QuizData> getQuizFromCache(final String id) {
return Result.tryCatch(() -> {
final Map<String, CourseDataShort> cachedCourseData = this.moodleCourseDataAsyncLoader
.getCachedCourseData();
final String courseId = getCourseId(id);
final String quizId = getQuizId(id);
if (cachedCourseData.containsKey(courseId)) {
final CourseDataShort courseData = cachedCourseData.get(courseId);
final CourseQuizShort quiz = courseData.quizzes
.stream()
.filter(q -> q.id.equals(quizId))
.findFirst()
.orElse(null);
if (quiz != null) {
final Map<String, String> additionalAttrs = new HashMap<>();
additionalAttrs.put(QuizData.ATTR_ADDITIONAL_CREATION_TIME,
String.valueOf(courseData.time_created));
additionalAttrs.put(QuizData.ATTR_ADDITIONAL_SHORT_NAME, courseData.short_name);
additionalAttrs.put(QuizData.ATTR_ADDITIONAL_ID_NUMBER, courseData.idnumber);
final LmsSetup lmsSetup = getApiTemplateDataSupplier().getLmsSetup();
final String urlPrefix = (lmsSetup.lmsApiUrl.endsWith(Constants.URL_PATH_SEPARATOR))
? lmsSetup.lmsApiUrl + MOODLE_QUIZ_START_URL_PATH
: lmsSetup.lmsApiUrl + Constants.URL_PATH_SEPARATOR + MOODLE_QUIZ_START_URL_PATH;
return createQuizData(lmsSetup, courseData, urlPrefix, additionalAttrs, quiz);
}
}
throw new RuntimeException("No quiz found in cache");
});
}
@Override
public Result<Collection<Result<QuizData>>> getQuizzesFromCache(final Set<String> ids) {
return Result.tryCatch(() -> {
final List<QuizData> cached = getCached();
final List<QuizData> available = (cached != null)
? cached
: Collections.emptyList();
final Map<String, QuizData> quizMapping = available
.stream()
.collect(Collectors.toMap(q -> q.id, Function.identity()));
if (!quizMapping.keySet().containsAll(ids)) {
final Map<String, QuizData> collect = quizzesSupplier(ids).get()
.stream()
.collect(Collectors.toMap(qd -> qd.id, Function.identity()));
if (collect != null) {
quizMapping.clear();
quizMapping.putAll(collect);
}
}
return ids
.stream()
.map(id -> {
final QuizData q = quizMapping.get(id);
return (q == null)
? Result.<QuizData> ofError(new NoSuchElementException("Quiz with id: " + id))
: Result.of(q);
})
.collect(Collectors.toList());
});
}
@Override
protected Supplier<List<QuizData>> quizzesSupplier(final Set<String> ids) {
return () -> getRestTemplate()
@ -231,6 +291,7 @@ public class MoodleCourseAccess extends AbstractCourseAccess {
}
@Override
protected Supplier<List<QuizData>> allQuizzesSupplier(final FilterMap filterMap) {
return () -> getRestTemplate()
.map(template -> collectAllQuizzes(template, filterMap))
@ -244,6 +305,9 @@ public class MoodleCourseAccess extends AbstractCourseAccess {
@Override
protected FetchStatus getFetchStatus() {
if (this.allQuizzesRequest.getState() != State.CLOSED) {
return FetchStatus.FETCH_ERROR;
}
if (this.moodleCourseDataAsyncLoader.isRunning()) {
return FetchStatus.ASYNC_FETCH_RUNNING;
}
@ -251,9 +315,8 @@ public class MoodleCourseAccess extends AbstractCourseAccess {
return FetchStatus.ALL_FETCHED;
}
@Override
protected AllQuizzesSupplier allQuizzesSupplier() {
return this.allQuizzesSupplier;
public void clearCache() {
this.moodleCourseDataAsyncLoader.clearCache();
}
private List<QuizData> collectAllQuizzes(
@ -261,7 +324,6 @@ public class MoodleCourseAccess extends AbstractCourseAccess {
final FilterMap filterMap) {
final LmsSetup lmsSetup = getApiTemplateDataSupplier().getLmsSetup();
final String urlPrefix = (lmsSetup.lmsApiUrl.endsWith(Constants.URL_PATH_SEPARATOR))
? lmsSetup.lmsApiUrl + MOODLE_QUIZ_START_URL_PATH
: lmsSetup.lmsApiUrl + Constants.URL_PATH_SEPARATOR + MOODLE_QUIZ_START_URL_PATH;
@ -272,7 +334,7 @@ public class MoodleCourseAccess extends AbstractCourseAccess {
// Verify and call the proper strategy to get the course and quiz data
Collection<CourseDataShort> courseQuizData = Collections.emptyList();
if (this.moodleCourseDataAsyncLoader.isRunning()) {
courseQuizData = this.moodleCourseDataAsyncLoader.getCachedCourseData();
courseQuizData = this.moodleCourseDataAsyncLoader.getCachedCourseData().values();
} else if (this.moodleCourseDataAsyncLoader.getLastRunTime() <= 0) {
// set cut time if available
if (fromCutTime >= 0) {
@ -282,7 +344,7 @@ public class MoodleCourseAccess extends AbstractCourseAccess {
this.moodleCourseDataAsyncLoader.loadAsync(restTemplate);
try {
Thread.sleep(INITIAL_WAIT_TIME);
courseQuizData = this.moodleCourseDataAsyncLoader.getCachedCourseData();
courseQuizData = this.moodleCourseDataAsyncLoader.getCachedCourseData().values();
} catch (final Exception e) {
log.error("Failed to wait for first load run: ", e);
return Collections.emptyList();
@ -290,7 +352,7 @@ public class MoodleCourseAccess extends AbstractCourseAccess {
} else if (this.moodleCourseDataAsyncLoader.isLongRunningTask()) {
// on long running tasks if we have a different fromCutTime as before
// kick off the lazy loading task immediately with the new time filter
courseQuizData = this.moodleCourseDataAsyncLoader.getCachedCourseData();
courseQuizData = this.moodleCourseDataAsyncLoader.getCachedCourseData().values();
if (fromCutTime > 0 && fromCutTime != this.moodleCourseDataAsyncLoader.getFromCutTime()) {
this.moodleCourseDataAsyncLoader.setFromCutTime(fromCutTime);
this.moodleCourseDataAsyncLoader.loadAsync(restTemplate);
@ -306,7 +368,7 @@ public class MoodleCourseAccess extends AbstractCourseAccess {
this.moodleCourseDataAsyncLoader.setFromCutTime(fromCutTime);
}
this.moodleCourseDataAsyncLoader.loadSync(restTemplate);
courseQuizData = this.moodleCourseDataAsyncLoader.getCachedCourseData();
courseQuizData = this.moodleCourseDataAsyncLoader.getCachedCourseData().values();
}
if (courseQuizData.isEmpty()) {
@ -317,7 +379,8 @@ public class MoodleCourseAccess extends AbstractCourseAccess {
}
private List<QuizData> getCached() {
final Collection<CourseDataShort> courseQuizData = this.moodleCourseDataAsyncLoader.getCachedCourseData();
final Collection<CourseDataShort> courseQuizData =
this.moodleCourseDataAsyncLoader.getCachedCourseData().values();
final LmsSetup lmsSetup = getApiTemplateDataSupplier().getLmsSetup();
final String urlPrefix = (lmsSetup.lmsApiUrl.endsWith(Constants.URL_PATH_SEPARATOR))
? lmsSetup.lmsApiUrl + MOODLE_QUIZ_START_URL_PATH
@ -534,34 +597,41 @@ public class MoodleCourseAccess extends AbstractCourseAccess {
final List<QuizData> courseAndQuiz = courseData.quizzes
.stream()
.map(courseQuizData -> {
final String startURI = uriPrefix + courseQuizData.course_module;
//additionalAttrs.put(QuizData.ATTR_ADDITIONAL_TIME_LIMIT, String.valueOf(courseQuizData.time_limit));
return new QuizData(
getInternalQuizId(
courseQuizData.course_module,
courseData.id,
courseData.short_name,
courseData.idnumber),
lmsSetup.getInstitutionId(),
lmsSetup.id,
lmsSetup.getLmsType(),
courseQuizData.name,
Constants.EMPTY_NOTE,
(courseQuizData.time_open != null && courseQuizData.time_open > 0)
? Utils.toDateTimeUTCUnix(courseQuizData.time_open)
: Utils.toDateTimeUTCUnix(courseData.start_date),
(courseQuizData.time_close != null && courseQuizData.time_close > 0)
? Utils.toDateTimeUTCUnix(courseQuizData.time_close)
: Utils.toDateTimeUTCUnix(courseData.end_date),
startURI,
additionalAttrs);
})
.map(courseQuizData -> createQuizData(lmsSetup, courseData, uriPrefix, additionalAttrs, courseQuizData))
.collect(Collectors.toList());
return courseAndQuiz;
}
private QuizData createQuizData(
final LmsSetup lmsSetup,
final CourseDataShort courseData,
final String uriPrefix,
final Map<String, String> additionalAttrs,
final CourseQuizShort courseQuizData) {
final String startURI = uriPrefix + courseQuizData.course_module;
return new QuizData(
getInternalQuizId(
courseQuizData.course_module,
courseData.id,
courseData.short_name,
courseData.idnumber),
lmsSetup.getInstitutionId(),
lmsSetup.id,
lmsSetup.getLmsType(),
courseQuizData.name,
Constants.EMPTY_NOTE,
(courseQuizData.time_open != null && courseQuizData.time_open > 0)
? Utils.toDateTimeUTCUnix(courseQuizData.time_open)
: Utils.toDateTimeUTCUnix(courseData.start_date),
(courseQuizData.time_close != null && courseQuizData.time_close > 0)
? Utils.toDateTimeUTCUnix(courseQuizData.time_close)
: Utils.toDateTimeUTCUnix(courseData.end_date),
startURI,
additionalAttrs);
}
private Result<MoodleAPIRestTemplate> getRestTemplate() {
if (this.restTemplate == null) {
final Result<MoodleAPIRestTemplate> templateRequest = this.moodleRestTemplateFactory

30
src/main/java/ch/ethz/seb/sebserver/webservice/servicelayer/lms/impl/moodle/MoodleCourseDataAsyncLoader.java
View file

@ -65,7 +65,7 @@ public class MoodleCourseDataAsyncLoader {
private final int maxSize;
private final int pageSize;
private final Set<CourseDataShort> cachedCourseData = new HashSet<>();
private final Map<String, CourseDataShort> cachedCourseData = new HashMap<>();
private final Set<String> newIds = new HashSet<>();
private String lmsSetup = Constants.EMPTY_NOTE;
@ -122,8 +122,8 @@ public class MoodleCourseDataAsyncLoader {
this.fromCutTime = fromCutTime;
}
public Set<CourseDataShort> getCachedCourseData() {
return new HashSet<>(this.cachedCourseData);
public Map<String, CourseDataShort> getCachedCourseData() {
return new HashMap<>(this.cachedCourseData);
}
public long getLastRunTime() {
@ -138,7 +138,7 @@ public class MoodleCourseDataAsyncLoader {
return this.lastLoadTime > 3 * Constants.SECOND_IN_MILLIS;
}
public Set<CourseDataShort> loadSync(final MoodleAPIRestTemplate restTemplate) {
public Map<String, CourseDataShort> loadSync(final MoodleAPIRestTemplate restTemplate) {
if (this.running) {
throw new IllegalStateException("Is already running asynchronously");
}
@ -168,14 +168,20 @@ public class MoodleCourseDataAsyncLoader {
}
public void clearCache() {
if (!isRunning()) {
this.cachedCourseData.clear();
this.newIds.clear();
}
}
private Runnable loadAndCache(final MoodleAPIRestTemplate restTemplate) {
return () -> {
//this.cachedCourseData.clear();
this.newIds.clear();
final long startTime = Utils.getMillisecondsNow();
loadAllQuizzes(restTemplate);
syncCache();
this.syncCache();
this.lastLoadTime = Utils.getMillisecondsNow() - startTime;
this.running = false;
@ -263,7 +269,7 @@ public class MoodleCourseDataAsyncLoader {
this.lmsSetup,
c);
} else {
this.cachedCourseData.add(c);
this.cachedCourseData.put(c.id, c);
this.newIds.add(c.id);
}
});
@ -461,13 +467,15 @@ public class MoodleCourseDataAsyncLoader {
private void syncCache() {
if (!this.cachedCourseData.isEmpty()) {
final Set<CourseDataShort> newData = this.cachedCourseData.stream()
.filter(data -> this.newIds.contains(data.id))
final Set<String> oldData = this.cachedCourseData
.keySet()
.stream()
.filter(id -> !this.newIds.contains(id))
.collect(Collectors.toSet());
synchronized (this.cachedCourseData) {
this.cachedCourseData.clear();
this.cachedCourseData.addAll(newData);
oldData.stream().forEach(this.cachedCourseData::remove);
}
}
this.newIds.clear();

11
src/main/java/ch/ethz/seb/sebserver/webservice/servicelayer/lms/impl/moodle/MoodleLmsAPITemplate.java
View file

@ -110,6 +110,17 @@ public class MoodleLmsAPITemplate implements LmsAPITemplate {
.collect(Collectors.toList());
}
@Override
public Result<QuizData> getQuizFromCache(final String id) {
return this.moodleCourseAccess.getQuizFromCache(id)
.orElse(() -> getQuiz(id));
}
@Override
public void clearCache() {
this.moodleCourseAccess.clearCache();
}
@Override
public Collection<Result<QuizData>> getQuizzesFromCache(final Set<String> ids) {
return this.moodleCourseAccess.getQuizzesFromCache(ids)

3
src/main/java/ch/ethz/seb/sebserver/webservice/servicelayer/session/impl/ExamSessionServiceImpl.java
View file

@ -123,7 +123,8 @@ public class ExamSessionServiceImpl implements ExamSessionService {
return Result.tryCatch(() -> {
final Collection<APIMessage> result = new ArrayList<>();
final Exam exam = this.examDAO.byPK(examId)
final Exam exam = this.examDAO
.getWithQuizDataFromCache(examId)
.getOrThrow();
// check lms connection

7
src/main/java/ch/ethz/seb/sebserver/webservice/weblayer/api/ExamAdministrationController.java
View file

@ -279,7 +279,8 @@ public class ExamAdministrationController extends EntityController<Exam, Exam> {
defaultValue = UserService.USERS_INSTITUTION_AS_DEFAULT) final Long institutionId) {
checkReadPrivilege(institutionId);
return this.examDAO.byPK(modelId)
return this.examDAO
.getWithQuizDataFromCache(modelId)
.flatMap(this.examAdminService::isRestricted)
.getOrThrow();
}
@ -407,9 +408,7 @@ public class ExamAdministrationController extends EntityController<Exam, Exam> {
@PathVariable final Long modelId) {
checkReadPrivilege(institutionId);
return this.entityDAO.byPK(modelId)
.flatMap(this.authorization::checkRead)
.flatMap(exam -> this.examAdminService.getProctoringServiceSettings(exam.id))
return this.examAdminService.getProctoringServiceSettings(modelId)
.getOrThrow();
}

13
src/main/resources/config/ehcache.xml
View file

@ -23,7 +23,7 @@
<tti unit="hours">24</tti>
</expiry>
<resources>
<heap unit="entries">2000</heap>
<heap unit="entries">3000</heap>
</resources>
</cache>
@ -93,7 +93,16 @@
</resources>
</cache>
<cache alias="QUIZ_DATA_CACHE">
<key-type>java.lang.String</key-type>
<value-type>ch.ethz.seb.sebserver.gbl.model.exam.QuizData</value-type>
<expiry>
<tti unit="minutes">10</tti>
</expiry>
<resources>
<heap unit="entries">10000</heap>
</resources>
</cache>
</config>