zervo/seb-server
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Added EHcache for caching and improved Moodle asnyc loading

Browse source
This commit is contained in:
anhefti 2021-01-12 10:10:30 +01:00
parent 5f30aa9c2e
commit eec4392f78
20 changed files with 685 additions and 218 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

14
pom.xml
View file

@ -290,11 +290,15 @@
<artifactId>flyway-core</artifactId>
</dependency>
<!-- JMX -->
<!-- <dependency> -->
<!-- <groupId>org.jolokia</groupId> -->
<!-- <artifactId>jolokia-core</artifactId> -->
<!-- </dependency> -->
<!-- EHCache -->
<dependency>
<groupId>org.ehcache</groupId>
<artifactId>ehcache</artifactId>
</dependency>
<dependency>
<groupId>javax.cache</groupId>
<artifactId>cache-api</artifactId>
</dependency>
<!-- Apache HTTP -->
<dependency>

2
src/main/java/ch/ethz/seb/sebserver/SEBServer.java
View file

@ -15,7 +15,6 @@ import org.springframework.boot.autoconfigure.SpringBootApplication;
import org.springframework.boot.autoconfigure.security.servlet.UserDetailsServiceAutoConfiguration;
import org.springframework.boot.web.embedded.tomcat.TomcatServletWebServerFactory;
import org.springframework.boot.web.servlet.server.ServletWebServerFactory;
import org.springframework.cache.annotation.EnableCaching;
import org.springframework.context.ApplicationContext;
import org.springframework.context.annotation.Bean;
import org.springframework.core.env.Environment;
@ -40,7 +39,6 @@ import ch.ethz.seb.sebserver.gbl.profile.ProdWebServiceProfile;
@SpringBootApplication(exclude = {
UserDetailsServiceAutoConfiguration.class,
})
@EnableCaching
public class SEBServer {
public static void main(final String[] args) {

9
src/main/java/ch/ethz/seb/sebserver/gui/content/QuizLookupList.java
View file

@ -45,6 +45,7 @@ import ch.ethz.seb.sebserver.gui.service.page.impl.ModalInputDialog;
import ch.ethz.seb.sebserver.gui.service.page.impl.PageAction;
import ch.ethz.seb.sebserver.gui.service.remote.webservice.api.RestService;
import ch.ethz.seb.sebserver.gui.service.remote.webservice.api.exam.CheckExamImported;
import ch.ethz.seb.sebserver.gui.service.remote.webservice.api.quiz.GetQuizData;
import ch.ethz.seb.sebserver.gui.service.remote.webservice.api.quiz.GetQuizPage;
import ch.ethz.seb.sebserver.gui.service.remote.webservice.auth.CurrentUser;
import ch.ethz.seb.sebserver.gui.service.remote.webservice.auth.CurrentUser.GrantCheck;
@ -297,7 +298,11 @@ public class QuizLookupList implements TemplateComposer {
final QuizData quizData,
final Function<String, String> institutionNameFunction) {
action.getSingleSelection();
final QuizData fullQuizData = this.pageService.getRestService().getBuilder(GetQuizData.class)
.withURIVariable(API.PARAM_MODEL_ID, quizData.getModelId())
.withQueryParam(QuizData.QUIZ_ATTR_LMS_SETUP_ID, String.valueOf(quizData.lmsSetupId))
.call()
.getOr(quizData);
final ModalInputDialog<Void> dialog = new ModalInputDialog<Void>(
action.pageContext().getParent().getShell(),
@ -307,7 +312,7 @@ public class QuizLookupList implements TemplateComposer {
dialog.open(
DETAILS_TITLE_TEXT_KEY,
action.pageContext(),
pc -> createDetailsForm(quizData, pc, institutionNameFunction));
pc -> createDetailsForm(fullQuizData, pc, institutionNameFunction));
return action;
}

66
src/main/java/ch/ethz/seb/sebserver/webservice/CacheConfig.java Normal file
View file

@ -0,0 +1,66 @@
/*
* Copyright (c) 2021 ETH Zürich, Educational Development and Technology (LET)
*
* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/.
*/
package ch.ethz.seb.sebserver.webservice;
import java.net.URI;
import java.net.URISyntaxException;
import java.util.Arrays;
import javax.cache.Caching;
import javax.cache.spi.CachingProvider;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.cache.CacheManager;
import org.springframework.cache.annotation.EnableCaching;
import org.springframework.cache.concurrent.ConcurrentMapCacheManager;
import org.springframework.cache.jcache.JCacheCacheManager;
import org.springframework.cache.jcache.config.JCacheConfigurerSupport;
import org.springframework.cache.support.CompositeCacheManager;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import ch.ethz.seb.sebserver.gbl.profile.WebServiceProfile;
@EnableCaching
@WebServiceProfile
@Configuration
public class CacheConfig extends JCacheConfigurerSupport {
private static final Logger log = LoggerFactory.getLogger(CacheConfig.class);
@Value("${spring.cache.jcache.config}")
String jCacheConfig;
@Override
@Bean
public CacheManager cacheManager() {
try {
final CachingProvider cachingProvider = Caching.getCachingProvider();
final javax.cache.CacheManager cacheManager =
cachingProvider.getCacheManager(new URI(this.jCacheConfig), this.getClass().getClassLoader());
System.out.println("cacheManager:" + cacheManager);
final CompositeCacheManager composite = new CompositeCacheManager();
composite.setCacheManagers(Arrays.asList(
new JCacheCacheManager(cacheManager),
new ConcurrentMapCacheManager()));
composite.setFallbackToNoOpCache(true);
return composite;
} catch (final URISyntaxException e) {
log.error("Failed to initialize caching with EHCache. Fallback to simple caching");
return new ConcurrentMapCacheManager();
}
}
}

84
src/main/java/ch/ethz/seb/sebserver/webservice/servicelayer/lms/impl/CourseAccess.java
View file

@ -20,55 +20,81 @@ import java.util.stream.Collectors;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.core.env.Environment;
import ch.ethz.seb.sebserver.gbl.Constants;
import ch.ethz.seb.sebserver.gbl.async.AsyncService;
import ch.ethz.seb.sebserver.gbl.async.CircuitBreaker;
import ch.ethz.seb.sebserver.gbl.async.MemoizingCircuitBreaker;
import ch.ethz.seb.sebserver.gbl.model.exam.Chapters;
import ch.ethz.seb.sebserver.gbl.model.exam.QuizData;
import ch.ethz.seb.sebserver.gbl.model.user.ExamineeAccountDetails;
import ch.ethz.seb.sebserver.gbl.util.Result;
import ch.ethz.seb.sebserver.webservice.servicelayer.dao.FilterMap;
import ch.ethz.seb.sebserver.webservice.servicelayer.lms.LmsAPIService;
public abstract class CourseAccess {
private static final Logger log = LoggerFactory.getLogger(CourseAccess.class);
protected final MemoizingCircuitBreaker<List<QuizData>> allQuizzesRequest;
public enum FetchStatus {
ALL_FETCHED,
ASYNC_FETCH_RUNNING,
FETCH_ERROR
}
protected final CircuitBreaker<List<QuizData>> quizzesRequest;
protected final CircuitBreaker<Chapters> chaptersRequest;
protected final CircuitBreaker<ExamineeAccountDetails> accountDetailRequest;
protected CourseAccess(final AsyncService asyncService) {
this.allQuizzesRequest = asyncService.createMemoizingCircuitBreaker(
allQuizzesSupplier(null),
3,
Constants.MINUTE_IN_MILLIS,
Constants.MINUTE_IN_MILLIS,
true,
Constants.HOUR_IN_MILLIS);
protected CourseAccess(
final AsyncService asyncService,
final Environment environment) {
this.quizzesRequest = asyncService.createCircuitBreaker(
3,
Constants.MINUTE_IN_MILLIS,
Constants.MINUTE_IN_MILLIS);
environment.getProperty(
"sebserver.webservice.circuitbreaker.quizzesRequest.attempts",
Integer.class,
3),
environment.getProperty(
"sebserver.webservice.circuitbreaker.quizzesRequest.blockingTime",
Long.class,
Constants.MINUTE_IN_MILLIS),
environment.getProperty(
"sebserver.webservice.circuitbreaker.quizzesRequest.timeToRecover",
Long.class,
Constants.MINUTE_IN_MILLIS));
this.chaptersRequest = asyncService.createCircuitBreaker(
3,
Constants.SECOND_IN_MILLIS * 10,
Constants.MINUTE_IN_MILLIS);
environment.getProperty(
"sebserver.webservice.circuitbreaker.chaptersRequest.attempts",
Integer.class,
3),
environment.getProperty(
"sebserver.webservice.circuitbreaker.chaptersRequest.blockingTime",
Long.class,
Constants.SECOND_IN_MILLIS * 10),
environment.getProperty(
"sebserver.webservice.circuitbreaker.chaptersRequest.timeToRecover",
Long.class,
Constants.MINUTE_IN_MILLIS));
this.accountDetailRequest = asyncService.createCircuitBreaker(
1,
Constants.SECOND_IN_MILLIS * 10,
Constants.SECOND_IN_MILLIS * 10);
environment.getProperty(
"sebserver.webservice.circuitbreaker.accountDetailRequest.attempts",
Integer.class,
2),
environment.getProperty(
"sebserver.webservice.circuitbreaker.accountDetailRequest.blockingTime",
Long.class,
Constants.SECOND_IN_MILLIS * 10),
environment.getProperty(
"sebserver.webservice.circuitbreaker.accountDetailRequest.timeToRecover",
Long.class,
Constants.SECOND_IN_MILLIS * 10));
}
public Result<Collection<Result<QuizData>>> getQuizzesFromCache(final Set<String> ids) {
return Result.tryCatch(() -> {
final List<QuizData> cached = this.allQuizzesRequest.getCached();
final List<QuizData> cached = allQuizzesSupplier().getAllCached();
final List<QuizData> available = (cached != null)
? cached
: quizzesSupplier(ids).get();
@ -101,11 +127,7 @@ public abstract class CourseAccess {
}
public Result<List<QuizData>> getQuizzes(final FilterMap filterMap) {
if (filterMap != null) {
this.allQuizzesRequest.setSupplier(allQuizzesSupplier(filterMap));
}
return this.allQuizzesRequest.get()
.map(LmsAPIService.quizzesFilterFunction(filterMap));
return allQuizzesSupplier().getAll(filterMap);
}
public Result<ExamineeAccountDetails> getExamineeAccountDetails(final String examineeSessionId) {
@ -139,8 +161,16 @@ public abstract class CourseAccess {
protected abstract Supplier<List<QuizData>> quizzesSupplier(final Set<String> ids);
protected abstract Supplier<List<QuizData>> allQuizzesSupplier(final FilterMap filterMap);
protected abstract AllQuizzesSupplier allQuizzesSupplier();
protected abstract Supplier<Chapters> getCourseChaptersSupplier(final String courseId);
protected abstract FetchStatus getFetchStatus();
protected interface AllQuizzesSupplier {
List<QuizData> getAllCached();
Result<List<QuizData>> getAll(final FilterMap filterMap);
}
}

63
src/main/java/ch/ethz/seb/sebserver/webservice/servicelayer/lms/impl/edx/OpenEdxCourseAccess.java
View file

@ -20,6 +20,7 @@ import java.util.stream.Collectors;
import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.core.env.Environment;
import org.springframework.http.HttpEntity;
import org.springframework.http.HttpHeaders;
import org.springframework.http.HttpMethod;
@ -41,6 +42,8 @@ import com.fasterxml.jackson.core.type.TypeReference;
import ch.ethz.seb.sebserver.gbl.Constants;
import ch.ethz.seb.sebserver.gbl.api.JSONMapper;
import ch.ethz.seb.sebserver.gbl.async.AsyncService;
import ch.ethz.seb.sebserver.gbl.async.CircuitBreaker.State;
import ch.ethz.seb.sebserver.gbl.async.MemoizingCircuitBreaker;
import ch.ethz.seb.sebserver.gbl.model.exam.Chapters;
import ch.ethz.seb.sebserver.gbl.model.exam.QuizData;
import ch.ethz.seb.sebserver.gbl.model.institution.LmsSetup;
@ -70,6 +73,8 @@ final class OpenEdxCourseAccess extends CourseAccess {
private final LmsSetup lmsSetup;
private final OpenEdxRestTemplateFactory openEdxRestTemplateFactory;
private final WebserviceInfo webserviceInfo;
private final MemoizingCircuitBreaker<List<QuizData>> allQuizzesRequest;
private final AllQuizzesSupplier allQuizzesSupplier;
private OAuth2RestTemplate restTemplate;
@ -78,13 +83,51 @@ final class OpenEdxCourseAccess extends CourseAccess {
final LmsSetup lmsSetup,
final OpenEdxRestTemplateFactory openEdxRestTemplateFactory,
final WebserviceInfo webserviceInfo,
final AsyncService asyncService) {
final AsyncService asyncService,
final Environment environment) {
super(asyncService);
super(asyncService, environment);
this.jsonMapper = jsonMapper;
this.lmsSetup = lmsSetup;
this.openEdxRestTemplateFactory = openEdxRestTemplateFactory;
this.webserviceInfo = webserviceInfo;
this.allQuizzesRequest = asyncService.createMemoizingCircuitBreaker(
quizzesSupplier(),
environment.getProperty(
"sebserver.webservice.circuitbreaker.allQuizzesRequest.attempts",
Integer.class,
3),
environment.getProperty(
"sebserver.webservice.circuitbreaker.allQuizzesRequest.blockingTime",
Long.class,
Constants.MINUTE_IN_MILLIS),
environment.getProperty(
"sebserver.webservice.circuitbreaker.allQuizzesRequest.timeToRecover",
Long.class,
Constants.MINUTE_IN_MILLIS),
environment.getProperty(
"sebserver.webservice.circuitbreaker.allQuizzesRequest.memoize",
Boolean.class,
true),
environment.getProperty(
"sebserver.webservice.circuitbreaker.allQuizzesRequest.memoizingTime",
Long.class,
Constants.HOUR_IN_MILLIS));
this.allQuizzesSupplier = new AllQuizzesSupplier() {
@Override
public List<QuizData> getAllCached() {
return OpenEdxCourseAccess.this.allQuizzesRequest.getCached();
}
@Override
public Result<List<QuizData>> getAll(final FilterMap filterMap) {
return OpenEdxCourseAccess.this.allQuizzesRequest.get();
}
};
}
LmsSetupTestResult initAPIAccess() {
@ -173,8 +216,7 @@ final class OpenEdxCourseAccess extends CourseAccess {
.getOrThrow();
}
@Override
protected Supplier<List<QuizData>> allQuizzesSupplier(final FilterMap filterMap) {
private Supplier<List<QuizData>> quizzesSupplier() {
return () -> getRestTemplate()
.map(this::collectAllQuizzes)
.getOrThrow();
@ -451,4 +493,17 @@ final class OpenEdxCourseAccess extends CourseAccess {
return Result.of(this.restTemplate);
}
@Override
protected FetchStatus getFetchStatus() {
if (this.allQuizzesRequest.getState() != State.CLOSED) {
return FetchStatus.FETCH_ERROR;
}
return FetchStatus.ALL_FETCHED;
}
@Override
protected AllQuizzesSupplier allQuizzesSupplier() {
return this.allQuizzesSupplier;
}
}

7
src/main/java/ch/ethz/seb/sebserver/webservice/servicelayer/lms/impl/edx/OpenEdxLmsAPITemplateFactory.java
View file

@ -11,6 +11,7 @@ package ch.ethz.seb.sebserver.webservice.servicelayer.lms.impl.edx;
import org.apache.commons.lang3.StringUtils;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.annotation.Lazy;
import org.springframework.core.env.Environment;
import org.springframework.stereotype.Service;
import ch.ethz.seb.sebserver.ClientHttpRequestFactoryService;
@ -33,6 +34,7 @@ public class OpenEdxLmsAPITemplateFactory {
private final JSONMapper jsonMapper;
private final WebserviceInfo webserviceInfo;
private final AsyncService asyncService;
private final Environment environment;
private final ClientCredentialService clientCredentialService;
private final ClientHttpRequestFactoryService clientHttpRequestFactoryService;
private final String[] alternativeTokenRequestPaths;
@ -42,6 +44,7 @@ public class OpenEdxLmsAPITemplateFactory {
final JSONMapper jsonMapper,
final WebserviceInfo webserviceInfo,
final AsyncService asyncService,
final Environment environment,
final ClientCredentialService clientCredentialService,
final ClientHttpRequestFactoryService clientHttpRequestFactoryService,
@Value("${sebserver.webservice.lms.openedx.api.token.request.paths}") final String alternativeTokenRequestPaths,
@ -50,6 +53,7 @@ public class OpenEdxLmsAPITemplateFactory {
this.jsonMapper = jsonMapper;
this.webserviceInfo = webserviceInfo;
this.asyncService = asyncService;
this.environment = environment;
this.clientCredentialService = clientCredentialService;
this.clientHttpRequestFactoryService = clientHttpRequestFactoryService;
this.alternativeTokenRequestPaths = (alternativeTokenRequestPaths != null)
@ -78,7 +82,8 @@ public class OpenEdxLmsAPITemplateFactory {
lmsSetup,
openEdxRestTemplateFactory,
this.webserviceInfo,
this.asyncService);
this.asyncService,
this.environment);
final OpenEdxCourseRestriction openEdxCourseRestriction = new OpenEdxCourseRestriction(
lmsSetup,

186
src/main/java/ch/ethz/seb/sebserver/webservice/servicelayer/lms/impl/moodle/MoodleCourseAccess.java
View file

@ -23,6 +23,7 @@ import org.apache.commons.lang3.StringUtils;
import org.joda.time.DateTime;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.core.env.Environment;
import org.springframework.util.LinkedMultiValueMap;
import org.springframework.util.MultiValueMap;
@ -34,6 +35,7 @@ import com.fasterxml.jackson.core.type.TypeReference;
import ch.ethz.seb.sebserver.gbl.Constants;
import ch.ethz.seb.sebserver.gbl.api.JSONMapper;
import ch.ethz.seb.sebserver.gbl.async.AsyncService;
import ch.ethz.seb.sebserver.gbl.async.CircuitBreaker;
import ch.ethz.seb.sebserver.gbl.model.exam.Chapters;
import ch.ethz.seb.sebserver.gbl.model.exam.QuizData;
import ch.ethz.seb.sebserver.gbl.model.institution.LmsSetup;
@ -43,6 +45,7 @@ import ch.ethz.seb.sebserver.gbl.util.Result;
import ch.ethz.seb.sebserver.gbl.util.Utils;
import ch.ethz.seb.sebserver.webservice.servicelayer.dao.FilterMap;
import ch.ethz.seb.sebserver.webservice.servicelayer.lms.impl.CourseAccess;
import ch.ethz.seb.sebserver.webservice.servicelayer.lms.impl.moodle.MoodleCourseDataAsyncLoader.CourseDataShort;
import ch.ethz.seb.sebserver.webservice.servicelayer.lms.impl.moodle.MoodleRestTemplateFactory.MoodleAPIRestTemplate;
/** Implements the LmsAPITemplate for Open edX LMS Course API access.
@ -73,7 +76,9 @@ public class MoodleCourseAccess extends CourseAccess {
private final JSONMapper jsonMapper;
private final LmsSetup lmsSetup;
private final MoodleRestTemplateFactory moodleRestTemplateFactory;
private final MoodleCourseDataLazyLoader moodleCourseDataLazyLoader;
private final MoodleCourseDataAsyncLoader moodleCourseDataAsyncLoader;
private final CircuitBreaker<List<QuizData>> allQuizzesRequest;
private final AllQuizzesSupplier allQuizzesSupplier;
private MoodleAPIRestTemplate restTemplate;
@ -81,14 +86,42 @@ public class MoodleCourseAccess extends CourseAccess {
final JSONMapper jsonMapper,
final LmsSetup lmsSetup,
final MoodleRestTemplateFactory moodleRestTemplateFactory,
final MoodleCourseDataLazyLoader moodleCourseDataLazyLoader,
final AsyncService asyncService) {
final MoodleCourseDataAsyncLoader moodleCourseDataAsyncLoader,
final AsyncService asyncService,
final Environment environment) {
super(asyncService);
super(asyncService, environment);
this.jsonMapper = jsonMapper;
this.lmsSetup = lmsSetup;
this.moodleCourseDataLazyLoader = moodleCourseDataLazyLoader;
this.moodleCourseDataAsyncLoader = moodleCourseDataAsyncLoader;
this.moodleRestTemplateFactory = moodleRestTemplateFactory;
this.allQuizzesRequest = asyncService.createCircuitBreaker(
environment.getProperty(
"sebserver.webservice.circuitbreaker.allQuizzesRequest.attempts",
Integer.class,
3),
environment.getProperty(
"sebserver.webservice.circuitbreaker.allQuizzesRequest.blockingTime",
Long.class,
Constants.MINUTE_IN_MILLIS),
environment.getProperty(
"sebserver.webservice.circuitbreaker.allQuizzesRequest.timeToRecover",
Long.class,
Constants.MINUTE_IN_MILLIS));
this.allQuizzesSupplier = new AllQuizzesSupplier() {
@Override
public List<QuizData> getAllCached() {
return getCached();
}
@Override
public Result<List<QuizData>> getAll(final FilterMap filterMap) {
return MoodleCourseAccess.this.allQuizzesRequest.protectedRun(allQuizzesSupplier(filterMap));
}
};
}
@Override
@ -175,7 +208,6 @@ public class MoodleCourseAccess extends CourseAccess {
}
@Override
protected Supplier<List<QuizData>> allQuizzesSupplier(final FilterMap filterMap) {
return () -> getRestTemplate()
.map(template -> collectAllQuizzes(template, filterMap))
@ -187,6 +219,20 @@ public class MoodleCourseAccess extends CourseAccess {
throw new UnsupportedOperationException("not available yet");
}
@Override
protected FetchStatus getFetchStatus() {
if (this.moodleCourseDataAsyncLoader.isRunning()) {
return FetchStatus.ASYNC_FETCH_RUNNING;
}
return FetchStatus.ALL_FETCHED;
}
@Override
protected AllQuizzesSupplier allQuizzesSupplier() {
return this.allQuizzesSupplier;
}
private List<QuizData> collectAllQuizzes(
final MoodleAPIRestTemplate restTemplate,
final FilterMap filterMap) {
@ -198,42 +244,42 @@ public class MoodleCourseAccess extends CourseAccess {
final DateTime quizFromTime = (filterMap != null) ? filterMap.getQuizFromTime() : null;
final long fromCutTime = (quizFromTime != null) ? Utils.toUnixTimeInSeconds(quizFromTime) : -1;
Collection<CourseData> courseQuizData = Collections.emptyList();
if (this.moodleCourseDataLazyLoader.isRunning()) {
courseQuizData = this.moodleCourseDataLazyLoader.getPreFilteredCourseIds();
} else if (this.moodleCourseDataLazyLoader.getLastRunTime() <= 0) {
Collection<CourseDataShort> courseQuizData = Collections.emptyList();
if (this.moodleCourseDataAsyncLoader.isRunning()) {
courseQuizData = this.moodleCourseDataAsyncLoader.getCachedCourseData();
} else if (this.moodleCourseDataAsyncLoader.getLastRunTime() <= 0) {
// set cut time if available
if (fromCutTime >= 0) {
this.moodleCourseDataLazyLoader.setFromCutTime(fromCutTime);
this.moodleCourseDataAsyncLoader.setFromCutTime(fromCutTime);
}
// first run async and wait some time, get what is there
this.moodleCourseDataLazyLoader.loadAsync(restTemplate);
this.moodleCourseDataAsyncLoader.loadAsync(restTemplate);
try {
Thread.sleep(INITIAL_WAIT_TIME);
courseQuizData = this.moodleCourseDataLazyLoader.getPreFilteredCourseIds();
courseQuizData = this.moodleCourseDataAsyncLoader.getCachedCourseData();
} catch (final Exception e) {
log.error("Failed to wait for first load run: ", e);
return Collections.emptyList();
}
} else if (this.moodleCourseDataLazyLoader.isLongRunningTask()) {
} else if (this.moodleCourseDataAsyncLoader.isLongRunningTask()) {
// on long running tasks if we have a different fromCutTime as before
// kick off the lazy loading task immediately with the new time filter
if (fromCutTime > 0 && fromCutTime != this.moodleCourseDataLazyLoader.getFromCutTime()) {
this.moodleCourseDataLazyLoader.setFromCutTime(fromCutTime);
this.moodleCourseDataLazyLoader.loadAsync(restTemplate);
if (fromCutTime > 0 && fromCutTime != this.moodleCourseDataAsyncLoader.getFromCutTime()) {
this.moodleCourseDataAsyncLoader.setFromCutTime(fromCutTime);
this.moodleCourseDataAsyncLoader.loadAsync(restTemplate);
// otherwise kick off only if the last fetch task was then minutes ago
} else if (Utils.getMillisecondsNow() - this.moodleCourseDataLazyLoader.getLastRunTime() > 10
} else if (Utils.getMillisecondsNow() - this.moodleCourseDataAsyncLoader.getLastRunTime() > 10
* Constants.MINUTE_IN_MILLIS) {
this.moodleCourseDataLazyLoader.loadAsync(restTemplate);
this.moodleCourseDataAsyncLoader.loadAsync(restTemplate);
}
courseQuizData = this.moodleCourseDataLazyLoader.getPreFilteredCourseIds();
courseQuizData = this.moodleCourseDataAsyncLoader.getCachedCourseData();
} else {
// just run the task in sync
if (fromCutTime >= 0) {
this.moodleCourseDataLazyLoader.setFromCutTime(fromCutTime);
this.moodleCourseDataAsyncLoader.setFromCutTime(fromCutTime);
}
this.moodleCourseDataLazyLoader.loadSync(restTemplate);
courseQuizData = this.moodleCourseDataLazyLoader.getPreFilteredCourseIds();
this.moodleCourseDataAsyncLoader.loadSync(restTemplate);
courseQuizData = this.moodleCourseDataAsyncLoader.getCachedCourseData();
}
if (courseQuizData.isEmpty()) {
@ -257,6 +303,29 @@ public class MoodleCourseAccess extends CourseAccess {
});
}
private List<QuizData> getCached() {
final Collection<CourseDataShort> courseQuizData = this.moodleCourseDataAsyncLoader.getCachedCourseData();
final String urlPrefix = (this.lmsSetup.lmsApiUrl.endsWith(Constants.URL_PATH_SEPARATOR))
? this.lmsSetup.lmsApiUrl + MOODLE_QUIZ_START_URL_PATH
: this.lmsSetup.lmsApiUrl + Constants.URL_PATH_SEPARATOR + MOODLE_QUIZ_START_URL_PATH;
return courseQuizData
.stream()
.reduce(
new ArrayList<>(),
(list, courseData) -> {
list.addAll(quizDataOf(
this.lmsSetup,
courseData,
urlPrefix));
return list;
},
(list1, list2) -> {
list1.addAll(list2);
return list1;
});
}
private List<QuizData> getQuizzesForIds(
final MoodleAPIRestTemplate restTemplate,
final Set<String> quizIds) {
@ -405,6 +474,46 @@ public class MoodleCourseAccess extends CourseAccess {
return courseAndQuiz;
}
private List<QuizData> quizDataOf(
final LmsSetup lmsSetup,
final CourseDataShort courseData,
final String uriPrefix) {
additionalAttrs.clear();
additionalAttrs.put(QuizData.ATTR_ADDITIONAL_CREATION_TIME, String.valueOf(courseData.time_created));
additionalAttrs.put(QuizData.ATTR_ADDITIONAL_SHORT_NAME, courseData.short_name);
additionalAttrs.put(QuizData.ATTR_ADDITIONAL_ID_NUMBER, courseData.idnumber);
final List<QuizData> courseAndQuiz = courseData.quizzes
.stream()
.map(courseQuizData -> {
final String startURI = uriPrefix + courseQuizData.course_module;
//additionalAttrs.put(QuizData.ATTR_ADDITIONAL_TIME_LIMIT, String.valueOf(courseQuizData.time_limit));
return new QuizData(
getInternalQuizId(
courseQuizData.course_module,
courseData.id,
courseData.short_name,
courseData.idnumber),
lmsSetup.getInstitutionId(),
lmsSetup.id,
lmsSetup.getLmsType(),
courseQuizData.name,
Constants.EMPTY_NOTE,
(courseQuizData.time_open != null && courseQuizData.time_open > 0)
? Utils.toDateTimeUTCUnix(courseQuizData.time_open)
: Utils.toDateTimeUTCUnix(courseData.start_date),
(courseQuizData.time_close != null && courseQuizData.time_close > 0)
? Utils.toDateTimeUTCUnix(courseQuizData.time_close)
: Utils.toDateTimeUTCUnix(courseData.end_date),
startURI,
additionalAttrs);
})
.collect(Collectors.toList());
return courseAndQuiz;
}
private Result<MoodleAPIRestTemplate> getRestTemplate() {
if (this.restTemplate == null) {
final Result<MoodleAPIRestTemplate> templateRequest = this.moodleRestTemplateFactory
@ -482,7 +591,7 @@ public class MoodleCourseAccess extends CourseAccess {
/** Maps the Moodle course API course data */
@JsonIgnoreProperties(ignoreUnknown = true)
static final class CourseData {
private static final class CourseData {
final String id;
final String short_name;
final String idnumber;
@ -516,35 +625,10 @@ public class MoodleCourseAccess extends CourseAccess {
this.end_date = end_date;
this.time_created = time_created;
}
@Override
public int hashCode() {
final int prime = 31;
int result = 1;
result = prime * result + ((this.id == null) ? 0 : this.id.hashCode());
return result;
}
@Override
public boolean equals(final Object obj) {
if (this == obj)
return true;
if (obj == null)
return false;
if (getClass() != obj.getClass())
return false;
final CourseData other = (CourseData) obj;
if (this.id == null) {
if (other.id != null)
return false;
} else if (!this.id.equals(other.id))
return false;
return true;
}
}
@JsonIgnoreProperties(ignoreUnknown = true)
static final class Courses {
private static final class Courses {
final Collection<CourseData> courses;
@JsonCreator
@ -555,7 +639,7 @@ public class MoodleCourseAccess extends CourseAccess {
}
@JsonIgnoreProperties(ignoreUnknown = true)
static final class CourseQuizData {
private static final class CourseQuizData {
final Collection<CourseQuiz> quizzes;
@JsonCreator

325
src/main/java/ch/ethz/seb/sebserver/webservice/servicelayer/lms/impl/moodle/MoodleCourseDataLazyLoader.java → src/main/java/ch/ethz/seb/sebserver/webservice/servicelayer/lms/impl/moodle/MoodleCourseDataAsyncLoader.java
View file

@ -28,8 +28,10 @@ import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.config.ConfigurableBeanFactory;
import org.springframework.context.annotation.Lazy;
import org.springframework.context.annotation.Scope;
import org.springframework.core.env.Environment;
import org.springframework.stereotype.Component;
import org.springframework.util.LinkedMultiValueMap;
import org.springframework.util.MultiValueMap;
import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
@ -40,40 +42,72 @@ import com.fasterxml.jackson.databind.JsonMappingException;
import ch.ethz.seb.sebserver.gbl.Constants;
import ch.ethz.seb.sebserver.gbl.api.JSONMapper;
import ch.ethz.seb.sebserver.gbl.async.AsyncRunner;
import ch.ethz.seb.sebserver.gbl.async.AsyncService;
import ch.ethz.seb.sebserver.gbl.async.CircuitBreaker;
import ch.ethz.seb.sebserver.gbl.profile.WebServiceProfile;
import ch.ethz.seb.sebserver.gbl.util.Utils;
import ch.ethz.seb.sebserver.webservice.servicelayer.lms.impl.moodle.MoodleCourseAccess.CourseData;
import ch.ethz.seb.sebserver.webservice.servicelayer.lms.impl.moodle.MoodleCourseAccess.CourseQuiz;
import ch.ethz.seb.sebserver.webservice.servicelayer.lms.impl.moodle.MoodleCourseAccess.CourseQuizData;
import ch.ethz.seb.sebserver.webservice.servicelayer.lms.impl.moodle.MoodleCourseAccess.Courses;
import ch.ethz.seb.sebserver.webservice.servicelayer.lms.impl.moodle.MoodleRestTemplateFactory.MoodleAPIRestTemplate;
@Lazy
@Component
@WebServiceProfile
@Scope(ConfigurableBeanFactory.SCOPE_PROTOTYPE)
public class MoodleCourseDataLazyLoader {
public class MoodleCourseDataAsyncLoader {
private static final Logger log = LoggerFactory.getLogger(MoodleCourseDataLazyLoader.class);
private static final Logger log = LoggerFactory.getLogger(MoodleCourseDataAsyncLoader.class);
private final JSONMapper jsonMapper;
private final AsyncRunner asyncRunner;
private final CircuitBreaker<String> moodleRestCall;
private final int maxSize;
private final int pageSize;
private final Set<CourseData> preFilteredCourseIds = new HashSet<>();
private final Set<CourseDataShort> cachedCourseData = new HashSet<>();
private String lmsSetup = Constants.EMPTY_NOTE;
private long lastRunTime = 0;
private long lastLoadTime = 0;
private boolean running = false;
private long fromCutTime;
public MoodleCourseDataLazyLoader(
public MoodleCourseDataAsyncLoader(
final JSONMapper jsonMapper,
final AsyncRunner asyncRunner) {
final AsyncService asyncService,
final AsyncRunner asyncRunner,
final Environment environment) {
this.jsonMapper = jsonMapper;
this.asyncRunner = asyncRunner;
this.fromCutTime = Utils.toUnixTimeInSeconds(DateTime.now(DateTimeZone.UTC).minusYears(3));
this.asyncRunner = asyncRunner;
this.moodleRestCall = asyncService.createCircuitBreaker(
environment.getProperty(
"sebserver.webservice.circuitbreaker.moodleRestCall.attempts",
Integer.class,
2),
environment.getProperty(
"sebserver.webservice.circuitbreaker.moodleRestCall.blockingTime",
Long.class,
Constants.SECOND_IN_MILLIS * 20),
environment.getProperty(
"sebserver.webservice.circuitbreaker.moodleRestCall.timeToRecover",
Long.class,
Constants.MINUTE_IN_MILLIS));
this.maxSize =
environment.getProperty("sebserver.webservice.cache.moodle.course.maxSize", Integer.class, 10000);
this.pageSize =
environment.getProperty("sebserver.webservice.cache.moodle.course.pageSize", Integer.class, 500);
}
public void init(final String lmsSetupName) {
if (Constants.EMPTY_NOTE.equals(this.lmsSetup)) {
this.lmsSetup = lmsSetupName;
} else {
throw new IllegalStateException(
"Invalid initialization of MoodleCourseDataAsyncLoader. It has already been initialized yet");
}
}
public long getFromCutTime() {
@ -84,8 +118,8 @@ public class MoodleCourseDataLazyLoader {
this.fromCutTime = fromCutTime;
}
public Set<CourseData> getPreFilteredCourseIds() {
return this.preFilteredCourseIds;
public Set<CourseDataShort> getCachedCourseData() {
return new HashSet<>(this.cachedCourseData);
}
public long getLastRunTime() {
@ -100,7 +134,7 @@ public class MoodleCourseDataLazyLoader {
return this.lastLoadTime > 30 * Constants.SECOND_IN_MILLIS;
}
public Set<CourseData> loadSync(final MoodleAPIRestTemplate restTemplate) {
public Set<CourseDataShort> loadSync(final MoodleAPIRestTemplate restTemplate) {
if (this.running) {
throw new IllegalStateException("Is already running asynchronously");
}
@ -109,9 +143,11 @@ public class MoodleCourseDataLazyLoader {
loadAndCache(restTemplate).run();
this.lastRunTime = Utils.getMillisecondsNow();
log.info("Loaded {} courses synchronously", this.preFilteredCourseIds.size());
log.info("LMS Setup: {} loaded {} courses synchronously",
this.lmsSetup,
this.cachedCourseData.size());
return this.preFilteredCourseIds;
return this.cachedCourseData;
}
public void loadAsync(final MoodleAPIRestTemplate restTemplate) {
@ -126,6 +162,7 @@ public class MoodleCourseDataLazyLoader {
private Runnable loadAndCache(final MoodleAPIRestTemplate restTemplate) {
return () -> {
this.cachedCourseData.clear();
final long startTime = Utils.getMillisecondsNow();
loadAllQuizzes(restTemplate);
@ -133,7 +170,9 @@ public class MoodleCourseDataLazyLoader {
this.lastLoadTime = Utils.getMillisecondsNow() - startTime;
this.running = false;
log.info("Loaded {} courses asynchronously", this.preFilteredCourseIds.size());
log.info("LMS Setup: {} loaded {} courses asynchronously",
this.lmsSetup,
this.cachedCourseData.size());
};
}
@ -151,8 +190,8 @@ public class MoodleCourseDataLazyLoader {
try {
// first get courses from Moodle for page
final Map<String, CourseData> courseData = new HashMap<>();
final Collection<CourseData> coursesPage = getCoursesPage(restTemplate, page, 1000);
final Map<String, CourseDataShort> courseData = new HashMap<>();
final Collection<CourseDataShort> coursesPage = getCoursesPage(restTemplate, page, this.pageSize);
if (coursesPage == null || coursesPage.isEmpty()) {
return false;
@ -168,7 +207,8 @@ public class MoodleCourseDataLazyLoader {
MoodleCourseAccess.MOODLE_COURSE_API_COURSE_IDS,
new ArrayList<>(courseData.keySet()));
final String quizzesJSON = restTemplate.callMoodleAPIFunction(
final String quizzesJSON = callMoodleRestAPI(
restTemplate,
MoodleCourseAccess.MOODLE_QUIZ_API_FUNCTION_NAME,
attributes);
@ -185,28 +225,36 @@ public class MoodleCourseDataLazyLoader {
.stream()
.filter(getQuizFilter())
.forEach(quiz -> {
final CourseData data = courseData.get(quiz.course);
final CourseDataShort data = courseData.get(quiz.course);
if (data != null) {
data.quizzes.add(quiz);
}
});
this.preFilteredCourseIds.addAll(
courseData.values().stream()
.filter(c -> !c.quizzes.isEmpty())
.collect(Collectors.toList()));
courseData.values().stream()
.filter(c -> !c.quizzes.isEmpty())
.forEach(c -> {
if (this.cachedCourseData.size() >= this.maxSize) {
log.error(
"LMS Setup: {} Cache is full and has reached its maximal size. Skip data: -> {}",
this.lmsSetup,
c);
} else {
this.cachedCourseData.add(c);
}
});
return true;
} else {
return false;
}
} catch (final Exception e) {
log.error("Unexpected exception while trying to get course data: ", e);
log.error("LMS Setup: {} Unexpected exception while trying to get course data: ", this.lmsSetup, e);
return false;
}
}
private Collection<CourseData> getCoursesPage(
private Collection<CourseDataShort> getCoursesPage(
final MoodleAPIRestTemplate restTemplate,
final int page,
final int size) throws JsonParseException, JsonMappingException, IOException {
@ -219,7 +267,8 @@ public class MoodleCourseDataLazyLoader {
attributes.add(MoodleCourseAccess.MOODLE_COURSE_API_SEARCH_PAGE, String.valueOf(page));
attributes.add(MoodleCourseAccess.MOODLE_COURSE_API_SEARCH_PAGE_SIZE, String.valueOf(size));
final String courseKeyPageJSON = restTemplate.callMoodleAPIFunction(
final String courseKeyPageJSON = callMoodleRestAPI(
restTemplate,
MoodleCourseAccess.MOODLE_COURSE_SEARCH_API_FUNCTION_NAME,
attributes);
@ -228,7 +277,9 @@ public class MoodleCourseDataLazyLoader {
CoursePage.class);
if (keysPage == null || keysPage.courseKeys == null || keysPage.courseKeys.isEmpty()) {
log.info("No courses found on page: {}", page);
if (log.isDebugEnabled()) {
log.debug("LMS Setup: {} No courses found on page: {}", this.lmsSetup, page);
}
return Collections.emptyList();
}
@ -238,7 +289,7 @@ public class MoodleCourseDataLazyLoader {
.map(key -> key.id)
.collect(Collectors.toSet());
final Collection<CourseData> result = getCoursesForIds(restTemplate, ids)
final Collection<CourseDataShort> result = getCoursesForIds(restTemplate, ids)
.stream()
.filter(getCourseFilter())
.collect(Collectors.toList());
@ -247,19 +298,19 @@ public class MoodleCourseDataLazyLoader {
return result;
} catch (final Exception e) {
log.error("Unexpected error while trying to get courses page: ", e);
log.error("LMS Setup: {} Unexpected error while trying to get courses page: ", this.lmsSetup, e);
return Collections.emptyList();
}
}
private Collection<CourseData> getCoursesForIds(
private Collection<CourseDataShort> getCoursesForIds(
final MoodleAPIRestTemplate restTemplate,
final Set<String> ids) {
try {
if (log.isDebugEnabled()) {
log.debug("Get courses for ids: {}", ids);
log.debug("LMS Setup: {} Get courses for ids: {}", this.lmsSetup, ids);
}
final String joinedIds = StringUtils.join(ids, Constants.COMMA);
@ -267,32 +318,52 @@ public class MoodleCourseDataLazyLoader {
final LinkedMultiValueMap<String, String> attributes = new LinkedMultiValueMap<>();
attributes.add(MoodleCourseAccess.MOODLE_COURSE_API_FIELD_NAME, MoodleCourseAccess.MOODLE_COURSE_API_IDS);
attributes.add(MoodleCourseAccess.MOODLE_COURSE_API_FIELD_VALUE, joinedIds);
final String coursePageJSON = restTemplate.callMoodleAPIFunction(
final String coursePageJSON = callMoodleRestAPI(
restTemplate,
MoodleCourseAccess.MOODLE_COURSE_BY_FIELD_API_FUNCTION_NAME,
attributes);
return this.jsonMapper.<Courses> readValue(
return this.jsonMapper.readValue(
coursePageJSON,
Courses.class).courses;
} catch (final Exception e) {
log.error("Unexpected error while trying to get courses for ids", e);
log.error("LMS Setup: {} Unexpected error while trying to get courses for ids", this.lmsSetup, e);
return Collections.emptyList();
}
}
private Predicate<CourseQuiz> getQuizFilter() {
private String callMoodleRestAPI(
final MoodleAPIRestTemplate restTemplate,
final String function,
final MultiValueMap<String, String> queryAttributes) {
return this.moodleRestCall
.protectedRun(() -> restTemplate.callMoodleAPIFunction(
function,
queryAttributes))
.getOrThrow();
}
private Predicate<CourseQuizShort> getQuizFilter() {
final long now = Utils.getSecondsNow();
return quiz -> {
if (quiz.time_close == null || quiz.time_close == 0 || quiz.time_close > now) {
return true;
}
log.info("remove quiz {} end_time {} now {}", quiz.name, quiz.time_close, now);
if (log.isDebugEnabled()) {
log.debug("LMS Setup: {} remove quiz {} end_time {} now {}",
this.lmsSetup,
quiz.name,
quiz.time_close,
now);
}
return false;
};
}
private Predicate<CourseData> getCourseFilter() {
private Predicate<CourseDataShort> getCourseFilter() {
final long now = Utils.getSecondsNow();
return course -> {
if (course.start_date < this.fromCutTime) {
@ -303,7 +374,13 @@ public class MoodleCourseDataLazyLoader {
return true;
}
log.info("remove course {} end_time {} now {}", course.short_name, course.end_date, now);
if (log.isDebugEnabled()) {
log.info("LMS Setup: {} remove course {} end_time {} now {}",
this.lmsSetup,
course.short_name,
course.end_date,
now);
}
return false;
};
}
@ -356,77 +433,107 @@ public class MoodleCourseDataLazyLoader {
}
// @JsonIgnoreProperties(ignoreUnknown = true)
// static final class CourseKeys {
// final Collection<CourseDataKey> courses;
//
// @JsonCreator
// protected CourseKeys(
// @JsonProperty(value = "courses") final Collection<CourseDataKey> courses) {
// this.courses = courses;
// }
// }
/** Maps the Moodle course API course data */
@JsonIgnoreProperties(ignoreUnknown = true)
static final class CourseDataShort {
final String id;
final String short_name;
final String idnumber;
final Long start_date; // unix-time seconds UTC
final Long end_date; // unix-time seconds UTC
final Long time_created; // unix-time seconds UTC
final Collection<CourseQuizShort> quizzes = new ArrayList<>();
// /** Maps the Moodle course API course data */
// @JsonIgnoreProperties(ignoreUnknown = true)
// static final class CourseDataKey {
// final String id;
// final String short_name;
// final Long start_date; // unix-time seconds UTC
// final Long end_date; // unix-time seconds UTC
// final Long time_created; // unix-time seconds UTC
// final Collection<CourseQuizKey> quizzes = new ArrayList<>();
//
// @JsonCreator
// protected CourseDataKey(
// @JsonProperty(value = "id") final String id,
// @JsonProperty(value = "shortname") final String short_name,
// @JsonProperty(value = "startdate") final Long start_date,
// @JsonProperty(value = "enddate") final Long end_date,
// @JsonProperty(value = "timecreated") final Long time_created) {
//
// this.id = id;
// this.short_name = short_name;
// this.start_date = start_date;
// this.end_date = end_date;
// this.time_created = time_created;
// }
//
// }
@JsonCreator
protected CourseDataShort(
@JsonProperty(value = "id") final String id,
@JsonProperty(value = "shortname") final String short_name,
@JsonProperty(value = "idnumber") final String idnumber,
@JsonProperty(value = "startdate") final Long start_date,
@JsonProperty(value = "enddate") final Long end_date,
@JsonProperty(value = "timecreated") final Long time_created) {
// @JsonIgnoreProperties(ignoreUnknown = true)
// static final class CourseQuizKeys {
// final Collection<CourseQuizKey> quizzes;
//
// @JsonCreator
// protected CourseQuizKeys(
// @JsonProperty(value = "quizzes") final Collection<CourseQuizKey> quizzes) {
// this.quizzes = quizzes;
// }
// }
//
// @JsonIgnoreProperties(ignoreUnknown = true)
// static final class CourseQuizKey {
// final String id;
// final String course;
// final String name;
// final Long time_open; // unix-time seconds UTC
// final Long time_close; // unix-time seconds UTC
//
// @JsonCreator
// protected CourseQuizKey(
// @JsonProperty(value = "id") final String id,
// @JsonProperty(value = "course") final String course,
// @JsonProperty(value = "name") final String name,
// @JsonProperty(value = "timeopen") final Long time_open,
// @JsonProperty(value = "timeclose") final Long time_close) {
//
// this.id = id;
// this.course = course;
// this.name = name;
// this.time_open = time_open;
// this.time_close = time_close;
// }
// }
this.id = id;
this.short_name = short_name;
this.idnumber = idnumber;
this.start_date = start_date;
this.end_date = end_date;
this.time_created = time_created;
}
@Override
public int hashCode() {
final int prime = 31;
int result = 1;
result = prime * result + ((this.id == null) ? 0 : this.id.hashCode());
return result;
}
@Override
public boolean equals(final Object obj) {
if (this == obj)
return true;
if (obj == null)
return false;
if (getClass() != obj.getClass())
return false;
final CourseDataShort other = (CourseDataShort) obj;
if (this.id == null) {
if (other.id != null)
return false;
} else if (!this.id.equals(other.id))
return false;
return true;
}
}
@JsonIgnoreProperties(ignoreUnknown = true)
private static final class Courses {
final Collection<CourseDataShort> courses;
@JsonCreator
protected Courses(
@JsonProperty(value = "courses") final Collection<CourseDataShort> courses) {
this.courses = courses;
}
}
@JsonIgnoreProperties(ignoreUnknown = true)
static final class CourseQuizData {
final Collection<CourseQuizShort> quizzes;
@JsonCreator
protected CourseQuizData(
@JsonProperty(value = "quizzes") final Collection<CourseQuizShort> quizzes) {
this.quizzes = quizzes;
}
}
@JsonIgnoreProperties(ignoreUnknown = true)
static final class CourseQuizShort {
final String id;
final String course;
final String course_module;
final String name;
final Long time_open; // unix-time seconds UTC
final Long time_close; // unix-time seconds UTC
@JsonCreator
protected CourseQuizShort(
@JsonProperty(value = "id") final String id,
@JsonProperty(value = "course") final String course,
@JsonProperty(value = "coursemodule") final String course_module,
@JsonProperty(value = "name") final String name,
@JsonProperty(value = "timeopen") final Long time_open,
@JsonProperty(value = "timeclose") final Long time_close) {
this.id = id;
this.course = course;
this.course_module = course_module;
this.name = name;
this.time_open = time_open;
this.time_close = time_close;
}
}
}

14
src/main/java/ch/ethz/seb/sebserver/webservice/servicelayer/lms/impl/moodle/MoodleLmsAPITemplateFactory.java
View file

@ -12,6 +12,7 @@ import org.apache.commons.lang3.StringUtils;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.ApplicationContext;
import org.springframework.context.annotation.Lazy;
import org.springframework.core.env.Environment;
import org.springframework.stereotype.Service;
import ch.ethz.seb.sebserver.ClientHttpRequestFactoryService;
@ -32,6 +33,7 @@ public class MoodleLmsAPITemplateFactory {
private final JSONMapper jsonMapper;
private final AsyncService asyncService;
private final Environment environment;
private final ClientCredentialService clientCredentialService;
private final ClientHttpRequestFactoryService clientHttpRequestFactoryService;
private final ApplicationContext applicationContext;
@ -40,6 +42,7 @@ public class MoodleLmsAPITemplateFactory {
protected MoodleLmsAPITemplateFactory(
final JSONMapper jsonMapper,
final AsyncService asyncService,
final Environment environment,
final ClientCredentialService clientCredentialService,
final ClientHttpRequestFactoryService clientHttpRequestFactoryService,
final ApplicationContext applicationContext,
@ -47,6 +50,7 @@ public class MoodleLmsAPITemplateFactory {
this.jsonMapper = jsonMapper;
this.asyncService = asyncService;
this.environment = environment;
this.clientCredentialService = clientCredentialService;
this.clientHttpRequestFactoryService = clientHttpRequestFactoryService;
this.applicationContext = applicationContext;
@ -62,8 +66,9 @@ public class MoodleLmsAPITemplateFactory {
return Result.tryCatch(() -> {
final MoodleCourseDataLazyLoader lazyLoaderPrototype =
this.applicationContext.getBean(MoodleCourseDataLazyLoader.class);
final MoodleCourseDataAsyncLoader asyncLoaderPrototype =
this.applicationContext.getBean(MoodleCourseDataAsyncLoader.class);
asyncLoaderPrototype.init(lmsSetup.name);
final MoodleRestTemplateFactory moodleRestTemplateFactory = new MoodleRestTemplateFactory(
this.jsonMapper,
@ -78,8 +83,9 @@ public class MoodleLmsAPITemplateFactory {
this.jsonMapper,
lmsSetup,
moodleRestTemplateFactory,
lazyLoaderPrototype,
this.asyncService);
asyncLoaderPrototype,
this.asyncService,
this.environment);
final MoodleCourseRestriction moodleCourseRestriction = new MoodleCourseRestriction(
this.jsonMapper,

4
src/main/java/ch/ethz/seb/sebserver/webservice/servicelayer/session/impl/SEBClientConnectionServiceImpl.java
View file

@ -489,9 +489,7 @@ public class SEBClientConnectionServiceImpl implements SEBClientConnectionServic
.filter(Objects::nonNull)
.filter(connection -> connection.pingIndicator != null &&
connection.clientConnection.status.establishedStatus)
.map(connection -> connection.pingIndicator.updateLogEvent())
.filter(Objects::nonNull)
.forEach(this.eventHandlingStrategy);
.forEach(connection -> connection.pingIndicator.updateLogEvent());
} catch (final Exception e) {
log.error("Failed to update ping events: ", e);

12
src/main/java/ch/ethz/seb/sebserver/webservice/servicelayer/session/impl/SEBClientNotificationServiceImpl.java
View file

@ -42,7 +42,7 @@ public class SEBClientNotificationServiceImpl implements SEBClientNotificationSe
private final ClientEventDAO clientEventDAO;
private final SEBClientInstructionService sebClientInstructionService;
private final Set<Long> pendingNotifications;
private final Set<Long> pendingNotifications = new HashSet<>();
public SEBClientNotificationServiceImpl(
final ClientEventDAO clientEventDAO,
@ -50,19 +50,25 @@ public class SEBClientNotificationServiceImpl implements SEBClientNotificationSe
this.clientEventDAO = clientEventDAO;
this.sebClientInstructionService = sebClientInstructionService;
this.pendingNotifications = new HashSet<>();
}
@Override
public Boolean hasAnyPendingNotification(final Long clientConnectionId) {
if (this.pendingNotifications.contains(clientConnectionId)) {
if (this.pendingNotifications.add(clientConnectionId)) {
return true;
}
final boolean hasAnyPendingNotification = !getPendingNotifications(clientConnectionId)
.getOr(Collections.emptyList())
.isEmpty();
if (hasAnyPendingNotification) {
// NOTE this is a quick and dirty way to keep cache pendingNotifications cache size short.
// TODO find a better way to do this.
if (this.pendingNotifications.size() > 100) {
this.pendingNotifications.clear();
}
this.pendingNotifications.add(clientConnectionId);
}

2
src/main/resources/config/application-dev-ws.properties
View file

@ -1,8 +1,6 @@
server.address=localhost
server.port=8090
#logging.file=log/sebserver.log
# data source configuration
spring.datasource.initialize=true
spring.datasource.initialization-mode=always

2
src/main/resources/config/application-dev.properties
View file

@ -6,6 +6,8 @@ server.servlet.context-path=/
server.tomcat.uri-encoding=UTF-8
logging.level.ch=INFO
logging.level.org.springframework.cache=INFO
logging.level.ch.ethz.seb.sebserver.webservice.servicelayer.lms.impl=DEBUG
sebserver.http.client.connect-timeout=150000
sebserver.http.client.connection-request-timeout=100000

4
src/main/resources/config/application-ws.properties
View file

@ -8,6 +8,10 @@ sebserver.init.adminaccount.gen-on-init=true
sebserver.init.organisation.name=SEB Server
sebserver.init.adminaccount.username=sebserver-admin
### webservice caching
spring.cache.jcache.provider=org.ehcache.jsr107.EhcacheCachingProvider
spring.cache.jcache.config=classpath:config/ehcache.xml
### webservice data source configuration
spring.datasource.username=root
spring.datasource.initialize=true

2
src/main/resources/config/application.properties
View file

@ -13,7 +13,7 @@ server.port=8080
server.servlet.context-path=/
# Tomcat
server.tomcat.max-threads=1000
server.tomcat.max-threads=2000
server.tomcat.accept-count=300
server.tomcat.uri-encoding=UTF-8

85
src/main/resources/config/ehcache.xml Normal file
View file

@ -0,0 +1,85 @@
<config xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xmlns="http://www.ehcache.org/v3"
xmlns:jsr107="http://www.ehcache.org/v3/jsr107"
xsi:schemaLocation="
http://www.ehcache.org/v3 http://www.ehcache.org/schema/ehcache-core-3.0.xsd
http://www.ehcache.org/v3/jsr107 http://www.ehcache.org/schema/ehcache-107-ext-3.0.xsd">
<cache alias="RUNNING_EXAM">
<key-type>java.lang.Long</key-type>
<value-type>ch.ethz.seb.sebserver.gbl.model.exam.Exam</value-type>
<expiry>
<tti unit="hours">24</tti>
</expiry>
<resources>
<heap unit="entries">100</heap>
</resources>
</cache>
<cache alias="ACTIVE_CLIENT_CONNECTION">
<key-type>java.lang.String</key-type>
<value-type>ch.ethz.seb.sebserver.webservice.servicelayer.session.impl.ClientConnectionDataInternal</value-type>
<expiry>
<tti unit="hours">24</tti>
</expiry>
<resources>
<heap unit="entries">2000</heap>
</resources>
</cache>
<cache alias="SEB_CONFIG_EXAM">
<key-type>java.lang.Long</key-type>
<value-type>ch.ethz.seb.sebserver.webservice.servicelayer.session.impl.InMemorySEBConfig</value-type>
<expiry>
<tti unit="hours">24</tti>
</expiry>
<resources>
<heap unit="entries">20</heap>
</resources>
</cache>
<cache alias="CACHE_NAME_PING_RECORD">
<key-type>java.lang.String</key-type>
<value-type>ch.ethz.seb.sebserver.webservice.datalayer.batis.model.ClientEventRecord</value-type>
<expiry>
<tti unit="hours">24</tti>
</expiry>
<resources>
<heap unit="entries">2000</heap>
</resources>
</cache>
<cache alias="CONNECTION_TOKENS_CACHE">
<key-type>java.lang.Long</key-type>
<value-type>ch.ethz.seb.sebserver.gbl.util.Result</value-type>
<expiry>
<tti unit="hours">24</tti>
</expiry>
<resources>
<heap unit="entries">100</heap>
</resources>
</cache>
<cache alias="ACCESS_TOKEN_STORE_CACHE">
<key-type>org.springframework.security.oauth2.common.OAuth2AccessToken</key-type>
<value-type>org.springframework.security.oauth2.provider.OAuth2Authentication</value-type>
<expiry>
<tti unit="hours">24</tti>
</expiry>
<resources>
<heap unit="entries">100</heap>
</resources>
</cache>
<cache alias="EXAM_CLIENT_DETAILS_CACHE">
<key-type>java.lang.String</key-type>
<value-type>ch.ethz.seb.sebserver.gbl.util.Result</value-type>
<expiry>
<tti unit="hours">24</tti>
</expiry>
<resources>
<heap unit="entries">2000</heap>
</resources>
</cache>
</config>

1
src/main/resources/logback-spring.xml
View file

@ -19,6 +19,7 @@
<root level="DEBUG" additivity="true">
<appender-ref ref="STDOUT" />
<appender-ref ref="FILE" />
</root>
<Logger name="ch.ethz.seb.SEB_SERVER_INIT" level="INFO" additivity="false">
<appender-ref ref="STDOUT" />

18
src/test/java/ch/ethz/seb/sebserver/webservice/servicelayer/lms/impl/moodle/MoodleCourseAccessTest.java
View file

@ -15,6 +15,9 @@ import static org.mockito.Mockito.*;
import java.util.TreeMap;
import org.junit.Test;
import org.mockito.Mock;
import org.springframework.core.env.Environment;
import org.springframework.mock.env.MockEnvironment;
import org.springframework.util.LinkedMultiValueMap;
import org.springframework.util.MultiValueMap;
@ -28,6 +31,9 @@ import ch.ethz.seb.sebserver.webservice.servicelayer.lms.impl.moodle.MoodleRestT
public class MoodleCourseAccessTest {
@Mock
Environment env = new MockEnvironment();
@Test
public void testGetExamineeAccountDetails() {
@ -68,7 +74,8 @@ public class MoodleCourseAccessTest {
null,
moodleRestTemplateFactory,
null,
mock(AsyncService.class));
mock(AsyncService.class),
this.env);
final String examId = "123";
final Result<ExamineeAccountDetails> examineeAccountDetails =
@ -116,7 +123,8 @@ public class MoodleCourseAccessTest {
null,
moodleRestTemplateFactory,
null,
mock(AsyncService.class));
mock(AsyncService.class),
this.env);
final LmsSetupTestResult initAPIAccess = moodleCourseAccess.initAPIAccess();
assertNotNull(initAPIAccess);
@ -138,7 +146,8 @@ public class MoodleCourseAccessTest {
null,
moodleRestTemplateFactory,
null,
mock(AsyncService.class));
mock(AsyncService.class),
this.env);
final LmsSetupTestResult initAPIAccess = moodleCourseAccess.initAPIAccess();
assertNotNull(initAPIAccess);
@ -159,7 +168,8 @@ public class MoodleCourseAccessTest {
null,
moodleRestTemplateFactory,
null,
mock(AsyncService.class));
mock(AsyncService.class),
this.env);
final LmsSetupTestResult initAPIAccess = moodleCourseAccess.initAPIAccess();
assertNotNull(initAPIAccess);

3
src/test/resources/application-test.properties
View file

@ -12,6 +12,9 @@ server.servlet.context-path=/
spring.main.allow-bean-definition-overriding=true
sebserver.password=test-password
spring.cache.jcache.provider=
spring.cache.jcache.config=classpath:config/ehcache.xml
spring.h2.console.enabled=true
spring.datasource.platform=h2
spring.datasource.url=jdbc:h2:mem:test;MODE=MySQL;DB_CLOSE_DELAY=-1;DB_CLOSE_ON_EXIT=FALSE