diff --git a/pom.xml b/pom.xml index 47304872..1ed4d64b 100644 --- a/pom.xml +++ b/pom.xml @@ -290,11 +290,15 @@ flyway-core - - - - - + + + org.ehcache + ehcache + + + javax.cache + cache-api + diff --git a/src/main/java/ch/ethz/seb/sebserver/SEBServer.java b/src/main/java/ch/ethz/seb/sebserver/SEBServer.java index 2bb3ba30..f0ae0de5 100644 --- a/src/main/java/ch/ethz/seb/sebserver/SEBServer.java +++ b/src/main/java/ch/ethz/seb/sebserver/SEBServer.java @@ -15,7 +15,6 @@ import org.springframework.boot.autoconfigure.SpringBootApplication; import org.springframework.boot.autoconfigure.security.servlet.UserDetailsServiceAutoConfiguration; import org.springframework.boot.web.embedded.tomcat.TomcatServletWebServerFactory; import org.springframework.boot.web.servlet.server.ServletWebServerFactory; -import org.springframework.cache.annotation.EnableCaching; import org.springframework.context.ApplicationContext; import org.springframework.context.annotation.Bean; import org.springframework.core.env.Environment; @@ -40,7 +39,6 @@ import ch.ethz.seb.sebserver.gbl.profile.ProdWebServiceProfile; @SpringBootApplication(exclude = { UserDetailsServiceAutoConfiguration.class, }) -@EnableCaching public class SEBServer { public static void main(final String[] args) { diff --git a/src/main/java/ch/ethz/seb/sebserver/gui/content/QuizLookupList.java b/src/main/java/ch/ethz/seb/sebserver/gui/content/QuizLookupList.java index 2df044aa..a6ae673e 100644 --- a/src/main/java/ch/ethz/seb/sebserver/gui/content/QuizLookupList.java +++ b/src/main/java/ch/ethz/seb/sebserver/gui/content/QuizLookupList.java @@ -45,6 +45,7 @@ import ch.ethz.seb.sebserver.gui.service.page.impl.ModalInputDialog; import ch.ethz.seb.sebserver.gui.service.page.impl.PageAction; import ch.ethz.seb.sebserver.gui.service.remote.webservice.api.RestService; import ch.ethz.seb.sebserver.gui.service.remote.webservice.api.exam.CheckExamImported; +import ch.ethz.seb.sebserver.gui.service.remote.webservice.api.quiz.GetQuizData; import ch.ethz.seb.sebserver.gui.service.remote.webservice.api.quiz.GetQuizPage; import ch.ethz.seb.sebserver.gui.service.remote.webservice.auth.CurrentUser; import ch.ethz.seb.sebserver.gui.service.remote.webservice.auth.CurrentUser.GrantCheck; @@ -297,7 +298,11 @@ public class QuizLookupList implements TemplateComposer { final QuizData quizData, final Function institutionNameFunction) { - action.getSingleSelection(); + final QuizData fullQuizData = this.pageService.getRestService().getBuilder(GetQuizData.class) + .withURIVariable(API.PARAM_MODEL_ID, quizData.getModelId()) + .withQueryParam(QuizData.QUIZ_ATTR_LMS_SETUP_ID, String.valueOf(quizData.lmsSetupId)) + .call() + .getOr(quizData); final ModalInputDialog dialog = new ModalInputDialog( action.pageContext().getParent().getShell(), @@ -307,7 +312,7 @@ public class QuizLookupList implements TemplateComposer { dialog.open( DETAILS_TITLE_TEXT_KEY, action.pageContext(), - pc -> createDetailsForm(quizData, pc, institutionNameFunction)); + pc -> createDetailsForm(fullQuizData, pc, institutionNameFunction)); return action; } diff --git a/src/main/java/ch/ethz/seb/sebserver/webservice/CacheConfig.java b/src/main/java/ch/ethz/seb/sebserver/webservice/CacheConfig.java new file mode 100644 index 00000000..19240cfe --- /dev/null +++ b/src/main/java/ch/ethz/seb/sebserver/webservice/CacheConfig.java @@ -0,0 +1,66 @@ +/* + * Copyright (c) 2021 ETH Zürich, Educational Development and Technology (LET) + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ + +package ch.ethz.seb.sebserver.webservice; + +import java.net.URI; +import java.net.URISyntaxException; +import java.util.Arrays; + +import javax.cache.Caching; +import javax.cache.spi.CachingProvider; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.cache.CacheManager; +import org.springframework.cache.annotation.EnableCaching; +import org.springframework.cache.concurrent.ConcurrentMapCacheManager; +import org.springframework.cache.jcache.JCacheCacheManager; +import org.springframework.cache.jcache.config.JCacheConfigurerSupport; +import org.springframework.cache.support.CompositeCacheManager; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +import ch.ethz.seb.sebserver.gbl.profile.WebServiceProfile; + +@EnableCaching +@WebServiceProfile +@Configuration +public class CacheConfig extends JCacheConfigurerSupport { + + private static final Logger log = LoggerFactory.getLogger(CacheConfig.class); + + @Value("${spring.cache.jcache.config}") + String jCacheConfig; + + @Override + @Bean + public CacheManager cacheManager() { + try { + final CachingProvider cachingProvider = Caching.getCachingProvider(); + final javax.cache.CacheManager cacheManager = + cachingProvider.getCacheManager(new URI(this.jCacheConfig), this.getClass().getClassLoader()); + System.out.println("cacheManager:" + cacheManager); + + final CompositeCacheManager composite = new CompositeCacheManager(); + composite.setCacheManagers(Arrays.asList( + new JCacheCacheManager(cacheManager), + new ConcurrentMapCacheManager())); + composite.setFallbackToNoOpCache(true); + + return composite; + + } catch (final URISyntaxException e) { + log.error("Failed to initialize caching with EHCache. Fallback to simple caching"); + return new ConcurrentMapCacheManager(); + } + + } + +} diff --git a/src/main/java/ch/ethz/seb/sebserver/webservice/servicelayer/lms/impl/CourseAccess.java b/src/main/java/ch/ethz/seb/sebserver/webservice/servicelayer/lms/impl/CourseAccess.java index f5e02d24..edc58243 100644 --- a/src/main/java/ch/ethz/seb/sebserver/webservice/servicelayer/lms/impl/CourseAccess.java +++ b/src/main/java/ch/ethz/seb/sebserver/webservice/servicelayer/lms/impl/CourseAccess.java @@ -20,55 +20,81 @@ import java.util.stream.Collectors; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.springframework.core.env.Environment; import ch.ethz.seb.sebserver.gbl.Constants; import ch.ethz.seb.sebserver.gbl.async.AsyncService; import ch.ethz.seb.sebserver.gbl.async.CircuitBreaker; -import ch.ethz.seb.sebserver.gbl.async.MemoizingCircuitBreaker; import ch.ethz.seb.sebserver.gbl.model.exam.Chapters; import ch.ethz.seb.sebserver.gbl.model.exam.QuizData; import ch.ethz.seb.sebserver.gbl.model.user.ExamineeAccountDetails; import ch.ethz.seb.sebserver.gbl.util.Result; import ch.ethz.seb.sebserver.webservice.servicelayer.dao.FilterMap; -import ch.ethz.seb.sebserver.webservice.servicelayer.lms.LmsAPIService; public abstract class CourseAccess { private static final Logger log = LoggerFactory.getLogger(CourseAccess.class); - protected final MemoizingCircuitBreaker> allQuizzesRequest; + public enum FetchStatus { + ALL_FETCHED, + ASYNC_FETCH_RUNNING, + FETCH_ERROR + } + protected final CircuitBreaker> quizzesRequest; protected final CircuitBreaker chaptersRequest; protected final CircuitBreaker accountDetailRequest; - protected CourseAccess(final AsyncService asyncService) { - this.allQuizzesRequest = asyncService.createMemoizingCircuitBreaker( - allQuizzesSupplier(null), - 3, - Constants.MINUTE_IN_MILLIS, - Constants.MINUTE_IN_MILLIS, - true, - Constants.HOUR_IN_MILLIS); + protected CourseAccess( + final AsyncService asyncService, + final Environment environment) { this.quizzesRequest = asyncService.createCircuitBreaker( - 3, - Constants.MINUTE_IN_MILLIS, - Constants.MINUTE_IN_MILLIS); + environment.getProperty( + "sebserver.webservice.circuitbreaker.quizzesRequest.attempts", + Integer.class, + 3), + environment.getProperty( + "sebserver.webservice.circuitbreaker.quizzesRequest.blockingTime", + Long.class, + Constants.MINUTE_IN_MILLIS), + environment.getProperty( + "sebserver.webservice.circuitbreaker.quizzesRequest.timeToRecover", + Long.class, + Constants.MINUTE_IN_MILLIS)); this.chaptersRequest = asyncService.createCircuitBreaker( - 3, - Constants.SECOND_IN_MILLIS * 10, - Constants.MINUTE_IN_MILLIS); + environment.getProperty( + "sebserver.webservice.circuitbreaker.chaptersRequest.attempts", + Integer.class, + 3), + environment.getProperty( + "sebserver.webservice.circuitbreaker.chaptersRequest.blockingTime", + Long.class, + Constants.SECOND_IN_MILLIS * 10), + environment.getProperty( + "sebserver.webservice.circuitbreaker.chaptersRequest.timeToRecover", + Long.class, + Constants.MINUTE_IN_MILLIS)); this.accountDetailRequest = asyncService.createCircuitBreaker( - 1, - Constants.SECOND_IN_MILLIS * 10, - Constants.SECOND_IN_MILLIS * 10); + environment.getProperty( + "sebserver.webservice.circuitbreaker.accountDetailRequest.attempts", + Integer.class, + 2), + environment.getProperty( + "sebserver.webservice.circuitbreaker.accountDetailRequest.blockingTime", + Long.class, + Constants.SECOND_IN_MILLIS * 10), + environment.getProperty( + "sebserver.webservice.circuitbreaker.accountDetailRequest.timeToRecover", + Long.class, + Constants.SECOND_IN_MILLIS * 10)); } public Result>> getQuizzesFromCache(final Set ids) { return Result.tryCatch(() -> { - final List cached = this.allQuizzesRequest.getCached(); + final List cached = allQuizzesSupplier().getAllCached(); final List available = (cached != null) ? cached : quizzesSupplier(ids).get(); @@ -101,11 +127,7 @@ public abstract class CourseAccess { } public Result> getQuizzes(final FilterMap filterMap) { - if (filterMap != null) { - this.allQuizzesRequest.setSupplier(allQuizzesSupplier(filterMap)); - } - return this.allQuizzesRequest.get() - .map(LmsAPIService.quizzesFilterFunction(filterMap)); + return allQuizzesSupplier().getAll(filterMap); } public Result getExamineeAccountDetails(final String examineeSessionId) { @@ -139,8 +161,16 @@ public abstract class CourseAccess { protected abstract Supplier> quizzesSupplier(final Set ids); - protected abstract Supplier> allQuizzesSupplier(final FilterMap filterMap); + protected abstract AllQuizzesSupplier allQuizzesSupplier(); protected abstract Supplier getCourseChaptersSupplier(final String courseId); + protected abstract FetchStatus getFetchStatus(); + + protected interface AllQuizzesSupplier { + List getAllCached(); + + Result> getAll(final FilterMap filterMap); + } + } diff --git a/src/main/java/ch/ethz/seb/sebserver/webservice/servicelayer/lms/impl/edx/OpenEdxCourseAccess.java b/src/main/java/ch/ethz/seb/sebserver/webservice/servicelayer/lms/impl/edx/OpenEdxCourseAccess.java index 95b7587b..69af7ac4 100644 --- a/src/main/java/ch/ethz/seb/sebserver/webservice/servicelayer/lms/impl/edx/OpenEdxCourseAccess.java +++ b/src/main/java/ch/ethz/seb/sebserver/webservice/servicelayer/lms/impl/edx/OpenEdxCourseAccess.java @@ -20,6 +20,7 @@ import java.util.stream.Collectors; import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.springframework.core.env.Environment; import org.springframework.http.HttpEntity; import org.springframework.http.HttpHeaders; import org.springframework.http.HttpMethod; @@ -41,6 +42,8 @@ import com.fasterxml.jackson.core.type.TypeReference; import ch.ethz.seb.sebserver.gbl.Constants; import ch.ethz.seb.sebserver.gbl.api.JSONMapper; import ch.ethz.seb.sebserver.gbl.async.AsyncService; +import ch.ethz.seb.sebserver.gbl.async.CircuitBreaker.State; +import ch.ethz.seb.sebserver.gbl.async.MemoizingCircuitBreaker; import ch.ethz.seb.sebserver.gbl.model.exam.Chapters; import ch.ethz.seb.sebserver.gbl.model.exam.QuizData; import ch.ethz.seb.sebserver.gbl.model.institution.LmsSetup; @@ -70,6 +73,8 @@ final class OpenEdxCourseAccess extends CourseAccess { private final LmsSetup lmsSetup; private final OpenEdxRestTemplateFactory openEdxRestTemplateFactory; private final WebserviceInfo webserviceInfo; + private final MemoizingCircuitBreaker> allQuizzesRequest; + private final AllQuizzesSupplier allQuizzesSupplier; private OAuth2RestTemplate restTemplate; @@ -78,13 +83,51 @@ final class OpenEdxCourseAccess extends CourseAccess { final LmsSetup lmsSetup, final OpenEdxRestTemplateFactory openEdxRestTemplateFactory, final WebserviceInfo webserviceInfo, - final AsyncService asyncService) { + final AsyncService asyncService, + final Environment environment) { - super(asyncService); + super(asyncService, environment); this.jsonMapper = jsonMapper; this.lmsSetup = lmsSetup; this.openEdxRestTemplateFactory = openEdxRestTemplateFactory; this.webserviceInfo = webserviceInfo; + + this.allQuizzesRequest = asyncService.createMemoizingCircuitBreaker( + quizzesSupplier(), + environment.getProperty( + "sebserver.webservice.circuitbreaker.allQuizzesRequest.attempts", + Integer.class, + 3), + environment.getProperty( + "sebserver.webservice.circuitbreaker.allQuizzesRequest.blockingTime", + Long.class, + Constants.MINUTE_IN_MILLIS), + environment.getProperty( + "sebserver.webservice.circuitbreaker.allQuizzesRequest.timeToRecover", + Long.class, + Constants.MINUTE_IN_MILLIS), + environment.getProperty( + "sebserver.webservice.circuitbreaker.allQuizzesRequest.memoize", + Boolean.class, + true), + environment.getProperty( + "sebserver.webservice.circuitbreaker.allQuizzesRequest.memoizingTime", + Long.class, + Constants.HOUR_IN_MILLIS)); + + this.allQuizzesSupplier = new AllQuizzesSupplier() { + + @Override + public List getAllCached() { + return OpenEdxCourseAccess.this.allQuizzesRequest.getCached(); + } + + @Override + public Result> getAll(final FilterMap filterMap) { + return OpenEdxCourseAccess.this.allQuizzesRequest.get(); + } + + }; } LmsSetupTestResult initAPIAccess() { @@ -173,8 +216,7 @@ final class OpenEdxCourseAccess extends CourseAccess { .getOrThrow(); } - @Override - protected Supplier> allQuizzesSupplier(final FilterMap filterMap) { + private Supplier> quizzesSupplier() { return () -> getRestTemplate() .map(this::collectAllQuizzes) .getOrThrow(); @@ -451,4 +493,17 @@ final class OpenEdxCourseAccess extends CourseAccess { return Result.of(this.restTemplate); } + @Override + protected FetchStatus getFetchStatus() { + if (this.allQuizzesRequest.getState() != State.CLOSED) { + return FetchStatus.FETCH_ERROR; + } + return FetchStatus.ALL_FETCHED; + } + + @Override + protected AllQuizzesSupplier allQuizzesSupplier() { + return this.allQuizzesSupplier; + } + } diff --git a/src/main/java/ch/ethz/seb/sebserver/webservice/servicelayer/lms/impl/edx/OpenEdxLmsAPITemplateFactory.java b/src/main/java/ch/ethz/seb/sebserver/webservice/servicelayer/lms/impl/edx/OpenEdxLmsAPITemplateFactory.java index 8133165f..faa55804 100644 --- a/src/main/java/ch/ethz/seb/sebserver/webservice/servicelayer/lms/impl/edx/OpenEdxLmsAPITemplateFactory.java +++ b/src/main/java/ch/ethz/seb/sebserver/webservice/servicelayer/lms/impl/edx/OpenEdxLmsAPITemplateFactory.java @@ -11,6 +11,7 @@ package ch.ethz.seb.sebserver.webservice.servicelayer.lms.impl.edx; import org.apache.commons.lang3.StringUtils; import org.springframework.beans.factory.annotation.Value; import org.springframework.context.annotation.Lazy; +import org.springframework.core.env.Environment; import org.springframework.stereotype.Service; import ch.ethz.seb.sebserver.ClientHttpRequestFactoryService; @@ -33,6 +34,7 @@ public class OpenEdxLmsAPITemplateFactory { private final JSONMapper jsonMapper; private final WebserviceInfo webserviceInfo; private final AsyncService asyncService; + private final Environment environment; private final ClientCredentialService clientCredentialService; private final ClientHttpRequestFactoryService clientHttpRequestFactoryService; private final String[] alternativeTokenRequestPaths; @@ -42,6 +44,7 @@ public class OpenEdxLmsAPITemplateFactory { final JSONMapper jsonMapper, final WebserviceInfo webserviceInfo, final AsyncService asyncService, + final Environment environment, final ClientCredentialService clientCredentialService, final ClientHttpRequestFactoryService clientHttpRequestFactoryService, @Value("${sebserver.webservice.lms.openedx.api.token.request.paths}") final String alternativeTokenRequestPaths, @@ -50,6 +53,7 @@ public class OpenEdxLmsAPITemplateFactory { this.jsonMapper = jsonMapper; this.webserviceInfo = webserviceInfo; this.asyncService = asyncService; + this.environment = environment; this.clientCredentialService = clientCredentialService; this.clientHttpRequestFactoryService = clientHttpRequestFactoryService; this.alternativeTokenRequestPaths = (alternativeTokenRequestPaths != null) @@ -78,7 +82,8 @@ public class OpenEdxLmsAPITemplateFactory { lmsSetup, openEdxRestTemplateFactory, this.webserviceInfo, - this.asyncService); + this.asyncService, + this.environment); final OpenEdxCourseRestriction openEdxCourseRestriction = new OpenEdxCourseRestriction( lmsSetup, diff --git a/src/main/java/ch/ethz/seb/sebserver/webservice/servicelayer/lms/impl/moodle/MoodleCourseAccess.java b/src/main/java/ch/ethz/seb/sebserver/webservice/servicelayer/lms/impl/moodle/MoodleCourseAccess.java index d2641b83..918186b5 100644 --- a/src/main/java/ch/ethz/seb/sebserver/webservice/servicelayer/lms/impl/moodle/MoodleCourseAccess.java +++ b/src/main/java/ch/ethz/seb/sebserver/webservice/servicelayer/lms/impl/moodle/MoodleCourseAccess.java @@ -23,6 +23,7 @@ import org.apache.commons.lang3.StringUtils; import org.joda.time.DateTime; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.springframework.core.env.Environment; import org.springframework.util.LinkedMultiValueMap; import org.springframework.util.MultiValueMap; @@ -34,6 +35,7 @@ import com.fasterxml.jackson.core.type.TypeReference; import ch.ethz.seb.sebserver.gbl.Constants; import ch.ethz.seb.sebserver.gbl.api.JSONMapper; import ch.ethz.seb.sebserver.gbl.async.AsyncService; +import ch.ethz.seb.sebserver.gbl.async.CircuitBreaker; import ch.ethz.seb.sebserver.gbl.model.exam.Chapters; import ch.ethz.seb.sebserver.gbl.model.exam.QuizData; import ch.ethz.seb.sebserver.gbl.model.institution.LmsSetup; @@ -43,6 +45,7 @@ import ch.ethz.seb.sebserver.gbl.util.Result; import ch.ethz.seb.sebserver.gbl.util.Utils; import ch.ethz.seb.sebserver.webservice.servicelayer.dao.FilterMap; import ch.ethz.seb.sebserver.webservice.servicelayer.lms.impl.CourseAccess; +import ch.ethz.seb.sebserver.webservice.servicelayer.lms.impl.moodle.MoodleCourseDataAsyncLoader.CourseDataShort; import ch.ethz.seb.sebserver.webservice.servicelayer.lms.impl.moodle.MoodleRestTemplateFactory.MoodleAPIRestTemplate; /** Implements the LmsAPITemplate for Open edX LMS Course API access. @@ -73,7 +76,9 @@ public class MoodleCourseAccess extends CourseAccess { private final JSONMapper jsonMapper; private final LmsSetup lmsSetup; private final MoodleRestTemplateFactory moodleRestTemplateFactory; - private final MoodleCourseDataLazyLoader moodleCourseDataLazyLoader; + private final MoodleCourseDataAsyncLoader moodleCourseDataAsyncLoader; + private final CircuitBreaker> allQuizzesRequest; + private final AllQuizzesSupplier allQuizzesSupplier; private MoodleAPIRestTemplate restTemplate; @@ -81,14 +86,42 @@ public class MoodleCourseAccess extends CourseAccess { final JSONMapper jsonMapper, final LmsSetup lmsSetup, final MoodleRestTemplateFactory moodleRestTemplateFactory, - final MoodleCourseDataLazyLoader moodleCourseDataLazyLoader, - final AsyncService asyncService) { + final MoodleCourseDataAsyncLoader moodleCourseDataAsyncLoader, + final AsyncService asyncService, + final Environment environment) { - super(asyncService); + super(asyncService, environment); this.jsonMapper = jsonMapper; this.lmsSetup = lmsSetup; - this.moodleCourseDataLazyLoader = moodleCourseDataLazyLoader; + this.moodleCourseDataAsyncLoader = moodleCourseDataAsyncLoader; this.moodleRestTemplateFactory = moodleRestTemplateFactory; + + this.allQuizzesRequest = asyncService.createCircuitBreaker( + environment.getProperty( + "sebserver.webservice.circuitbreaker.allQuizzesRequest.attempts", + Integer.class, + 3), + environment.getProperty( + "sebserver.webservice.circuitbreaker.allQuizzesRequest.blockingTime", + Long.class, + Constants.MINUTE_IN_MILLIS), + environment.getProperty( + "sebserver.webservice.circuitbreaker.allQuizzesRequest.timeToRecover", + Long.class, + Constants.MINUTE_IN_MILLIS)); + + this.allQuizzesSupplier = new AllQuizzesSupplier() { + + @Override + public List getAllCached() { + return getCached(); + } + + @Override + public Result> getAll(final FilterMap filterMap) { + return MoodleCourseAccess.this.allQuizzesRequest.protectedRun(allQuizzesSupplier(filterMap)); + } + }; } @Override @@ -175,7 +208,6 @@ public class MoodleCourseAccess extends CourseAccess { } - @Override protected Supplier> allQuizzesSupplier(final FilterMap filterMap) { return () -> getRestTemplate() .map(template -> collectAllQuizzes(template, filterMap)) @@ -187,6 +219,20 @@ public class MoodleCourseAccess extends CourseAccess { throw new UnsupportedOperationException("not available yet"); } + @Override + protected FetchStatus getFetchStatus() { + if (this.moodleCourseDataAsyncLoader.isRunning()) { + return FetchStatus.ASYNC_FETCH_RUNNING; + } + + return FetchStatus.ALL_FETCHED; + } + + @Override + protected AllQuizzesSupplier allQuizzesSupplier() { + return this.allQuizzesSupplier; + } + private List collectAllQuizzes( final MoodleAPIRestTemplate restTemplate, final FilterMap filterMap) { @@ -198,42 +244,42 @@ public class MoodleCourseAccess extends CourseAccess { final DateTime quizFromTime = (filterMap != null) ? filterMap.getQuizFromTime() : null; final long fromCutTime = (quizFromTime != null) ? Utils.toUnixTimeInSeconds(quizFromTime) : -1; - Collection courseQuizData = Collections.emptyList(); - if (this.moodleCourseDataLazyLoader.isRunning()) { - courseQuizData = this.moodleCourseDataLazyLoader.getPreFilteredCourseIds(); - } else if (this.moodleCourseDataLazyLoader.getLastRunTime() <= 0) { + Collection courseQuizData = Collections.emptyList(); + if (this.moodleCourseDataAsyncLoader.isRunning()) { + courseQuizData = this.moodleCourseDataAsyncLoader.getCachedCourseData(); + } else if (this.moodleCourseDataAsyncLoader.getLastRunTime() <= 0) { // set cut time if available if (fromCutTime >= 0) { - this.moodleCourseDataLazyLoader.setFromCutTime(fromCutTime); + this.moodleCourseDataAsyncLoader.setFromCutTime(fromCutTime); } // first run async and wait some time, get what is there - this.moodleCourseDataLazyLoader.loadAsync(restTemplate); + this.moodleCourseDataAsyncLoader.loadAsync(restTemplate); try { Thread.sleep(INITIAL_WAIT_TIME); - courseQuizData = this.moodleCourseDataLazyLoader.getPreFilteredCourseIds(); + courseQuizData = this.moodleCourseDataAsyncLoader.getCachedCourseData(); } catch (final Exception e) { log.error("Failed to wait for first load run: ", e); return Collections.emptyList(); } - } else if (this.moodleCourseDataLazyLoader.isLongRunningTask()) { + } else if (this.moodleCourseDataAsyncLoader.isLongRunningTask()) { // on long running tasks if we have a different fromCutTime as before // kick off the lazy loading task immediately with the new time filter - if (fromCutTime > 0 && fromCutTime != this.moodleCourseDataLazyLoader.getFromCutTime()) { - this.moodleCourseDataLazyLoader.setFromCutTime(fromCutTime); - this.moodleCourseDataLazyLoader.loadAsync(restTemplate); + if (fromCutTime > 0 && fromCutTime != this.moodleCourseDataAsyncLoader.getFromCutTime()) { + this.moodleCourseDataAsyncLoader.setFromCutTime(fromCutTime); + this.moodleCourseDataAsyncLoader.loadAsync(restTemplate); // otherwise kick off only if the last fetch task was then minutes ago - } else if (Utils.getMillisecondsNow() - this.moodleCourseDataLazyLoader.getLastRunTime() > 10 + } else if (Utils.getMillisecondsNow() - this.moodleCourseDataAsyncLoader.getLastRunTime() > 10 * Constants.MINUTE_IN_MILLIS) { - this.moodleCourseDataLazyLoader.loadAsync(restTemplate); + this.moodleCourseDataAsyncLoader.loadAsync(restTemplate); } - courseQuizData = this.moodleCourseDataLazyLoader.getPreFilteredCourseIds(); + courseQuizData = this.moodleCourseDataAsyncLoader.getCachedCourseData(); } else { // just run the task in sync if (fromCutTime >= 0) { - this.moodleCourseDataLazyLoader.setFromCutTime(fromCutTime); + this.moodleCourseDataAsyncLoader.setFromCutTime(fromCutTime); } - this.moodleCourseDataLazyLoader.loadSync(restTemplate); - courseQuizData = this.moodleCourseDataLazyLoader.getPreFilteredCourseIds(); + this.moodleCourseDataAsyncLoader.loadSync(restTemplate); + courseQuizData = this.moodleCourseDataAsyncLoader.getCachedCourseData(); } if (courseQuizData.isEmpty()) { @@ -257,6 +303,29 @@ public class MoodleCourseAccess extends CourseAccess { }); } + private List getCached() { + final Collection courseQuizData = this.moodleCourseDataAsyncLoader.getCachedCourseData(); + + final String urlPrefix = (this.lmsSetup.lmsApiUrl.endsWith(Constants.URL_PATH_SEPARATOR)) + ? this.lmsSetup.lmsApiUrl + MOODLE_QUIZ_START_URL_PATH + : this.lmsSetup.lmsApiUrl + Constants.URL_PATH_SEPARATOR + MOODLE_QUIZ_START_URL_PATH; + return courseQuizData + .stream() + .reduce( + new ArrayList<>(), + (list, courseData) -> { + list.addAll(quizDataOf( + this.lmsSetup, + courseData, + urlPrefix)); + return list; + }, + (list1, list2) -> { + list1.addAll(list2); + return list1; + }); + } + private List getQuizzesForIds( final MoodleAPIRestTemplate restTemplate, final Set quizIds) { @@ -405,6 +474,46 @@ public class MoodleCourseAccess extends CourseAccess { return courseAndQuiz; } + private List quizDataOf( + final LmsSetup lmsSetup, + final CourseDataShort courseData, + final String uriPrefix) { + + additionalAttrs.clear(); + additionalAttrs.put(QuizData.ATTR_ADDITIONAL_CREATION_TIME, String.valueOf(courseData.time_created)); + additionalAttrs.put(QuizData.ATTR_ADDITIONAL_SHORT_NAME, courseData.short_name); + additionalAttrs.put(QuizData.ATTR_ADDITIONAL_ID_NUMBER, courseData.idnumber); + + final List courseAndQuiz = courseData.quizzes + .stream() + .map(courseQuizData -> { + final String startURI = uriPrefix + courseQuizData.course_module; + //additionalAttrs.put(QuizData.ATTR_ADDITIONAL_TIME_LIMIT, String.valueOf(courseQuizData.time_limit)); + return new QuizData( + getInternalQuizId( + courseQuizData.course_module, + courseData.id, + courseData.short_name, + courseData.idnumber), + lmsSetup.getInstitutionId(), + lmsSetup.id, + lmsSetup.getLmsType(), + courseQuizData.name, + Constants.EMPTY_NOTE, + (courseQuizData.time_open != null && courseQuizData.time_open > 0) + ? Utils.toDateTimeUTCUnix(courseQuizData.time_open) + : Utils.toDateTimeUTCUnix(courseData.start_date), + (courseQuizData.time_close != null && courseQuizData.time_close > 0) + ? Utils.toDateTimeUTCUnix(courseQuizData.time_close) + : Utils.toDateTimeUTCUnix(courseData.end_date), + startURI, + additionalAttrs); + }) + .collect(Collectors.toList()); + + return courseAndQuiz; + } + private Result getRestTemplate() { if (this.restTemplate == null) { final Result templateRequest = this.moodleRestTemplateFactory @@ -482,7 +591,7 @@ public class MoodleCourseAccess extends CourseAccess { /** Maps the Moodle course API course data */ @JsonIgnoreProperties(ignoreUnknown = true) - static final class CourseData { + private static final class CourseData { final String id; final String short_name; final String idnumber; @@ -516,35 +625,10 @@ public class MoodleCourseAccess extends CourseAccess { this.end_date = end_date; this.time_created = time_created; } - - @Override - public int hashCode() { - final int prime = 31; - int result = 1; - result = prime * result + ((this.id == null) ? 0 : this.id.hashCode()); - return result; - } - - @Override - public boolean equals(final Object obj) { - if (this == obj) - return true; - if (obj == null) - return false; - if (getClass() != obj.getClass()) - return false; - final CourseData other = (CourseData) obj; - if (this.id == null) { - if (other.id != null) - return false; - } else if (!this.id.equals(other.id)) - return false; - return true; - } } @JsonIgnoreProperties(ignoreUnknown = true) - static final class Courses { + private static final class Courses { final Collection courses; @JsonCreator @@ -555,7 +639,7 @@ public class MoodleCourseAccess extends CourseAccess { } @JsonIgnoreProperties(ignoreUnknown = true) - static final class CourseQuizData { + private static final class CourseQuizData { final Collection quizzes; @JsonCreator diff --git a/src/main/java/ch/ethz/seb/sebserver/webservice/servicelayer/lms/impl/moodle/MoodleCourseDataLazyLoader.java b/src/main/java/ch/ethz/seb/sebserver/webservice/servicelayer/lms/impl/moodle/MoodleCourseDataAsyncLoader.java similarity index 51% rename from src/main/java/ch/ethz/seb/sebserver/webservice/servicelayer/lms/impl/moodle/MoodleCourseDataLazyLoader.java rename to src/main/java/ch/ethz/seb/sebserver/webservice/servicelayer/lms/impl/moodle/MoodleCourseDataAsyncLoader.java index 89474626..c264befc 100644 --- a/src/main/java/ch/ethz/seb/sebserver/webservice/servicelayer/lms/impl/moodle/MoodleCourseDataLazyLoader.java +++ b/src/main/java/ch/ethz/seb/sebserver/webservice/servicelayer/lms/impl/moodle/MoodleCourseDataAsyncLoader.java @@ -28,8 +28,10 @@ import org.slf4j.LoggerFactory; import org.springframework.beans.factory.config.ConfigurableBeanFactory; import org.springframework.context.annotation.Lazy; import org.springframework.context.annotation.Scope; +import org.springframework.core.env.Environment; import org.springframework.stereotype.Component; import org.springframework.util.LinkedMultiValueMap; +import org.springframework.util.MultiValueMap; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; @@ -40,40 +42,72 @@ import com.fasterxml.jackson.databind.JsonMappingException; import ch.ethz.seb.sebserver.gbl.Constants; import ch.ethz.seb.sebserver.gbl.api.JSONMapper; import ch.ethz.seb.sebserver.gbl.async.AsyncRunner; +import ch.ethz.seb.sebserver.gbl.async.AsyncService; +import ch.ethz.seb.sebserver.gbl.async.CircuitBreaker; import ch.ethz.seb.sebserver.gbl.profile.WebServiceProfile; import ch.ethz.seb.sebserver.gbl.util.Utils; -import ch.ethz.seb.sebserver.webservice.servicelayer.lms.impl.moodle.MoodleCourseAccess.CourseData; -import ch.ethz.seb.sebserver.webservice.servicelayer.lms.impl.moodle.MoodleCourseAccess.CourseQuiz; -import ch.ethz.seb.sebserver.webservice.servicelayer.lms.impl.moodle.MoodleCourseAccess.CourseQuizData; -import ch.ethz.seb.sebserver.webservice.servicelayer.lms.impl.moodle.MoodleCourseAccess.Courses; import ch.ethz.seb.sebserver.webservice.servicelayer.lms.impl.moodle.MoodleRestTemplateFactory.MoodleAPIRestTemplate; @Lazy @Component @WebServiceProfile @Scope(ConfigurableBeanFactory.SCOPE_PROTOTYPE) -public class MoodleCourseDataLazyLoader { +public class MoodleCourseDataAsyncLoader { - private static final Logger log = LoggerFactory.getLogger(MoodleCourseDataLazyLoader.class); + private static final Logger log = LoggerFactory.getLogger(MoodleCourseDataAsyncLoader.class); private final JSONMapper jsonMapper; private final AsyncRunner asyncRunner; + private final CircuitBreaker moodleRestCall; + private final int maxSize; + private final int pageSize; - private final Set preFilteredCourseIds = new HashSet<>(); + private final Set cachedCourseData = new HashSet<>(); + private String lmsSetup = Constants.EMPTY_NOTE; private long lastRunTime = 0; private long lastLoadTime = 0; private boolean running = false; private long fromCutTime; - public MoodleCourseDataLazyLoader( + public MoodleCourseDataAsyncLoader( final JSONMapper jsonMapper, - final AsyncRunner asyncRunner) { + final AsyncService asyncService, + final AsyncRunner asyncRunner, + final Environment environment) { this.jsonMapper = jsonMapper; - this.asyncRunner = asyncRunner; this.fromCutTime = Utils.toUnixTimeInSeconds(DateTime.now(DateTimeZone.UTC).minusYears(3)); + this.asyncRunner = asyncRunner; + + this.moodleRestCall = asyncService.createCircuitBreaker( + environment.getProperty( + "sebserver.webservice.circuitbreaker.moodleRestCall.attempts", + Integer.class, + 2), + environment.getProperty( + "sebserver.webservice.circuitbreaker.moodleRestCall.blockingTime", + Long.class, + Constants.SECOND_IN_MILLIS * 20), + environment.getProperty( + "sebserver.webservice.circuitbreaker.moodleRestCall.timeToRecover", + Long.class, + Constants.MINUTE_IN_MILLIS)); + + this.maxSize = + environment.getProperty("sebserver.webservice.cache.moodle.course.maxSize", Integer.class, 10000); + this.pageSize = + environment.getProperty("sebserver.webservice.cache.moodle.course.pageSize", Integer.class, 500); + } + + public void init(final String lmsSetupName) { + if (Constants.EMPTY_NOTE.equals(this.lmsSetup)) { + this.lmsSetup = lmsSetupName; + } else { + throw new IllegalStateException( + "Invalid initialization of MoodleCourseDataAsyncLoader. It has already been initialized yet"); + } } public long getFromCutTime() { @@ -84,8 +118,8 @@ public class MoodleCourseDataLazyLoader { this.fromCutTime = fromCutTime; } - public Set getPreFilteredCourseIds() { - return this.preFilteredCourseIds; + public Set getCachedCourseData() { + return new HashSet<>(this.cachedCourseData); } public long getLastRunTime() { @@ -100,7 +134,7 @@ public class MoodleCourseDataLazyLoader { return this.lastLoadTime > 30 * Constants.SECOND_IN_MILLIS; } - public Set loadSync(final MoodleAPIRestTemplate restTemplate) { + public Set loadSync(final MoodleAPIRestTemplate restTemplate) { if (this.running) { throw new IllegalStateException("Is already running asynchronously"); } @@ -109,9 +143,11 @@ public class MoodleCourseDataLazyLoader { loadAndCache(restTemplate).run(); this.lastRunTime = Utils.getMillisecondsNow(); - log.info("Loaded {} courses synchronously", this.preFilteredCourseIds.size()); + log.info("LMS Setup: {} loaded {} courses synchronously", + this.lmsSetup, + this.cachedCourseData.size()); - return this.preFilteredCourseIds; + return this.cachedCourseData; } public void loadAsync(final MoodleAPIRestTemplate restTemplate) { @@ -126,6 +162,7 @@ public class MoodleCourseDataLazyLoader { private Runnable loadAndCache(final MoodleAPIRestTemplate restTemplate) { return () -> { + this.cachedCourseData.clear(); final long startTime = Utils.getMillisecondsNow(); loadAllQuizzes(restTemplate); @@ -133,7 +170,9 @@ public class MoodleCourseDataLazyLoader { this.lastLoadTime = Utils.getMillisecondsNow() - startTime; this.running = false; - log.info("Loaded {} courses asynchronously", this.preFilteredCourseIds.size()); + log.info("LMS Setup: {} loaded {} courses asynchronously", + this.lmsSetup, + this.cachedCourseData.size()); }; } @@ -151,8 +190,8 @@ public class MoodleCourseDataLazyLoader { try { // first get courses from Moodle for page - final Map courseData = new HashMap<>(); - final Collection coursesPage = getCoursesPage(restTemplate, page, 1000); + final Map courseData = new HashMap<>(); + final Collection coursesPage = getCoursesPage(restTemplate, page, this.pageSize); if (coursesPage == null || coursesPage.isEmpty()) { return false; @@ -168,7 +207,8 @@ public class MoodleCourseDataLazyLoader { MoodleCourseAccess.MOODLE_COURSE_API_COURSE_IDS, new ArrayList<>(courseData.keySet())); - final String quizzesJSON = restTemplate.callMoodleAPIFunction( + final String quizzesJSON = callMoodleRestAPI( + restTemplate, MoodleCourseAccess.MOODLE_QUIZ_API_FUNCTION_NAME, attributes); @@ -185,28 +225,36 @@ public class MoodleCourseDataLazyLoader { .stream() .filter(getQuizFilter()) .forEach(quiz -> { - final CourseData data = courseData.get(quiz.course); + final CourseDataShort data = courseData.get(quiz.course); if (data != null) { data.quizzes.add(quiz); } }); - this.preFilteredCourseIds.addAll( - courseData.values().stream() - .filter(c -> !c.quizzes.isEmpty()) - .collect(Collectors.toList())); + courseData.values().stream() + .filter(c -> !c.quizzes.isEmpty()) + .forEach(c -> { + if (this.cachedCourseData.size() >= this.maxSize) { + log.error( + "LMS Setup: {} Cache is full and has reached its maximal size. Skip data: -> {}", + this.lmsSetup, + c); + } else { + this.cachedCourseData.add(c); + } + }); return true; } else { return false; } } catch (final Exception e) { - log.error("Unexpected exception while trying to get course data: ", e); + log.error("LMS Setup: {} Unexpected exception while trying to get course data: ", this.lmsSetup, e); return false; } } - private Collection getCoursesPage( + private Collection getCoursesPage( final MoodleAPIRestTemplate restTemplate, final int page, final int size) throws JsonParseException, JsonMappingException, IOException { @@ -219,7 +267,8 @@ public class MoodleCourseDataLazyLoader { attributes.add(MoodleCourseAccess.MOODLE_COURSE_API_SEARCH_PAGE, String.valueOf(page)); attributes.add(MoodleCourseAccess.MOODLE_COURSE_API_SEARCH_PAGE_SIZE, String.valueOf(size)); - final String courseKeyPageJSON = restTemplate.callMoodleAPIFunction( + final String courseKeyPageJSON = callMoodleRestAPI( + restTemplate, MoodleCourseAccess.MOODLE_COURSE_SEARCH_API_FUNCTION_NAME, attributes); @@ -228,7 +277,9 @@ public class MoodleCourseDataLazyLoader { CoursePage.class); if (keysPage == null || keysPage.courseKeys == null || keysPage.courseKeys.isEmpty()) { - log.info("No courses found on page: {}", page); + if (log.isDebugEnabled()) { + log.debug("LMS Setup: {} No courses found on page: {}", this.lmsSetup, page); + } return Collections.emptyList(); } @@ -238,7 +289,7 @@ public class MoodleCourseDataLazyLoader { .map(key -> key.id) .collect(Collectors.toSet()); - final Collection result = getCoursesForIds(restTemplate, ids) + final Collection result = getCoursesForIds(restTemplate, ids) .stream() .filter(getCourseFilter()) .collect(Collectors.toList()); @@ -247,19 +298,19 @@ public class MoodleCourseDataLazyLoader { return result; } catch (final Exception e) { - log.error("Unexpected error while trying to get courses page: ", e); + log.error("LMS Setup: {} Unexpected error while trying to get courses page: ", this.lmsSetup, e); return Collections.emptyList(); } } - private Collection getCoursesForIds( + private Collection getCoursesForIds( final MoodleAPIRestTemplate restTemplate, final Set ids) { try { if (log.isDebugEnabled()) { - log.debug("Get courses for ids: {}", ids); + log.debug("LMS Setup: {} Get courses for ids: {}", this.lmsSetup, ids); } final String joinedIds = StringUtils.join(ids, Constants.COMMA); @@ -267,32 +318,52 @@ public class MoodleCourseDataLazyLoader { final LinkedMultiValueMap attributes = new LinkedMultiValueMap<>(); attributes.add(MoodleCourseAccess.MOODLE_COURSE_API_FIELD_NAME, MoodleCourseAccess.MOODLE_COURSE_API_IDS); attributes.add(MoodleCourseAccess.MOODLE_COURSE_API_FIELD_VALUE, joinedIds); - final String coursePageJSON = restTemplate.callMoodleAPIFunction( + final String coursePageJSON = callMoodleRestAPI( + restTemplate, MoodleCourseAccess.MOODLE_COURSE_BY_FIELD_API_FUNCTION_NAME, attributes); - return this.jsonMapper. readValue( + return this.jsonMapper.readValue( coursePageJSON, Courses.class).courses; } catch (final Exception e) { - log.error("Unexpected error while trying to get courses for ids", e); + log.error("LMS Setup: {} Unexpected error while trying to get courses for ids", this.lmsSetup, e); return Collections.emptyList(); } } - private Predicate getQuizFilter() { + private String callMoodleRestAPI( + final MoodleAPIRestTemplate restTemplate, + final String function, + final MultiValueMap queryAttributes) { + + return this.moodleRestCall + .protectedRun(() -> restTemplate.callMoodleAPIFunction( + function, + queryAttributes)) + .getOrThrow(); + + } + + private Predicate getQuizFilter() { final long now = Utils.getSecondsNow(); return quiz -> { if (quiz.time_close == null || quiz.time_close == 0 || quiz.time_close > now) { return true; } - log.info("remove quiz {} end_time {} now {}", quiz.name, quiz.time_close, now); + if (log.isDebugEnabled()) { + log.debug("LMS Setup: {} remove quiz {} end_time {} now {}", + this.lmsSetup, + quiz.name, + quiz.time_close, + now); + } return false; }; } - private Predicate getCourseFilter() { + private Predicate getCourseFilter() { final long now = Utils.getSecondsNow(); return course -> { if (course.start_date < this.fromCutTime) { @@ -303,7 +374,13 @@ public class MoodleCourseDataLazyLoader { return true; } - log.info("remove course {} end_time {} now {}", course.short_name, course.end_date, now); + if (log.isDebugEnabled()) { + log.info("LMS Setup: {} remove course {} end_time {} now {}", + this.lmsSetup, + course.short_name, + course.end_date, + now); + } return false; }; } @@ -356,77 +433,107 @@ public class MoodleCourseDataLazyLoader { } -// @JsonIgnoreProperties(ignoreUnknown = true) -// static final class CourseKeys { -// final Collection courses; -// -// @JsonCreator -// protected CourseKeys( -// @JsonProperty(value = "courses") final Collection courses) { -// this.courses = courses; -// } -// } + /** Maps the Moodle course API course data */ + @JsonIgnoreProperties(ignoreUnknown = true) + static final class CourseDataShort { + final String id; + final String short_name; + final String idnumber; + final Long start_date; // unix-time seconds UTC + final Long end_date; // unix-time seconds UTC + final Long time_created; // unix-time seconds UTC + final Collection quizzes = new ArrayList<>(); -// /** Maps the Moodle course API course data */ -// @JsonIgnoreProperties(ignoreUnknown = true) -// static final class CourseDataKey { -// final String id; -// final String short_name; -// final Long start_date; // unix-time seconds UTC -// final Long end_date; // unix-time seconds UTC -// final Long time_created; // unix-time seconds UTC -// final Collection quizzes = new ArrayList<>(); -// -// @JsonCreator -// protected CourseDataKey( -// @JsonProperty(value = "id") final String id, -// @JsonProperty(value = "shortname") final String short_name, -// @JsonProperty(value = "startdate") final Long start_date, -// @JsonProperty(value = "enddate") final Long end_date, -// @JsonProperty(value = "timecreated") final Long time_created) { -// -// this.id = id; -// this.short_name = short_name; -// this.start_date = start_date; -// this.end_date = end_date; -// this.time_created = time_created; -// } -// -// } + @JsonCreator + protected CourseDataShort( + @JsonProperty(value = "id") final String id, + @JsonProperty(value = "shortname") final String short_name, + @JsonProperty(value = "idnumber") final String idnumber, + @JsonProperty(value = "startdate") final Long start_date, + @JsonProperty(value = "enddate") final Long end_date, + @JsonProperty(value = "timecreated") final Long time_created) { -// @JsonIgnoreProperties(ignoreUnknown = true) -// static final class CourseQuizKeys { -// final Collection quizzes; -// -// @JsonCreator -// protected CourseQuizKeys( -// @JsonProperty(value = "quizzes") final Collection quizzes) { -// this.quizzes = quizzes; -// } -// } -// -// @JsonIgnoreProperties(ignoreUnknown = true) -// static final class CourseQuizKey { -// final String id; -// final String course; -// final String name; -// final Long time_open; // unix-time seconds UTC -// final Long time_close; // unix-time seconds UTC -// -// @JsonCreator -// protected CourseQuizKey( -// @JsonProperty(value = "id") final String id, -// @JsonProperty(value = "course") final String course, -// @JsonProperty(value = "name") final String name, -// @JsonProperty(value = "timeopen") final Long time_open, -// @JsonProperty(value = "timeclose") final Long time_close) { -// -// this.id = id; -// this.course = course; -// this.name = name; -// this.time_open = time_open; -// this.time_close = time_close; -// } -// } + this.id = id; + this.short_name = short_name; + this.idnumber = idnumber; + this.start_date = start_date; + this.end_date = end_date; + this.time_created = time_created; + } + + @Override + public int hashCode() { + final int prime = 31; + int result = 1; + result = prime * result + ((this.id == null) ? 0 : this.id.hashCode()); + return result; + } + + @Override + public boolean equals(final Object obj) { + if (this == obj) + return true; + if (obj == null) + return false; + if (getClass() != obj.getClass()) + return false; + final CourseDataShort other = (CourseDataShort) obj; + if (this.id == null) { + if (other.id != null) + return false; + } else if (!this.id.equals(other.id)) + return false; + return true; + } + } + + @JsonIgnoreProperties(ignoreUnknown = true) + private static final class Courses { + final Collection courses; + + @JsonCreator + protected Courses( + @JsonProperty(value = "courses") final Collection courses) { + this.courses = courses; + } + } + + @JsonIgnoreProperties(ignoreUnknown = true) + static final class CourseQuizData { + final Collection quizzes; + + @JsonCreator + protected CourseQuizData( + @JsonProperty(value = "quizzes") final Collection quizzes) { + this.quizzes = quizzes; + } + } + + @JsonIgnoreProperties(ignoreUnknown = true) + static final class CourseQuizShort { + final String id; + final String course; + final String course_module; + final String name; + final Long time_open; // unix-time seconds UTC + final Long time_close; // unix-time seconds UTC + + @JsonCreator + protected CourseQuizShort( + @JsonProperty(value = "id") final String id, + @JsonProperty(value = "course") final String course, + @JsonProperty(value = "coursemodule") final String course_module, + @JsonProperty(value = "name") final String name, + @JsonProperty(value = "timeopen") final Long time_open, + @JsonProperty(value = "timeclose") final Long time_close) { + + this.id = id; + this.course = course; + this.course_module = course_module; + this.name = name; + this.time_open = time_open; + this.time_close = time_close; + } + } } diff --git a/src/main/java/ch/ethz/seb/sebserver/webservice/servicelayer/lms/impl/moodle/MoodleLmsAPITemplateFactory.java b/src/main/java/ch/ethz/seb/sebserver/webservice/servicelayer/lms/impl/moodle/MoodleLmsAPITemplateFactory.java index 0afdabb0..f49523e7 100644 --- a/src/main/java/ch/ethz/seb/sebserver/webservice/servicelayer/lms/impl/moodle/MoodleLmsAPITemplateFactory.java +++ b/src/main/java/ch/ethz/seb/sebserver/webservice/servicelayer/lms/impl/moodle/MoodleLmsAPITemplateFactory.java @@ -12,6 +12,7 @@ import org.apache.commons.lang3.StringUtils; import org.springframework.beans.factory.annotation.Value; import org.springframework.context.ApplicationContext; import org.springframework.context.annotation.Lazy; +import org.springframework.core.env.Environment; import org.springframework.stereotype.Service; import ch.ethz.seb.sebserver.ClientHttpRequestFactoryService; @@ -32,6 +33,7 @@ public class MoodleLmsAPITemplateFactory { private final JSONMapper jsonMapper; private final AsyncService asyncService; + private final Environment environment; private final ClientCredentialService clientCredentialService; private final ClientHttpRequestFactoryService clientHttpRequestFactoryService; private final ApplicationContext applicationContext; @@ -40,6 +42,7 @@ public class MoodleLmsAPITemplateFactory { protected MoodleLmsAPITemplateFactory( final JSONMapper jsonMapper, final AsyncService asyncService, + final Environment environment, final ClientCredentialService clientCredentialService, final ClientHttpRequestFactoryService clientHttpRequestFactoryService, final ApplicationContext applicationContext, @@ -47,6 +50,7 @@ public class MoodleLmsAPITemplateFactory { this.jsonMapper = jsonMapper; this.asyncService = asyncService; + this.environment = environment; this.clientCredentialService = clientCredentialService; this.clientHttpRequestFactoryService = clientHttpRequestFactoryService; this.applicationContext = applicationContext; @@ -62,8 +66,9 @@ public class MoodleLmsAPITemplateFactory { return Result.tryCatch(() -> { - final MoodleCourseDataLazyLoader lazyLoaderPrototype = - this.applicationContext.getBean(MoodleCourseDataLazyLoader.class); + final MoodleCourseDataAsyncLoader asyncLoaderPrototype = + this.applicationContext.getBean(MoodleCourseDataAsyncLoader.class); + asyncLoaderPrototype.init(lmsSetup.name); final MoodleRestTemplateFactory moodleRestTemplateFactory = new MoodleRestTemplateFactory( this.jsonMapper, @@ -78,8 +83,9 @@ public class MoodleLmsAPITemplateFactory { this.jsonMapper, lmsSetup, moodleRestTemplateFactory, - lazyLoaderPrototype, - this.asyncService); + asyncLoaderPrototype, + this.asyncService, + this.environment); final MoodleCourseRestriction moodleCourseRestriction = new MoodleCourseRestriction( this.jsonMapper, diff --git a/src/main/java/ch/ethz/seb/sebserver/webservice/servicelayer/session/impl/SEBClientConnectionServiceImpl.java b/src/main/java/ch/ethz/seb/sebserver/webservice/servicelayer/session/impl/SEBClientConnectionServiceImpl.java index 3c525551..004fc8bd 100644 --- a/src/main/java/ch/ethz/seb/sebserver/webservice/servicelayer/session/impl/SEBClientConnectionServiceImpl.java +++ b/src/main/java/ch/ethz/seb/sebserver/webservice/servicelayer/session/impl/SEBClientConnectionServiceImpl.java @@ -489,9 +489,7 @@ public class SEBClientConnectionServiceImpl implements SEBClientConnectionServic .filter(Objects::nonNull) .filter(connection -> connection.pingIndicator != null && connection.clientConnection.status.establishedStatus) - .map(connection -> connection.pingIndicator.updateLogEvent()) - .filter(Objects::nonNull) - .forEach(this.eventHandlingStrategy); + .forEach(connection -> connection.pingIndicator.updateLogEvent()); } catch (final Exception e) { log.error("Failed to update ping events: ", e); diff --git a/src/main/java/ch/ethz/seb/sebserver/webservice/servicelayer/session/impl/SEBClientNotificationServiceImpl.java b/src/main/java/ch/ethz/seb/sebserver/webservice/servicelayer/session/impl/SEBClientNotificationServiceImpl.java index 19254f92..93059772 100644 --- a/src/main/java/ch/ethz/seb/sebserver/webservice/servicelayer/session/impl/SEBClientNotificationServiceImpl.java +++ b/src/main/java/ch/ethz/seb/sebserver/webservice/servicelayer/session/impl/SEBClientNotificationServiceImpl.java @@ -42,7 +42,7 @@ public class SEBClientNotificationServiceImpl implements SEBClientNotificationSe private final ClientEventDAO clientEventDAO; private final SEBClientInstructionService sebClientInstructionService; - private final Set pendingNotifications; + private final Set pendingNotifications = new HashSet<>(); public SEBClientNotificationServiceImpl( final ClientEventDAO clientEventDAO, @@ -50,19 +50,25 @@ public class SEBClientNotificationServiceImpl implements SEBClientNotificationSe this.clientEventDAO = clientEventDAO; this.sebClientInstructionService = sebClientInstructionService; - this.pendingNotifications = new HashSet<>(); } @Override public Boolean hasAnyPendingNotification(final Long clientConnectionId) { - if (this.pendingNotifications.contains(clientConnectionId)) { + + if (this.pendingNotifications.add(clientConnectionId)) { return true; } final boolean hasAnyPendingNotification = !getPendingNotifications(clientConnectionId) .getOr(Collections.emptyList()) .isEmpty(); + if (hasAnyPendingNotification) { + // NOTE this is a quick and dirty way to keep cache pendingNotifications cache size short. + // TODO find a better way to do this. + if (this.pendingNotifications.size() > 100) { + this.pendingNotifications.clear(); + } this.pendingNotifications.add(clientConnectionId); } diff --git a/src/main/resources/config/application-dev-ws.properties b/src/main/resources/config/application-dev-ws.properties index 50dd97d5..3966d495 100644 --- a/src/main/resources/config/application-dev-ws.properties +++ b/src/main/resources/config/application-dev-ws.properties @@ -1,8 +1,6 @@ server.address=localhost server.port=8090 -#logging.file=log/sebserver.log - # data source configuration spring.datasource.initialize=true spring.datasource.initialization-mode=always diff --git a/src/main/resources/config/application-dev.properties b/src/main/resources/config/application-dev.properties index 507d76bf..5ec2e5a1 100644 --- a/src/main/resources/config/application-dev.properties +++ b/src/main/resources/config/application-dev.properties @@ -6,6 +6,8 @@ server.servlet.context-path=/ server.tomcat.uri-encoding=UTF-8 logging.level.ch=INFO +logging.level.org.springframework.cache=INFO +logging.level.ch.ethz.seb.sebserver.webservice.servicelayer.lms.impl=DEBUG sebserver.http.client.connect-timeout=150000 sebserver.http.client.connection-request-timeout=100000 diff --git a/src/main/resources/config/application-ws.properties b/src/main/resources/config/application-ws.properties index 559f1061..efa28906 100644 --- a/src/main/resources/config/application-ws.properties +++ b/src/main/resources/config/application-ws.properties @@ -8,6 +8,10 @@ sebserver.init.adminaccount.gen-on-init=true sebserver.init.organisation.name=SEB Server sebserver.init.adminaccount.username=sebserver-admin +### webservice caching +spring.cache.jcache.provider=org.ehcache.jsr107.EhcacheCachingProvider +spring.cache.jcache.config=classpath:config/ehcache.xml + ### webservice data source configuration spring.datasource.username=root spring.datasource.initialize=true diff --git a/src/main/resources/config/application.properties b/src/main/resources/config/application.properties index eff5fc27..023e512e 100644 --- a/src/main/resources/config/application.properties +++ b/src/main/resources/config/application.properties @@ -13,7 +13,7 @@ server.port=8080 server.servlet.context-path=/ # Tomcat -server.tomcat.max-threads=1000 +server.tomcat.max-threads=2000 server.tomcat.accept-count=300 server.tomcat.uri-encoding=UTF-8 diff --git a/src/main/resources/config/ehcache.xml b/src/main/resources/config/ehcache.xml new file mode 100644 index 00000000..1791f8b0 --- /dev/null +++ b/src/main/resources/config/ehcache.xml @@ -0,0 +1,85 @@ + + + + java.lang.Long + ch.ethz.seb.sebserver.gbl.model.exam.Exam + + 24 + + + 100 + + + + + java.lang.String + ch.ethz.seb.sebserver.webservice.servicelayer.session.impl.ClientConnectionDataInternal + + 24 + + + 2000 + + + + + java.lang.Long + ch.ethz.seb.sebserver.webservice.servicelayer.session.impl.InMemorySEBConfig + + 24 + + + 20 + + + + + java.lang.String + ch.ethz.seb.sebserver.webservice.datalayer.batis.model.ClientEventRecord + + 24 + + + 2000 + + + + + java.lang.Long + ch.ethz.seb.sebserver.gbl.util.Result + + 24 + + + 100 + + + + + org.springframework.security.oauth2.common.OAuth2AccessToken + org.springframework.security.oauth2.provider.OAuth2Authentication + + 24 + + + 100 + + + + + java.lang.String + ch.ethz.seb.sebserver.gbl.util.Result + + 24 + + + 2000 + + + + \ No newline at end of file diff --git a/src/main/resources/logback-spring.xml b/src/main/resources/logback-spring.xml index 1e8ea704..2b12492d 100644 --- a/src/main/resources/logback-spring.xml +++ b/src/main/resources/logback-spring.xml @@ -19,6 +19,7 @@ + diff --git a/src/test/java/ch/ethz/seb/sebserver/webservice/servicelayer/lms/impl/moodle/MoodleCourseAccessTest.java b/src/test/java/ch/ethz/seb/sebserver/webservice/servicelayer/lms/impl/moodle/MoodleCourseAccessTest.java index 9c2244b8..ec1da20a 100644 --- a/src/test/java/ch/ethz/seb/sebserver/webservice/servicelayer/lms/impl/moodle/MoodleCourseAccessTest.java +++ b/src/test/java/ch/ethz/seb/sebserver/webservice/servicelayer/lms/impl/moodle/MoodleCourseAccessTest.java @@ -15,6 +15,9 @@ import static org.mockito.Mockito.*; import java.util.TreeMap; import org.junit.Test; +import org.mockito.Mock; +import org.springframework.core.env.Environment; +import org.springframework.mock.env.MockEnvironment; import org.springframework.util.LinkedMultiValueMap; import org.springframework.util.MultiValueMap; @@ -28,6 +31,9 @@ import ch.ethz.seb.sebserver.webservice.servicelayer.lms.impl.moodle.MoodleRestT public class MoodleCourseAccessTest { + @Mock + Environment env = new MockEnvironment(); + @Test public void testGetExamineeAccountDetails() { @@ -68,7 +74,8 @@ public class MoodleCourseAccessTest { null, moodleRestTemplateFactory, null, - mock(AsyncService.class)); + mock(AsyncService.class), + this.env); final String examId = "123"; final Result examineeAccountDetails = @@ -116,7 +123,8 @@ public class MoodleCourseAccessTest { null, moodleRestTemplateFactory, null, - mock(AsyncService.class)); + mock(AsyncService.class), + this.env); final LmsSetupTestResult initAPIAccess = moodleCourseAccess.initAPIAccess(); assertNotNull(initAPIAccess); @@ -138,7 +146,8 @@ public class MoodleCourseAccessTest { null, moodleRestTemplateFactory, null, - mock(AsyncService.class)); + mock(AsyncService.class), + this.env); final LmsSetupTestResult initAPIAccess = moodleCourseAccess.initAPIAccess(); assertNotNull(initAPIAccess); @@ -159,7 +168,8 @@ public class MoodleCourseAccessTest { null, moodleRestTemplateFactory, null, - mock(AsyncService.class)); + mock(AsyncService.class), + this.env); final LmsSetupTestResult initAPIAccess = moodleCourseAccess.initAPIAccess(); assertNotNull(initAPIAccess); diff --git a/src/test/resources/application-test.properties b/src/test/resources/application-test.properties index 853b0aae..59e6955f 100644 --- a/src/test/resources/application-test.properties +++ b/src/test/resources/application-test.properties @@ -12,6 +12,9 @@ server.servlet.context-path=/ spring.main.allow-bean-definition-overriding=true sebserver.password=test-password +spring.cache.jcache.provider= +spring.cache.jcache.config=classpath:config/ehcache.xml + spring.h2.console.enabled=true spring.datasource.platform=h2 spring.datasource.url=jdbc:h2:mem:test;MODE=MySQL;DB_CLOSE_DELAY=-1;DB_CLOSE_ON_EXIT=FALSE