diff --git a/src/main/java/ch/ethz/seb/sebserver/webservice/servicelayer/lms/impl/moodle/MockupRestTemplateFactory.java b/src/main/java/ch/ethz/seb/sebserver/webservice/servicelayer/lms/impl/moodle/MockupRestTemplateFactory.java index 60b49d3b..4cf512e0 100644 --- a/src/main/java/ch/ethz/seb/sebserver/webservice/servicelayer/lms/impl/moodle/MockupRestTemplateFactory.java +++ b/src/main/java/ch/ethz/seb/sebserver/webservice/servicelayer/lms/impl/moodle/MockupRestTemplateFactory.java @@ -39,6 +39,7 @@ import ch.ethz.seb.sebserver.webservice.servicelayer.lms.impl.moodle.MoodleUtils import ch.ethz.seb.sebserver.webservice.servicelayer.lms.impl.moodle.plugin.MoodlePluginCourseAccess; import ch.ethz.seb.sebserver.webservice.servicelayer.lms.impl.moodle.plugin.MoodlePluginCourseRestriction; +@Deprecated public class MockupRestTemplateFactory implements MoodleRestTemplateFactory { private final APITemplateDataSupplier apiTemplateDataSupplier; @@ -220,8 +221,8 @@ public class MockupRestTemplateFactory implements MoodleRestTemplateFactory { private String respondCourses(final MultiValueMap queryAttributes) { try { - final List ids = queryAttributes.get(MoodlePluginCourseAccess.CRITERIA_COURSE_IDS); - final String from = queryAttributes.getFirst(MoodlePluginCourseAccess.CRITERIA_LIMIT_FROM); + final List ids = queryAttributes.get(MoodlePluginCourseAccess.PARAM_COURSE_ID); + final String from = queryAttributes.getFirst(MoodlePluginCourseAccess.PARAM_PAGE_START); System.out.println("************* from: " + from); final List courses; if (ids != null && !ids.isEmpty()) { @@ -242,7 +243,7 @@ public class MockupRestTemplateFactory implements MoodleRestTemplateFactory { } final Map response = new HashMap<>(); - response.put("courses", courses); + response.put("results", courses); final JSONMapper jsonMapper = new JSONMapper(); final String result = jsonMapper.writeValueAsString(response); System.out.println("******** courses response: " + result); diff --git a/src/main/java/ch/ethz/seb/sebserver/webservice/servicelayer/lms/impl/moodle/MoodleUtils.java b/src/main/java/ch/ethz/seb/sebserver/webservice/servicelayer/lms/impl/moodle/MoodleUtils.java index 3540d1c7..fab90ae8 100644 --- a/src/main/java/ch/ethz/seb/sebserver/webservice/servicelayer/lms/impl/moodle/MoodleUtils.java +++ b/src/main/java/ch/ethz/seb/sebserver/webservice/servicelayer/lms/impl/moodle/MoodleUtils.java @@ -278,6 +278,39 @@ public abstract class MoodleUtils { } } + @JsonIgnoreProperties(ignoreUnknown = true) + public static final class CoursesPagePlugin { + public final String coursecount; + public final Integer needle; + public final Integer perpage; + + public CoursesPagePlugin( + @JsonProperty("coursecount") final String coursecount, + @JsonProperty("needle") final Integer needle, + @JsonProperty("perpage") final Integer perpage) { + this.coursecount = coursecount; + this.needle = needle; + this.perpage = perpage; + } + } + + @JsonIgnoreProperties(ignoreUnknown = true) + public static final class CoursesPlugin { + public final CoursesPagePlugin stats; + public final Collection results; + public final Collection warnings; + + @JsonCreator + public CoursesPlugin( + @JsonProperty("stats") final CoursesPagePlugin stats, + @JsonProperty("courses") final Collection results, + @JsonProperty("warnings") final Collection warnings) { + this.stats = stats; + this.results = results; + this.warnings = warnings; + } + } + @JsonIgnoreProperties(ignoreUnknown = true) public static final class Courses { public final Collection courses; @@ -453,27 +486,41 @@ public abstract class MoodleUtils { } } + @JsonIgnoreProperties(ignoreUnknown = true) + public static final class MoodleQuizRestrictions { + public final Collection data; + public final Collection warnings; + + public MoodleQuizRestrictions( + @JsonProperty("data") final Collection data, + @JsonProperty("warnings") final Collection warnings) { + + this.data = data; + this.warnings = warnings; + } + } + @JsonIgnoreProperties(ignoreUnknown = true) public static final class MoodleQuizRestriction { - public final String quiz_id; - public final String config_keys; - public final String browser_exam_keys; - public final String quit_link; - public final String quit_secret; + public final String quizid; + public final String configkeys; + public final String browserkeys; + public final String quitlink; + public final String quitsecret; @JsonCreator public MoodleQuizRestriction( - @JsonProperty("quiz_id") final String quiz_id, - @JsonProperty("config_keys") final String config_keys, - @JsonProperty("browser_exam_keys") final String browser_exam_keys, - @JsonProperty("quit_link") final String quit_link, - @JsonProperty("quit_secret") final String quit_secret) { + @JsonProperty("quizid") final String quizid, + @JsonProperty("configkeys") final String configkeys, + @JsonProperty("browserkeys") final String browserkeys, + @JsonProperty("quitlink") final String quitlink, + @JsonProperty("quitsecret") final String quitsecret) { - this.quiz_id = quiz_id; - this.config_keys = config_keys; - this.browser_exam_keys = browser_exam_keys; - this.quit_link = quit_link; - this.quit_secret = quit_secret; + this.quizid = quizid; + this.configkeys = configkeys; + this.browserkeys = browserkeys; + this.quitlink = quitlink; + this.quitsecret = quitsecret; } } diff --git a/src/main/java/ch/ethz/seb/sebserver/webservice/servicelayer/lms/impl/moodle/legacy/MoodleCourseDataAsyncLoader.java b/src/main/java/ch/ethz/seb/sebserver/webservice/servicelayer/lms/impl/moodle/legacy/MoodleCourseDataAsyncLoader.java index b535d912..e3020f26 100644 --- a/src/main/java/ch/ethz/seb/sebserver/webservice/servicelayer/lms/impl/moodle/legacy/MoodleCourseDataAsyncLoader.java +++ b/src/main/java/ch/ethz/seb/sebserver/webservice/servicelayer/lms/impl/moodle/legacy/MoodleCourseDataAsyncLoader.java @@ -85,7 +85,10 @@ public class MoodleCourseDataAsyncLoader { final Environment environment) { this.jsonMapper = jsonMapper; - this.fromCutTime = Utils.toUnixTimeInSeconds(DateTime.now(DateTimeZone.UTC).minusYears(3)); + final int yearsBeforeNow = environment.getProperty( + "sebserver.webservice.lms.moodle.fetch.cutoffdate.yearsBeforeNow", + Integer.class, 3); + this.fromCutTime = Utils.toUnixTimeInSeconds(DateTime.now(DateTimeZone.UTC).minusYears(yearsBeforeNow)); this.asyncRunner = asyncRunner; this.moodleRestCall = asyncService.createCircuitBreaker( diff --git a/src/main/java/ch/ethz/seb/sebserver/webservice/servicelayer/lms/impl/moodle/plugin/MoodlePluginCourseAccess.java b/src/main/java/ch/ethz/seb/sebserver/webservice/servicelayer/lms/impl/moodle/plugin/MoodlePluginCourseAccess.java index 5ad9b7bd..c3b4aecd 100644 --- a/src/main/java/ch/ethz/seb/sebserver/webservice/servicelayer/lms/impl/moodle/plugin/MoodlePluginCourseAccess.java +++ b/src/main/java/ch/ethz/seb/sebserver/webservice/servicelayer/lms/impl/moodle/plugin/MoodlePluginCourseAccess.java @@ -24,8 +24,8 @@ import java.util.stream.Collectors; import java.util.stream.Stream; import org.apache.commons.lang3.BooleanUtils; -import org.apache.commons.lang3.StringUtils; import org.joda.time.DateTime; +import org.joda.time.DateTimeZone; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.cache.CacheManager; @@ -59,6 +59,7 @@ import ch.ethz.seb.sebserver.webservice.servicelayer.lms.impl.moodle.MoodleRestT import ch.ethz.seb.sebserver.webservice.servicelayer.lms.impl.moodle.MoodleUtils; import ch.ethz.seb.sebserver.webservice.servicelayer.lms.impl.moodle.MoodleUtils.CourseData; import ch.ethz.seb.sebserver.webservice.servicelayer.lms.impl.moodle.MoodleUtils.Courses; +import ch.ethz.seb.sebserver.webservice.servicelayer.lms.impl.moodle.MoodleUtils.CoursesPlugin; import ch.ethz.seb.sebserver.webservice.servicelayer.lms.impl.moodle.MoodleUtils.MoodleUserDetails; public class MoodlePluginCourseAccess extends AbstractCachedCourseAccess implements CourseAccessAPI { @@ -66,18 +67,21 @@ public class MoodlePluginCourseAccess extends AbstractCachedCourseAccess impleme private static final Logger log = LoggerFactory.getLogger(MoodlePluginCourseAccess.class); public static final String MOODLE_QUIZ_START_URL_PATH = "mod/quiz/view.php?id="; - public static final String COURSES_API_FUNCTION_NAME = "quizaccess_sebserver_get_courses"; + public static final String COURSES_API_FUNCTION_NAME = "quizaccess_sebserver_get_exams"; public static final String USERS_API_FUNCTION_NAME = "core_user_get_users_by_field"; public static final String ATTR_FIELD = "field"; public static final String ATTR_VALUE = "value"; public static final String ATTR_ID = "id"; public static final String ATTR_SHORTNAME = "shortname"; - public static final String CRITERIA_COURSE_IDS = "ids"; - public static final String CRITERIA_FROM_DATE = "from_date"; - public static final String CRITERIA_TO_DATE = "to_date"; - public static final String CRITERIA_LIMIT_FROM = "limitfrom"; - public static final String CRITERIA_LIMIT_NUM = "limitnum"; + + public static final String PARAM_COURSE_ID = "courseid"; + public static final String PARAM_SQL_CONDITIONS = "conditions"; + public static final String PARAM_PAGE_START = "startneedle"; + public static final String PARAM_PAGE_SIZE = "perpage"; + + public static final String SQL_CONDITION_TEMPLATE = + "(startdate >= %s or timecreated >=%s) and (enddate is null or enddate is 0 or enddate is >= %s)"; private final JSONMapper jsonMapper; private final MoodleRestTemplateFactory restTemplateFactory; @@ -85,6 +89,7 @@ public class MoodlePluginCourseAccess extends AbstractCachedCourseAccess impleme private final boolean prependShortCourseName; private final int pageSize; private final int maxSize; + private final int cutoffTimeOffset; private MoodleAPIRestTemplate restTemplate; @@ -119,7 +124,11 @@ public class MoodlePluginCourseAccess extends AbstractCachedCourseAccess impleme this.maxSize = environment.getProperty("sebserver.webservice.cache.moodle.course.maxSize", Integer.class, 10000); this.pageSize = - environment.getProperty("sebserver.webservice.cache.moodle.course.pageSize", Integer.class, 10); + environment.getProperty("sebserver.webservice.cache.moodle.course.pageSize", Integer.class, 500); + + this.cutoffTimeOffset = environment.getProperty( + "sebserver.webservice.lms.moodle.fetch.cutoffdate.yearsBeforeNow", + Integer.class, 3); } @Override @@ -169,7 +178,10 @@ public class MoodlePluginCourseAccess extends AbstractCachedCourseAccess impleme int page = 0; int failedAttempts = 0; - final DateTime quizFromTime = filterMap.getQuizFromTime(); + DateTime quizFromTime = filterMap.getQuizFromTime(); + if (quizFromTime == null) { + quizFromTime = DateTime.now(DateTimeZone.UTC).minusYears(this.cutoffTimeOffset); + } final Predicate quizFilter = LmsAPIService.quizFilterPredicate(filterMap); while (!asyncQuizFetchBuffer.finished && !asyncQuizFetchBuffer.canceled) { @@ -401,11 +413,20 @@ public class MoodlePluginCourseAccess extends AbstractCachedCourseAccess impleme final String lmsName = getLmsSetupName(); try { // get course ids per page - final String fromDate = String.valueOf(Utils.toUnixTimeInSeconds(quizFromTime)); + final long filterDate = Utils.toUnixTimeInSeconds(quizFromTime); + final long defaultCutOff = Utils.toUnixTimeInSeconds( + DateTime.now(DateTimeZone.UTC).minusYears(this.cutoffTimeOffset)); + final long cutoffDate = (filterDate < defaultCutOff) ? filterDate : defaultCutOff; + final String sqlCondition = String.format( + SQL_CONDITION_TEMPLATE, + String.valueOf(cutoffDate), + String.valueOf(cutoffDate), + String.valueOf(filterDate)); final String fromElement = String.valueOf(page * size); final LinkedMultiValueMap attributes = new LinkedMultiValueMap<>(); - attributes.add(CRITERIA_FROM_DATE, fromDate); - attributes.add(CRITERIA_LIMIT_FROM, fromElement); + attributes.add(PARAM_SQL_CONDITIONS, sqlCondition); + attributes.add(PARAM_PAGE_START, fromElement); + attributes.add(PARAM_PAGE_SIZE, String.valueOf(size)); final String courseKeyPageJSON = this.protectedMoodlePageCall .protectedRun(() -> restTemplate.callMoodleAPIFunction( @@ -413,7 +434,7 @@ public class MoodlePluginCourseAccess extends AbstractCachedCourseAccess impleme attributes)) .getOrThrow(); - final Courses coursePage = this.jsonMapper.readValue(courseKeyPageJSON, Courses.class); + final CoursesPlugin coursePage = this.jsonMapper.readValue(courseKeyPageJSON, CoursesPlugin.class); if (coursePage == null) { log.error("No CoursePage Response"); @@ -425,7 +446,7 @@ public class MoodlePluginCourseAccess extends AbstractCachedCourseAccess impleme } Collection result; - if (coursePage.courses == null || coursePage.courses.isEmpty()) { + if (coursePage.results == null || coursePage.results.isEmpty()) { if (log.isDebugEnabled()) { log.debug("LMS Setup: {} No courses found on page: {}", lmsName, page); if (log.isTraceEnabled()) { @@ -434,7 +455,7 @@ public class MoodlePluginCourseAccess extends AbstractCachedCourseAccess impleme } result = Collections.emptyList(); } else { - result = coursePage.courses; + result = coursePage.results; } if (log.isDebugEnabled()) { @@ -476,7 +497,9 @@ public class MoodlePluginCourseAccess extends AbstractCachedCourseAccess impleme lmsSetup, courseData, urlPrefix, - this.prependShortCourseName).stream()) + this.prependShortCourseName) + .stream() + .filter(q -> internalIds.contains(q.id))) .collect(Collectors.toList()); } catch (final Exception e) { @@ -497,19 +520,17 @@ public class MoodlePluginCourseAccess extends AbstractCachedCourseAccess impleme log.debug("Get courses for ids: {} on LMS: {}", courseIds, lmsSetup); } - final String joinedIds = StringUtils.join(courseIds, Constants.COMMA); - final LinkedMultiValueMap attributes = new LinkedMultiValueMap<>(); - attributes.add(CRITERIA_COURSE_IDS, joinedIds); + attributes.put(PARAM_COURSE_ID, new ArrayList<>(courseIds)); final String coursePageJSON = restTemplate.callMoodleAPIFunction( COURSES_API_FUNCTION_NAME, attributes); - final Courses courses = this.jsonMapper.readValue( + final CoursesPlugin courses = this.jsonMapper.readValue( coursePageJSON, - Courses.class); + CoursesPlugin.class); - if (courses.courses == null || courses.courses.isEmpty()) { + if (courses.results == null || courses.results.isEmpty()) { log.warn("No courses found for ids: {} on LMS: {}", courseIds, lmsSetup.name); if (courses != null && courses.warnings != null && !courses.warnings.isEmpty()) { @@ -518,7 +539,7 @@ public class MoodlePluginCourseAccess extends AbstractCachedCourseAccess impleme return Collections.emptyList(); } - return courses.courses; + return courses.results; } catch (final Exception e) { log.error("Unexpected error while trying to get courses for ids", e); return Collections.emptyList(); @@ -539,4 +560,18 @@ public class MoodlePluginCourseAccess extends AbstractCachedCourseAccess impleme return Result.of(this.restTemplate); } + protected String toTestString() { + final StringBuilder builder = new StringBuilder(); + builder.append("MoodlePluginCourseAccess [pageSize="); + builder.append(this.pageSize); + builder.append(", maxSize="); + builder.append(this.maxSize); + builder.append(", cutoffTimeOffset="); + builder.append(this.cutoffTimeOffset); + builder.append(", restTemplate="); + builder.append(this.restTemplate); + builder.append("]"); + return builder.toString(); + } + } diff --git a/src/main/java/ch/ethz/seb/sebserver/webservice/servicelayer/lms/impl/moodle/plugin/MoodlePluginCourseRestriction.java b/src/main/java/ch/ethz/seb/sebserver/webservice/servicelayer/lms/impl/moodle/plugin/MoodlePluginCourseRestriction.java index addf0ff2..f3ffc6ab 100644 --- a/src/main/java/ch/ethz/seb/sebserver/webservice/servicelayer/lms/impl/moodle/plugin/MoodlePluginCourseRestriction.java +++ b/src/main/java/ch/ethz/seb/sebserver/webservice/servicelayer/lms/impl/moodle/plugin/MoodlePluginCourseRestriction.java @@ -33,6 +33,7 @@ import ch.ethz.seb.sebserver.webservice.servicelayer.lms.impl.moodle.MoodleAPIRe import ch.ethz.seb.sebserver.webservice.servicelayer.lms.impl.moodle.MoodleRestTemplateFactory; import ch.ethz.seb.sebserver.webservice.servicelayer.lms.impl.moodle.MoodleUtils; import ch.ethz.seb.sebserver.webservice.servicelayer.lms.impl.moodle.MoodleUtils.MoodleQuizRestriction; +import ch.ethz.seb.sebserver.webservice.servicelayer.lms.impl.moodle.MoodleUtils.MoodleQuizRestrictions; public class MoodlePluginCourseRestriction implements SEBRestrictionAPI { @@ -110,16 +111,7 @@ public class MoodlePluginCourseRestriction implements SEBRestrictionAPI { addQuery, new LinkedMultiValueMap<>()); - try { - - final MoodleQuizRestriction moodleRestriction = this.jsonMapper.readValue( - srJSON, - MoodleUtils.MoodleQuizRestriction.class); - - return toSEBRestriction(exam, moodleRestriction); - } catch (final Exception e) { - throw new RuntimeException("Unexpected error while get SEB restriction: ", e); - } + return restrictionFromJson(exam, srJSON); }); } @@ -160,16 +152,7 @@ public class MoodlePluginCourseRestriction implements SEBRestrictionAPI { addQuery, queryAttributes); - try { - - final MoodleQuizRestriction moodleRestriction = this.jsonMapper.readValue( - srJSON, - MoodleUtils.MoodleQuizRestriction.class); - - return toSEBRestriction(exam, moodleRestriction); - } catch (final Exception e) { - throw new RuntimeException("Unexpected error while get SEB restriction: ", e); - } + return restrictionFromJson(exam, srJSON); }); } @@ -200,16 +183,56 @@ public class MoodlePluginCourseRestriction implements SEBRestrictionAPI { }); } - private SEBRestriction toSEBRestriction(final Exam exam, final MoodleQuizRestriction moodleRestriction) { + private SEBRestriction restrictionFromJson(final Exam exam, final String srJSON) { + try { + // fist try to get from multiple data + final MoodleQuizRestrictions moodleRestrictions = this.jsonMapper.readValue( + srJSON, + MoodleUtils.MoodleQuizRestrictions.class); + + return toSEBRestriction(exam, moodleRestrictions); + } catch (final Exception e) { + try { + // then try to get from single + final MoodleQuizRestriction moodleRestriction = this.jsonMapper.readValue( + srJSON, + MoodleUtils.MoodleQuizRestriction.class); + + return toSEBRestriction(exam, moodleRestriction); + } catch (final Exception ee) { + throw new RuntimeException("Unexpected error while get SEB restriction: ", ee); + } + } + } + + private SEBRestriction toSEBRestriction( + final Exam exam, + final MoodleQuizRestrictions moodleRestrictions) { + + if (moodleRestrictions.warnings != null && !moodleRestrictions.warnings.isEmpty()) { + log.warn("Moodle restriction call warnings: {}", moodleRestrictions.warnings); + } + + if (moodleRestrictions.data == null || moodleRestrictions.data.isEmpty()) { + throw new IllegalArgumentException("Expecting MoodleQuizRestriction not available. Exam: " + exam); + } + + return toSEBRestriction(exam, moodleRestrictions.data.iterator().next()); + } + + private SEBRestriction toSEBRestriction( + final Exam exam, + final MoodleQuizRestriction moodleRestriction) { + final List configKeys = Arrays.asList(StringUtils.split( - moodleRestriction.config_keys, + moodleRestriction.configkeys, Constants.LIST_SEPARATOR)); final List browserExamKeys = new ArrayList<>(Arrays.asList(StringUtils.split( - moodleRestriction.browser_exam_keys, + moodleRestriction.browserkeys, Constants.LIST_SEPARATOR))); final Map additionalProperties = new HashMap<>(); - additionalProperties.put(ATTRIBUTE_QUIT_URL, moodleRestriction.quit_link); - additionalProperties.put(ATTRIBUTE_QUIT_SECRET, moodleRestriction.quit_secret); + additionalProperties.put(ATTRIBUTE_QUIT_URL, moodleRestriction.quitlink); + additionalProperties.put(ATTRIBUTE_QUIT_SECRET, moodleRestriction.quitsecret); return new SEBRestriction( exam.id, diff --git a/src/main/resources/config/application-dev-ws.properties b/src/main/resources/config/application-dev-ws.properties index 0cbfad95..25b871d9 100644 --- a/src/main/resources/config/application-dev-ws.properties +++ b/src/main/resources/config/application-dev-ws.properties @@ -52,6 +52,7 @@ sebserver.webservice.api.pagination.maxPageSize=500 sebserver.webservice.lms.openedx.api.token.request.paths=/oauth2/access_token sebserver.webservice.lms.moodle.api.token.request.paths= sebserver.webservice.lms.address.alias=lms.mockup.com=lms.address.alias +sebserver.webservice.cache.moodle.course.pageSize=10 springdoc.api-docs.enabled=true springdoc.swagger-ui.enabled=true diff --git a/src/main/resources/config/application-ws.properties b/src/main/resources/config/application-ws.properties index 94a94b85..7fd5b02b 100644 --- a/src/main/resources/config/application-ws.properties +++ b/src/main/resources/config/application-ws.properties @@ -80,6 +80,7 @@ sebserver.webservice.api.pagination.maxPageSize=500 sebserver.webservice.lms.openedx.api.token.request.paths=/oauth2/access_token sebserver.webservice.lms.moodle.api.token.request.paths=/login/token.php sebserver.webservice.lms.moodle.prependShortCourseName=true +sebserver.webservice.lms.moodle.fetch.cutoffdate.yearsBeforeNow=2 sebserver.webservice.lms.address.alias= sebserver.webservice.proctoring.resetBroadcastOnLeav=true diff --git a/src/test/java/ch/ethz/seb/sebserver/webservice/servicelayer/lms/impl/moodle/MoodleMockupRestTemplateFactory.java b/src/test/java/ch/ethz/seb/sebserver/webservice/servicelayer/lms/impl/moodle/MoodleMockupRestTemplateFactory.java new file mode 100644 index 00000000..71885140 --- /dev/null +++ b/src/test/java/ch/ethz/seb/sebserver/webservice/servicelayer/lms/impl/moodle/MoodleMockupRestTemplateFactory.java @@ -0,0 +1,333 @@ +/* + * Copyright (c) 2022 ETH Zürich, Educational Development and Technology (LET) + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ + +package ch.ethz.seb.sebserver.webservice.servicelayer.lms.impl.moodle; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; + +import org.apache.commons.lang3.StringUtils; +import org.assertj.core.util.Arrays; +import org.springframework.http.HttpEntity; +import org.springframework.http.HttpHeaders; +import org.springframework.http.MediaType; +import org.springframework.util.LinkedMultiValueMap; +import org.springframework.util.MultiValueMap; +import org.springframework.web.util.UriComponentsBuilder; + +import com.fasterxml.jackson.core.JsonProcessingException; + +import ch.ethz.seb.sebserver.gbl.Constants; +import ch.ethz.seb.sebserver.gbl.api.JSONMapper; +import ch.ethz.seb.sebserver.gbl.model.institution.LmsSetup.LmsType; +import ch.ethz.seb.sebserver.gbl.model.institution.LmsSetupTestResult; +import ch.ethz.seb.sebserver.gbl.util.Result; +import ch.ethz.seb.sebserver.gbl.util.Utils; +import ch.ethz.seb.sebserver.webservice.servicelayer.lms.APITemplateDataSupplier; +import ch.ethz.seb.sebserver.webservice.servicelayer.lms.impl.moodle.MoodleUtils.MoodleQuizRestriction; +import ch.ethz.seb.sebserver.webservice.servicelayer.lms.impl.moodle.plugin.MoodlePluginCourseAccess; +import ch.ethz.seb.sebserver.webservice.servicelayer.lms.impl.moodle.plugin.MoodlePluginCourseRestriction; + +public class MoodleMockupRestTemplateFactory implements MoodleRestTemplateFactory { + + private final APITemplateDataSupplier apiTemplateDataSupplier; + + public MoodleMockupRestTemplateFactory(final APITemplateDataSupplier apiTemplateDataSupplier) { + this.apiTemplateDataSupplier = apiTemplateDataSupplier; + } + + @Override + public LmsSetupTestResult test() { + return LmsSetupTestResult.ofOkay(LmsType.MOODLE_PLUGIN); + } + + @Override + public APITemplateDataSupplier getApiTemplateDataSupplier() { + return this.apiTemplateDataSupplier; + } + + @Override + public Set getKnownTokenAccessPaths() { + final Set paths = new HashSet<>(); + paths.add(MoodleAPIRestTemplate.MOODLE_DEFAULT_TOKEN_REQUEST_PATH); + return paths; + } + + @Override + public Result createRestTemplate() { + return Result.of(new MockupMoodleRestTemplate(this.apiTemplateDataSupplier.getLmsSetup().lmsApiUrl)); + } + + @Override + public Result createRestTemplate(final String accessTokenPath) { + return Result.of(new MockupMoodleRestTemplate(this.apiTemplateDataSupplier.getLmsSetup().lmsApiUrl)); + } + + public static final class MockupMoodleRestTemplate implements MoodleAPIRestTemplate { + + private final String accessToken = "MockupMoodleRestTemplate-Test-Token"; + private final String url; + public final Collection testLog = new ArrayList<>(); + public final Collection> callLog = new ArrayList<>(); + + private final List courses = new ArrayList<>(); + + public MockupMoodleRestTemplate(final String url) { + this.url = url; + + for (int i = 0; i < 20; i++) { + this.courses.add(new MockCD( + String.valueOf(i), + getQuizzesForCourse(i))); + } + } + + @Override + public String getService() { + return "mockup-service"; + } + + @Override + public void setService(final String service) { + } + + @Override + public CharSequence getAccessToken() { + return this.accessToken; + } + + @Override + public void testAPIConnection(final String... functions) { + this.testLog.add("testAPIConnection functions: " + Arrays.asList(functions)); + } + + @Override + public String callMoodleAPIFunction(final String functionName) { + return callMoodleAPIFunction(functionName, null, null); + } + + @Override + public String callMoodleAPIFunction( + final String functionName, + final MultiValueMap queryAttributes) { + return callMoodleAPIFunction(functionName, null, queryAttributes); + } + + @Override + public String callMoodleAPIFunction( + final String functionName, + final MultiValueMap queryParams, + final MultiValueMap queryAttributes) { + + final UriComponentsBuilder queryParam = UriComponentsBuilder + .fromHttpUrl(this.url + MOODLE_DEFAULT_REST_API_PATH) + .queryParam(REST_REQUEST_TOKEN_NAME, this.accessToken) + .queryParam(REST_REQUEST_FUNCTION_NAME, functionName) + .queryParam(REST_REQUEST_FORMAT_NAME, "json"); + + if (queryParams != null && !queryParams.isEmpty()) { + queryParam.queryParams(queryParams); + } + + final boolean usePOST = queryAttributes != null && !queryAttributes.isEmpty(); + HttpEntity functionReqEntity; + if (usePOST) { + final HttpHeaders headers = new HttpHeaders(); + headers.set( + HttpHeaders.CONTENT_TYPE, + MediaType.APPLICATION_FORM_URLENCODED_VALUE); + + final String body = Utils.toAppFormUrlEncodedBody(queryAttributes); + functionReqEntity = new HttpEntity<>(body, headers); + + } else { + functionReqEntity = new HttpEntity<>(new LinkedMultiValueMap<>()); + } + + this.testLog.add("callMoodleAPIFunction: " + functionName); + this.callLog.add(functionReqEntity); + + if (MoodlePluginCourseAccess.COURSES_API_FUNCTION_NAME.equals(functionName)) { + return respondCourses(queryAttributes); + } else if (MoodlePluginCourseAccess.USERS_API_FUNCTION_NAME.equals(functionName)) { + return respondUsers(queryAttributes); + } else if (MoodlePluginCourseRestriction.RESTRICTION_GET_FUNCTION_NAME.equals(functionName)) { + + return respondGetRestriction( + queryParams.getFirst(MoodlePluginCourseRestriction.ATTRIBUTE_QUIZ_ID), + queryAttributes); + } else if (MoodlePluginCourseRestriction.RESTRICTION_SET_FUNCTION_NAME.equals(functionName)) { + return respondSetRestriction( + queryParams.getFirst(MoodlePluginCourseRestriction.ATTRIBUTE_QUIZ_ID), + queryAttributes); + } + + else { + throw new RuntimeException("Unknown function: " + functionName); + } + } + + @Override + public String toString() { + final StringBuilder builder = new StringBuilder(); + builder.append("MockupMoodleRestTemplate [accessToken="); + builder.append(this.accessToken); + builder.append(", url="); + builder.append(this.url); + builder.append(", testLog="); + builder.append(this.testLog); + builder.append(", callLog="); + builder.append(this.callLog); + builder.append("]"); + return builder.toString(); + } + + @SuppressWarnings("unused") + private static final class MockCD { + public final String id; + public final String shortname; + public final String categoryid; + public final String fullname; + public final String displayname; + public final String idnumber; + public final Long startdate; // unix-time seconds UTC + public final Long enddate; // unix-time seconds UTC + public final Long timecreated; // unix-time seconds UTC + public final boolean visible; + public final Collection quizzes; + + public MockCD(final String num, final Collection quizzes) { + this.id = num; + this.shortname = "c" + num; + this.categoryid = "mock"; + this.fullname = "course" + num; + this.displayname = this.fullname; + this.idnumber = "i" + num; + this.startdate = Long.valueOf(num); + this.enddate = null; + this.timecreated = Long.valueOf(num); + this.visible = true; + this.quizzes = quizzes; + } + } + + @SuppressWarnings("unused") + private static final class MockQ { + public final String id; + public final String coursemodule; + public final String course; + public final String name; + public final String intro; + public final Long timeopen; // unix-time seconds UTC + public final Long timeclose; // unix-time seconds UTC + + public MockQ(final String courseId, final String num) { + this.id = num; + this.coursemodule = num; + this.course = courseId; + this.name = "quiz " + num; + this.intro = this.name; + this.timeopen = Long.valueOf(num); + this.timeclose = null; + } + } + + private String respondCourses(final MultiValueMap queryAttributes) { + try { + final List ids = queryAttributes.get(MoodlePluginCourseAccess.PARAM_COURSE_ID); + final String from = queryAttributes.getFirst(MoodlePluginCourseAccess.PARAM_PAGE_START); + final String size = queryAttributes.getFirst(MoodlePluginCourseAccess.PARAM_PAGE_SIZE); + System.out.println("************* from: " + from); + final List courses; + if (ids != null && !ids.isEmpty()) { + courses = this.courses.stream().filter(c -> ids.contains(c.id)).collect(Collectors.toList()); + } else if (from != null && Integer.valueOf(from) < this.courses.size()) { + + final int to = Integer.valueOf(from) + Integer.valueOf(size); + courses = this.courses.subList( + Integer.valueOf(from), + (to < this.courses.size()) ? to : this.courses.size() - 1); + } else { + courses = new ArrayList<>(); + } + + final Map response = new HashMap<>(); + response.put("results", courses); + final JSONMapper jsonMapper = new JSONMapper(); + final String result = jsonMapper.writeValueAsString(response); + System.out.println("******** courses response: " + result); + return result; + } catch (final JsonProcessingException e) { + e.printStackTrace(); + return ""; + } + } + + private final Map restrcitions = new HashMap<>(); + + private String respondSetRestriction(final String quizId, final MultiValueMap queryAttributes) { + final List configKeys = queryAttributes.get(MoodlePluginCourseRestriction.ATTRIBUTE_CONFIG_KEYS); + final List beks = queryAttributes.get(MoodlePluginCourseRestriction.ATTRIBUTE_BROWSER_EXAM_KEYS); + final String quitURL = queryAttributes.getFirst(MoodlePluginCourseRestriction.ATTRIBUTE_QUIT_URL); + final String quitSecret = queryAttributes.getFirst(MoodlePluginCourseRestriction.ATTRIBUTE_QUIT_SECRET); + + final MoodleQuizRestriction moodleQuizRestriction = new MoodleQuizRestriction( + quizId, + StringUtils.join(configKeys, Constants.LIST_SEPARATOR), + StringUtils.join(beks, Constants.LIST_SEPARATOR), + quitURL, + quitSecret); + this.restrcitions.put(quizId, moodleQuizRestriction); + + final JSONMapper jsonMapper = new JSONMapper(); + try { + return jsonMapper.writeValueAsString(moodleQuizRestriction); + } catch (final JsonProcessingException e) { + e.printStackTrace(); + return ""; + } + } + + private String respondGetRestriction(final String quizId, final MultiValueMap queryAttributes) { + final MoodleQuizRestriction moodleQuizRestriction = this.restrcitions.get(quizId); + if (moodleQuizRestriction != null) { + final JSONMapper jsonMapper = new JSONMapper(); + try { + return jsonMapper.writeValueAsString(moodleQuizRestriction); + } catch (final JsonProcessingException e) { + e.printStackTrace(); + return ""; + } + } + return ""; + } + + private Collection getQuizzesForCourse(final int courseId) { + final String id = String.valueOf(courseId); + final Collection result = new ArrayList<>(); + result.add(new MockQ(id, "10" + id)); + if (courseId % 2 > 0) { + result.add(new MockQ(id, "11" + id)); + } + + return result; + } + + private String respondUsers(final MultiValueMap queryAttributes) { + // TODO + return ""; + } + + } +} diff --git a/src/test/java/ch/ethz/seb/sebserver/webservice/servicelayer/lms/impl/moodle/plugin/MoodlePluginCourseAccessTest.java b/src/test/java/ch/ethz/seb/sebserver/webservice/servicelayer/lms/impl/moodle/plugin/MoodlePluginCourseAccessTest.java new file mode 100644 index 00000000..5b376787 --- /dev/null +++ b/src/test/java/ch/ethz/seb/sebserver/webservice/servicelayer/lms/impl/moodle/plugin/MoodlePluginCourseAccessTest.java @@ -0,0 +1,286 @@ +/* + * Copyright (c) 2023 ETH Zürich, Educational Development and Technology (LET) + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ + +package ch.ethz.seb.sebserver.webservice.servicelayer.lms.impl.moodle.plugin; + +import static org.junit.Assert.*; + +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import org.joda.time.DateTimeUtils; +import org.junit.Test; +import org.springframework.cache.support.NoOpCacheManager; +import org.springframework.mock.env.MockEnvironment; + +import ch.ethz.seb.sebserver.gbl.api.JSONMapper; +import ch.ethz.seb.sebserver.gbl.async.AsyncRunner; +import ch.ethz.seb.sebserver.gbl.async.AsyncService; +import ch.ethz.seb.sebserver.gbl.client.ClientCredentials; +import ch.ethz.seb.sebserver.gbl.client.ProxyData; +import ch.ethz.seb.sebserver.gbl.model.exam.QuizData; +import ch.ethz.seb.sebserver.gbl.model.institution.LmsSetup; +import ch.ethz.seb.sebserver.gbl.model.institution.LmsSetup.LmsType; +import ch.ethz.seb.sebserver.gbl.model.institution.LmsSetupTestResult; +import ch.ethz.seb.sebserver.gbl.util.Result; +import ch.ethz.seb.sebserver.webservice.servicelayer.dao.FilterMap; +import ch.ethz.seb.sebserver.webservice.servicelayer.lms.APITemplateDataSupplier; +import ch.ethz.seb.sebserver.webservice.servicelayer.lms.CourseAccessAPI; +import ch.ethz.seb.sebserver.webservice.servicelayer.lms.CourseAccessAPI.AsyncQuizFetchBuffer; +import ch.ethz.seb.sebserver.webservice.servicelayer.lms.impl.moodle.MoodleMockupRestTemplateFactory; + +public class MoodlePluginCourseAccessTest { + + @Test + public void testSetup() { + final MoodlePluginCourseAccess candidate = crateMockup(); + + assertEquals("MoodlePluginCourseAccess [" + + "pageSize=500, " + + "maxSize=10000, " + + "cutoffTimeOffset=3, " + + "restTemplate=null]", candidate.toTestString()); + + final LmsSetupTestResult testCourseAccessAPI = candidate.testCourseAccessAPI(); + + assertTrue(testCourseAccessAPI.isOk()); + + assertEquals("MoodlePluginCourseAccess [pageSize=500, maxSize=10000, cutoffTimeOffset=3, " + + "restTemplate=MockupMoodleRestTemplate [accessToken=MockupMoodleRestTemplate-Test-Token, url=https://test.org/, " + + "testLog=[testAPIConnection functions: [quizaccess_sebserver_get_exams, core_user_get_users_by_field]], " + + "callLog=[]]]", + candidate.toTestString()); + } + + @Test + public void testFetchQuizzes() { + + DateTimeUtils.setCurrentMillisFixed(0); + + final MoodlePluginCourseAccess candidate = crateMockup(); + final FilterMap filterMap = new FilterMap(); + final AsyncQuizFetchBuffer asyncQuizFetchBuffer = new CourseAccessAPI.AsyncQuizFetchBuffer(); + candidate.fetchQuizzes(filterMap, asyncQuizFetchBuffer); + + assertFalse(asyncQuizFetchBuffer.canceled); + assertTrue(asyncQuizFetchBuffer.finished); + assertNull(asyncQuizFetchBuffer.error); + + assertEquals( + "MoodlePluginCourseAccess [pageSize=500, maxSize=10000, cutoffTimeOffset=3, " + + "restTemplate=MockupMoodleRestTemplate [accessToken=MockupMoodleRestTemplate-Test-Token, url=https://test.org/, " + + "testLog=[" + + "callMoodleAPIFunction: quizaccess_sebserver_get_exams, " + + "callMoodleAPIFunction: quizaccess_sebserver_get_exams], " + + "callLog=[" + + "= -94694400 or timecreated >=-94694400) and (enddate is null or enddate is 0 or enddate is >= -94694400)&startneedle=0&perpage=500,[Content-Type:\"application/x-www-form-urlencoded\"]>, " + + "= -94694400 or timecreated >=-94694400) and (enddate is null or enddate is 0 or enddate is >= -94694400)&startneedle=500&perpage=500,[Content-Type:\"application/x-www-form-urlencoded\"]>]]]", + candidate.toTestString()); + + final List ids = + asyncQuizFetchBuffer.buffer.stream().map(q -> q.id).sorted().collect(Collectors.toList()); + + assertEquals( + "[100:0:c0:i0, " + + "1010:10:c10:i10, " + + "1011:11:c11:i11, " + + "1012:12:c12:i12, " + + "1013:13:c13:i13, " + + "1014:14:c14:i14, " + + "1015:15:c15:i15, " + + "1016:16:c16:i16, " + + "1017:17:c17:i17, " + + "1018:18:c18:i18, " + + "101:1:c1:i1, " + + "102:2:c2:i2, " + + "103:3:c3:i3, " + + "104:4:c4:i4, " + + "105:5:c5:i5, " + + "106:6:c6:i6, " + + "107:7:c7:i7, " + + "108:8:c8:i8, " + + "109:9:c9:i9, " + + "1111:11:c11:i11, " + + "1113:13:c13:i13, " + + "1115:15:c15:i15, " + + "1117:17:c17:i17, " + + "111:1:c1:i1, " + + "113:3:c3:i3, " + + "115:5:c5:i5, " + + "117:7:c7:i7, " + + "119:9:c9:i9]", + ids.toString()); + + DateTimeUtils.setCurrentMillisSystem(); + } + + @Test + public void testFetchQuizzes_smallPaging() { + + DateTimeUtils.setCurrentMillisFixed(0); + + final Map params = new HashMap<>(); + params.put("sebserver.webservice.cache.moodle.course.pageSize", "5"); + + final MoodlePluginCourseAccess candidate = crateMockup(params); + final FilterMap filterMap = new FilterMap(); + final AsyncQuizFetchBuffer asyncQuizFetchBuffer = new CourseAccessAPI.AsyncQuizFetchBuffer(); + candidate.fetchQuizzes(filterMap, asyncQuizFetchBuffer); + + assertFalse(asyncQuizFetchBuffer.canceled); + assertTrue(asyncQuizFetchBuffer.finished); + assertNull(asyncQuizFetchBuffer.error); + + assertEquals( + "MoodlePluginCourseAccess [pageSize=5, maxSize=10000, cutoffTimeOffset=3, " + + "restTemplate=MockupMoodleRestTemplate [accessToken=MockupMoodleRestTemplate-Test-Token, url=https://test.org/, " + + "testLog=[" + + "callMoodleAPIFunction: quizaccess_sebserver_get_exams, " + + "callMoodleAPIFunction: quizaccess_sebserver_get_exams, " + + "callMoodleAPIFunction: quizaccess_sebserver_get_exams, " + + "callMoodleAPIFunction: quizaccess_sebserver_get_exams, " + + "callMoodleAPIFunction: quizaccess_sebserver_get_exams], " + + "callLog=[" + + "= -94694400 or timecreated >=-94694400) and (enddate is null or enddate is 0 or enddate is >= -94694400)&startneedle=0&perpage=5,[Content-Type:\"application/x-www-form-urlencoded\"]>, " + + "= -94694400 or timecreated >=-94694400) and (enddate is null or enddate is 0 or enddate is >= -94694400)&startneedle=5&perpage=5,[Content-Type:\"application/x-www-form-urlencoded\"]>, " + + "= -94694400 or timecreated >=-94694400) and (enddate is null or enddate is 0 or enddate is >= -94694400)&startneedle=10&perpage=5,[Content-Type:\"application/x-www-form-urlencoded\"]>, " + + "= -94694400 or timecreated >=-94694400) and (enddate is null or enddate is 0 or enddate is >= -94694400)&startneedle=15&perpage=5,[Content-Type:\"application/x-www-form-urlencoded\"]>, " + + "= -94694400 or timecreated >=-94694400) and (enddate is null or enddate is 0 or enddate is >= -94694400)&startneedle=20&perpage=5,[Content-Type:\"application/x-www-form-urlencoded\"]>]]]", + candidate.toTestString()); + + final List ids = + asyncQuizFetchBuffer.buffer.stream().map(q -> q.id).sorted().collect(Collectors.toList()); + + assertEquals( + "[100:0:c0:i0, " + + "1010:10:c10:i10, " + + "1011:11:c11:i11, " + + "1012:12:c12:i12, " + + "1013:13:c13:i13, " + + "1014:14:c14:i14, " + + "1015:15:c15:i15, " + + "1016:16:c16:i16, " + + "1017:17:c17:i17, " + + "1018:18:c18:i18, " + + "101:1:c1:i1, " + + "102:2:c2:i2, " + + "103:3:c3:i3, " + + "104:4:c4:i4, " + + "105:5:c5:i5, " + + "106:6:c6:i6, " + + "107:7:c7:i7, " + + "108:8:c8:i8, " + + "109:9:c9:i9, " + + "1111:11:c11:i11, " + + "1113:13:c13:i13, " + + "1115:15:c15:i15, " + + "1117:17:c17:i17, " + + "111:1:c1:i1, " + + "113:3:c3:i3, " + + "115:5:c5:i5, " + + "117:7:c7:i7, " + + "119:9:c9:i9]", + ids.toString()); + + DateTimeUtils.setCurrentMillisSystem(); + } + + @Test + public void testGetQuizzesForCourseIds() { + + DateTimeUtils.setCurrentMillisFixed(0); + final MoodlePluginCourseAccess candidate = crateMockup(); + + final Set ids = Stream.of( + "101:1:c1:i1", + "117:7:c7:i7") + .collect(Collectors.toSet()); + + final Result> quizzesCall = candidate.getQuizzes(ids); + + if (quizzesCall.hasError()) { + quizzesCall.getError().printStackTrace(); + } + + assertFalse(quizzesCall.hasError()); + final Collection quizzes = quizzesCall.get(); + assertNotNull(quizzes); + assertFalse(quizzes.isEmpty()); + assertTrue(quizzes.size() == 2); + final QuizData q1 = quizzes.iterator().next(); + assertEquals( + "QuizData [id=101:1:c1:i1, institutionId=1, lmsSetupId=1, lmsType=MOODLE_PLUGIN, name=c1 : quiz 101, description=quiz 101, startTime=1970-01-01T00:01:41.000Z, endTime=null, startURL=https://test.org/mod/quiz/view.php?id=101]", + q1.toString()); + + final Set idsGet = quizzes.stream().map(q -> q.id).collect(Collectors.toSet()); + + assertEquals(ids, idsGet); + + assertEquals( + "MoodlePluginCourseAccess [pageSize=500, maxSize=10000, cutoffTimeOffset=3, " + + "restTemplate=MockupMoodleRestTemplate [accessToken=MockupMoodleRestTemplate-Test-Token, url=https://test.org/, " + + "testLog=[callMoodleAPIFunction: quizaccess_sebserver_get_exams], " + + "callLog=[]]]", + candidate.toTestString()); + + DateTimeUtils.setCurrentMillisSystem(); + } + + private MoodlePluginCourseAccess crateMockup() { + return crateMockup(Collections.emptyMap()); + } + + private MoodlePluginCourseAccess crateMockup(final Map env) { + final JSONMapper jsonMapper = new JSONMapper(); + final AsyncService asyncService = new AsyncService(new AsyncRunner()); + final MockEnvironment mockEnvironment = new MockEnvironment(); + if (!env.isEmpty()) { + env.entrySet().stream().forEach(entry -> mockEnvironment.setProperty(entry.getKey(), entry.getValue())); + } + + final LmsSetup lmsSetup = + new LmsSetup(1L, 1L, "test-Moodle", LmsType.MOODLE_PLUGIN, "lms-user", "lms-user-secret", + "https://test.org/", null, null, null, null, null, null, null); + + final APITemplateDataSupplier apiTemplateDataSupplier = new APITemplateDataSupplier() { + + @Override + public LmsSetup getLmsSetup() { + return lmsSetup; + } + + @Override + public ClientCredentials getLmsClientCredentials() { + return new ClientCredentials("lms-user", "lms-user-secret"); + } + + @Override + public ProxyData getProxyData() { + return null; + } + + }; + + final MoodleMockupRestTemplateFactory moodleMockupRestTemplateFactory = + new MoodleMockupRestTemplateFactory(apiTemplateDataSupplier); + + return new MoodlePluginCourseAccess( + jsonMapper, + asyncService, + moodleMockupRestTemplateFactory, + new NoOpCacheManager(), + mockEnvironment); + } + +}