Merge remote-tracking branch 'origin/SEBSERV-449_MoodlePluginLuca' into

dev-1.5

src/main/java/ch/ethz/seb/sebserver/ClientHttpRequestFactoryService.java

@@ -76,7 +76,7 @@ public class ClientHttpRequestFactoryService {
             final ClientCredentialService clientCredentialService,
             @Value("${sebserver.http.client.connect-timeout:15000}") final int connectTimeout,
             @Value("${sebserver.http.client.connection-request-timeout:20000}") final int connectionRequestTimeout,
-            @Value("${sebserver.http.client.read-timeout:20000}") final int readTimeout) {
+            @Value("${sebserver.http.client.read-timeout:30000}") final int readTimeout) {
 
         this.environment = environment;
         this.clientCredentialService = clientCredentialService;

src/main/java/ch/ethz/seb/sebserver/gui/content/exam/QuizLookupList.java

@@ -59,6 +59,7 @@ import ch.ethz.seb.sebserver.gui.table.ColumnDefinition.TableFilterAttribute;
 import ch.ethz.seb.sebserver.gui.table.EntityTable;
 import ch.ethz.seb.sebserver.gui.table.TableFilter.CriteriaType;
 import ch.ethz.seb.sebserver.gui.widget.WidgetFactory;
+import ch.ethz.seb.sebserver.gui.widget.WidgetFactory.ImageIcon;
 
 @Lazy
 @Component
@@ -109,6 +110,8 @@ public class QuizLookupList implements TemplateComposer {
             new LocTextKey("sebserver.quizdiscovery.quiz.import.existing");
     private final static LocTextKey TEXT_FETCH_NOTE =
             new LocTextKey("sebserver.quizdiscovery.list.fetchnote");
+    private final static LocTextKey TEXT_FETCH_NOTE_TOOLTIP =
+            new LocTextKey("sebserver.quizdiscovery.list.fetchnote.tooltip");
 
     private final static String TEXT_KEY_ADDITIONAL_ATTR_PREFIX =
             "sebserver.quizdiscovery.quiz.details.additional.";
@@ -448,7 +451,7 @@ public class QuizLookupList implements TemplateComposer {
         }
     }
 
-    private boolean showingFetchNote = false;
+    private Composite warningPanel = null;
 
     private void handelPageReload(
             final Composite notePanel,
@@ -456,29 +459,28 @@ public class QuizLookupList implements TemplateComposer {
 
         if (table.isComplete()) {
             PageService.clearComposite(notePanel);
-            this.showingFetchNote = false;
+            if (this.warningPanel != null) {
-        } else {
+                this.warningPanel.dispose();
-            if (!this.showingFetchNote) {
-                final Composite warningPanel = this.widgetFactory.createWarningPanel(notePanel, 15, true);
-                GridData gridData = new GridData(SWT.CENTER, SWT.CENTER, false, true);
-                gridData.heightHint = 28;
-                gridData.widthHint = 25;
-                gridData.verticalIndent = 5;
-                final Label action = new Label(warningPanel, SWT.NONE);
-                action.setImage(WidgetFactory.ImageIcon.SWITCH.getImage(notePanel.getDisplay()));
-                action.setLayoutData(gridData);
-                action.addListener(SWT.MouseDown, event -> {
-                    table.applyFilter();
-                });
-
-                final Label text = new Label(warningPanel, SWT.NONE);
-                text.setData(RWT.MARKUP_ENABLED, Boolean.TRUE);
-                text.setText(this.pageService.getI18nSupport().getText(TEXT_FETCH_NOTE));
-                gridData = new GridData(SWT.LEFT, SWT.FILL, true, true);
-                gridData.heightHint = 16;
-                text.setLayoutData(gridData);
-                this.showingFetchNote = true;
             }
+            this.warningPanel = null;
+        } else {
+            if (this.warningPanel != null && !this.warningPanel.isDisposed()) {
+                this.warningPanel.dispose();
+            }
+
+            this.warningPanel = this.widgetFactory.createWarningPanel(notePanel, 15, true);
+            this.widgetFactory.imageButton(
+                    ImageIcon.SWITCH,
+                    this.warningPanel,
+                    TEXT_FETCH_NOTE_TOOLTIP,
+                    event -> table.applyFilter());
+
+            final Label text = new Label(this.warningPanel, SWT.NONE);
+            text.setData(RWT.MARKUP_ENABLED, Boolean.TRUE);
+            text.setText(this.pageService.getI18nSupport().getText(TEXT_FETCH_NOTE));
+            final GridData gridData = new GridData(SWT.LEFT, SWT.FILL, true, true);
+            gridData.heightHint = 28;
+            text.setLayoutData(gridData);
         }
         notePanel.getParent().layout(true, true);
     }

src/main/java/ch/ethz/seb/sebserver/gui/content/monitoring/MonitoringRunningExam.java

@@ -154,7 +154,7 @@ public class MonitoringRunningExam implements TemplateComposer {
                 pageContext.getParent(),
                 new LocTextKey(
                         "sebserver.monitoring.exam",
-                        StringEscapeUtils.escapeHtml4(exam.name)));
+                        StringEscapeUtils.escapeXml11(exam.name)));
 
         final Composite tablePane = new Composite(content, SWT.NONE);
         tablePane.setLayout(new GridLayout());

src/main/java/ch/ethz/seb/sebserver/gui/service/i18n/impl/PolyglotPageServiceImpl.java

@@ -270,7 +270,16 @@ public final class PolyglotPageServiceImpl implements PolyglotPageService {
 
         return label -> {
             if (locTextKey != null) {
-                label.setText(i18nSupport.getText(locTextKey));
+                try {
+                    label.setText(i18nSupport.getText(locTextKey));
+                } catch (final Exception e) {
+                    label.setData(RWT.MARKUP_ENABLED, false);
+                    try {
+                        label.setText(i18nSupport.getText(locTextKey));
+                    } catch (final Exception ee) {
+                        label.setText(locTextKey.name);
+                    }
+                }
             }
             if (i18nSupport.hasText(locToolTipKey)) {
                 label.setToolTipText(Utils.formatLineBreaks(i18nSupport.getText(locToolTipKey)));

src/main/java/ch/ethz/seb/sebserver/webservice/servicelayer/lms/impl/QuizLookupServiceImpl.java

@@ -22,6 +22,7 @@ import java.util.stream.Collectors;
 import org.joda.time.DateTime;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.annotation.Value;
 import org.springframework.context.annotation.Lazy;
 import org.springframework.core.env.Environment;
 import org.springframework.stereotype.Service;
@@ -53,16 +54,19 @@ public class QuizLookupServiceImpl implements QuizLookupService {
     private final UserService userService;
     private final LmsSetupDAO lmsSetupDAO;
     private final AsyncRunner asyncRunner;
+    private final long fetchedDataValiditySeconds;
 
     public QuizLookupServiceImpl(
             final UserService userService,
             final LmsSetupDAO lmsSetupDAO,
             final AsyncService asyncService,
-            final Environment environment) {
+            final Environment environment,
+            @Value("${sebserver.webservice.lms.datafetch.validity.seconds:600}") final long fetchedDataValiditySeconds) {
 
         this.userService = userService;
         this.lmsSetupDAO = lmsSetupDAO;
         this.asyncRunner = asyncService.getAsyncRunner();
+        this.fetchedDataValiditySeconds = fetchedDataValiditySeconds;
     }
 
     @Override
@@ -158,7 +162,10 @@ public class QuizLookupServiceImpl implements QuizLookupService {
         }
 
         if (!asyncLookup.isValid(filterMap)) {
-            this.lookups.remove(userId);
+            final AsyncLookup removed = this.lookups.remove(userId);
+            if (removed != null) {
+                removed.cancel();
+            }
             this.createNewAsyncLookup(userId, filterMap, lmsAPITemplateSupplier);
         }
 
@@ -198,7 +205,12 @@ public class QuizLookupServiceImpl implements QuizLookupService {
             }
 
             final LookupFilterCriteria criteria = new LookupFilterCriteria(filterMap);
-            final AsyncLookup asyncLookup = new AsyncLookup(userInstitutionId, userId, criteria, buffers);
+            final AsyncLookup asyncLookup = new AsyncLookup(
+                    userInstitutionId,
+                    userId,
+                    criteria,
+                    buffers,
+                    this.fetchedDataValiditySeconds);
 
             if (log.isDebugEnabled()) {
                 log.debug("Create new AsyncLookup: user={} criteria={}", userId, criteria);
@@ -278,18 +290,21 @@ public class QuizLookupServiceImpl implements QuizLookupService {
         final Collection<AsyncQuizFetchBuffer> asyncBuffers;
         final long timeCreated;
         long timeCompleted = Long.MAX_VALUE;
+        private final long fetchedDataValiditySeconds;
 
         public AsyncLookup(
                 final long institutionId,
                 final String userId,
                 final LookupFilterCriteria lookupFilterCriteria,
-                final Collection<AsyncQuizFetchBuffer> asyncBuffers) {
+                final Collection<AsyncQuizFetchBuffer> asyncBuffers,
+                final long fetchedDataValiditySeconds) {
 
             this.institutionId = institutionId;
             this.userId = userId;
             this.lookupFilterCriteria = lookupFilterCriteria;
             this.asyncBuffers = asyncBuffers;
             this.timeCreated = Utils.getMillisecondsNow();
+            this.fetchedDataValiditySeconds = fetchedDataValiditySeconds;
         }
 
         LookupResult getAvailable() {
@@ -307,10 +322,7 @@ public class QuizLookupServiceImpl implements QuizLookupService {
 
         boolean isUpToDate() {
             final long now = Utils.getMillisecondsNow();
-            if (now - this.timeCreated > 5 * Constants.MINUTE_IN_MILLIS) {
+            if (now - this.timeCreated > this.fetchedDataValiditySeconds * Constants.SECOND_IN_MILLIS) {
-                return false;
-            }
-            if (now - this.timeCompleted > Constants.MINUTE_IN_MILLIS) {
                 return false;
             }
             return true;

src/main/java/ch/ethz/seb/sebserver/webservice/servicelayer/lms/impl/mockup/MockCourseAccessAPI.java

@@ -63,7 +63,8 @@ public class MockCourseAccessAPI implements CourseAccessAPI {
                 "quiz1", institutionId, lmsSetupId, lmsType, "Demo Quiz 1 (MOCKUP)", "<p>Demo Quiz Mockup</p>",
                 "2020-01-01T09:00:00Z", null, "http://lms.mockup.com/api/"));
         this.mockups.add(new QuizData(
-                "quiz2", institutionId, lmsSetupId, lmsType, "Demo Quiz 2 (MOCKUP)", "<p>Demo Quiz Mockup</p>",
+                "quiz2", institutionId, lmsSetupId, lmsType, "Demo Quiz 2 (MOCKUP) äöüèÜÄÖ ?< ",
+                "<p>Demo Quiz Mockup</p>",
                 "2020-01-01T09:00:00Z", "2025-01-01T09:00:00Z", "http://lms.mockup.com/api/"));
         this.mockups.add(new QuizData(
                 "quiz3", institutionId, lmsSetupId, lmsType, "Demo Quiz 3 (MOCKUP)", "<p>Demo Quiz Mockup</p>",

src/main/java/ch/ethz/seb/sebserver/webservice/servicelayer/lms/impl/moodle/plugin/MoodlePluginCourseAccess.java

@@ -61,6 +61,7 @@ import ch.ethz.seb.sebserver.webservice.servicelayer.lms.impl.moodle.MoodleUtils
 import ch.ethz.seb.sebserver.webservice.servicelayer.lms.impl.moodle.MoodleUtils.Courses;
 import ch.ethz.seb.sebserver.webservice.servicelayer.lms.impl.moodle.MoodleUtils.CoursesPlugin;
 import ch.ethz.seb.sebserver.webservice.servicelayer.lms.impl.moodle.MoodleUtils.MoodleUserDetails;
+import io.micrometer.core.instrument.util.StringUtils;
 
 public class MoodlePluginCourseAccess extends AbstractCachedCourseAccess implements CourseAccessAPI {
 
@@ -92,6 +93,7 @@ public class MoodlePluginCourseAccess extends AbstractCachedCourseAccess impleme
     private final int pageSize;
     private final int maxSize;
     private final int cutoffTimeOffset;
+    private final boolean applyNameCriteria;
 
     private MoodleAPIRestTemplate restTemplate;
 
@@ -100,11 +102,13 @@ public class MoodlePluginCourseAccess extends AbstractCachedCourseAccess impleme
             final AsyncService asyncService,
             final MoodleRestTemplateFactory restTemplateFactory,
             final CacheManager cacheManager,
-            final Environment environment) {
+            final Environment environment,
+            final boolean applyNameCriteria) {
 
         super(cacheManager);
         this.jsonMapper = jsonMapper;
         this.restTemplateFactory = restTemplateFactory;
+        this.applyNameCriteria = applyNameCriteria;
 
         this.prependShortCourseName = BooleanUtils.toBoolean(environment.getProperty(
                 "sebserver.webservice.lms.moodle.prependShortCourseName",
@@ -118,7 +122,7 @@ public class MoodlePluginCourseAccess extends AbstractCachedCourseAccess impleme
                 environment.getProperty(
                         "sebserver.webservice.circuitbreaker.moodleRestCall.blockingTime",
                         Long.class,
-                        Constants.SECOND_IN_MILLIS * 20),
+                        Constants.SECOND_IN_MILLIS * 30),
                 environment.getProperty(
                         "sebserver.webservice.circuitbreaker.moodleRestCall.timeToRecover",
                         Long.class,
@@ -179,10 +183,11 @@ public class MoodlePluginCourseAccess extends AbstractCachedCourseAccess impleme
                 quizFromTime = DateTime.now(DateTimeZone.UTC).minusYears(this.cutoffTimeOffset);
             }
             final Predicate<QuizData> quizFilter = LmsAPIService.quizFilterPredicate(filterMap);
+            final String quizName = filterMap.getQuizName();
 
             while (!asyncQuizFetchBuffer.finished && !asyncQuizFetchBuffer.canceled) {
                 try {
-                    fetchQuizzesPage(page, quizFromTime, asyncQuizFetchBuffer, quizFilter);
+                    fetchQuizzesPage(page, quizFromTime, quizName, asyncQuizFetchBuffer, quizFilter);
                     page++;
                 } catch (final Exception e) {
                     log.error("Unexpected error while trying to fetch moodle quiz page: {}", page, e);
@@ -366,6 +371,7 @@ public class MoodlePluginCourseAccess extends AbstractCachedCourseAccess impleme
     private void fetchQuizzesPage(
             final int page,
             final DateTime quizFromTime,
+            final String nameCondition,
             final AsyncQuizFetchBuffer asyncQuizFetchBuffer,
             final Predicate<QuizData> quizFilter) throws JsonParseException, JsonMappingException, IOException {
 
@@ -377,7 +383,7 @@ public class MoodlePluginCourseAccess extends AbstractCachedCourseAccess impleme
                 : lmsSetup.lmsApiUrl + Constants.URL_PATH_SEPARATOR + MOODLE_QUIZ_START_URL_PATH;
 
         final Collection<CourseData> fetchCoursesPage =
-                fetchCoursesPage(restTemplate, quizFromTime, page, this.pageSize);
+                fetchCoursesPage(restTemplate, quizFromTime, nameCondition, page, this.pageSize);
         // finish if page is empty (no courses left
         if (fetchCoursesPage.isEmpty()) {
             asyncQuizFetchBuffer.finish();
@@ -403,6 +409,7 @@ public class MoodlePluginCourseAccess extends AbstractCachedCourseAccess impleme
     private Collection<CourseData> fetchCoursesPage(
             final MoodleAPIRestTemplate restTemplate,
             final DateTime quizFromTime,
+            final String nameCondition,
             final int page,
             final int size) throws JsonParseException, JsonMappingException, IOException {
 
@@ -417,13 +424,21 @@ public class MoodlePluginCourseAccess extends AbstractCachedCourseAccess impleme
             final long defaultCutOff = Utils.toUnixTimeInSeconds(
                     DateTime.now(DateTimeZone.UTC).minusYears(this.cutoffTimeOffset));
             final long cutoffDate = (filterDate < defaultCutOff) ? filterDate : defaultCutOff;
-            final String sqlCondition = String.format(
+            String sqlCondition = String.format(
                     SQL_CONDITION_TEMPLATE,
                     String.valueOf(cutoffDate),
                     String.valueOf(filterDate));
             final String fromElement = String.valueOf(page * size);
             final LinkedMultiValueMap<String, String> attributes = new LinkedMultiValueMap<>();
 
+            // TODO clarify with Amr and Luca if this is OK
+            //      and if it is possible to apply the nameCondition also the the course name (shortname)
+            if (this.applyNameCriteria && StringUtils.isNotBlank(nameCondition)) {
+                sqlCondition = sqlCondition + " AND (m.name LIKE '" +
+                        Utils.toSQLWildcard(nameCondition) +
+                        "')";
+            }
+
             // Note: courseid[]=0 means all courses. Moodle don't like empty parameter
             attributes.add(PARAM_COURSE_ID_ARRAY, "0");
             attributes.add(PARAM_SQL_CONDITIONS, sqlCondition);

src/main/java/ch/ethz/seb/sebserver/webservice/servicelayer/lms/impl/moodle/plugin/MooldePluginLmsAPITemplateFactory.java

@@ -46,6 +46,7 @@ public class MooldePluginLmsAPITemplateFactory implements LmsAPITemplateFactory
     private final ExamConfigurationValueService examConfigurationValueService;
     private final ClientHttpRequestFactoryService clientHttpRequestFactoryService;
     private final String[] alternativeTokenRequestPaths;
+    private final boolean applyNameCriteria;
 
     protected MooldePluginLmsAPITemplateFactory(
             final JSONMapper jsonMapper,
@@ -55,7 +56,8 @@ public class MooldePluginLmsAPITemplateFactory implements LmsAPITemplateFactory
             final ClientCredentialService clientCredentialService,
             final ExamConfigurationValueService examConfigurationValueService,
             final ClientHttpRequestFactoryService clientHttpRequestFactoryService,
-            @Value("${sebserver.webservice.lms.moodle.api.token.request.paths:}") final String alternativeTokenRequestPaths) {
+            @Value("${sebserver.webservice.lms.moodle.api.token.request.paths:}") final String alternativeTokenRequestPaths,
+            @Value("${sebserver.webservice.lms.moodle.fetch.applyNameCriteria:false}") final boolean applyNameCriteria) {
 
         this.jsonMapper = jsonMapper;
         this.cacheManager = cacheManager;
@@ -67,6 +69,7 @@ public class MooldePluginLmsAPITemplateFactory implements LmsAPITemplateFactory
         this.alternativeTokenRequestPaths = (alternativeTokenRequestPaths != null)
                 ? StringUtils.split(alternativeTokenRequestPaths, Constants.LIST_SEPARATOR)
                 : null;
+        this.applyNameCriteria = applyNameCriteria;
     }
 
     @Override
@@ -90,7 +93,8 @@ public class MooldePluginLmsAPITemplateFactory implements LmsAPITemplateFactory
                     this.asyncService,
                     moodleRestTemplateFactory,
                     this.cacheManager,
-                    this.environment);
+                    this.environment,
+                    this.applyNameCriteria);
 
             final MoodlePluginCourseRestriction moodlePluginCourseRestriction = new MoodlePluginCourseRestriction(
                     this.jsonMapper,

src/main/resources/config/application-dev-ws.properties

@@ -18,7 +18,7 @@ spring.datasource.hikari.leakDetectionThreshold=2000
 
 sebserver.http.client.connect-timeout=15000
 sebserver.http.client.connection-request-timeout=10000
-sebserver.http.client.read-timeout=20000
+sebserver.http.client.read-timeout=30000
 sebserver.webservice.distributed.updateInterval=1000
 sebserver.webservice.distributed.connectionUpdate=2000
 sebserver.webservice.clean-db-on-startup=false
@@ -52,7 +52,7 @@ sebserver.webservice.api.pagination.maxPageSize=500
 sebserver.webservice.lms.openedx.api.token.request.paths=/oauth2/access_token
 sebserver.webservice.lms.moodle.api.token.request.paths=
 sebserver.webservice.lms.address.alias=lms.mockup.com=lms.address.alias
-sebserver.webservice.cache.moodle.course.pageSize=10
+sebserver.webservice.cache.moodle.course.pageSize=250
 
 springdoc.api-docs.enabled=true
 springdoc.swagger-ui.enabled=true

src/main/resources/config/application-ws.properties

@@ -81,8 +81,10 @@ sebserver.webservice.lms.openedx.api.token.request.paths=/oauth2/access_token
 sebserver.webservice.lms.moodle.api.token.request.paths=/login/token.php
 sebserver.webservice.lms.moodle.prependShortCourseName=true
 sebserver.webservice.lms.moodle.fetch.cutoffdate.yearsBeforeNow=2
+sebserver.webservice.lms.moodle.fetch.applyNameCriteria=false
 sebserver.webservice.lms.olat.sendAdditionalAttributesWithRestriction=false
 sebserver.webservice.lms.address.alias=
+sebserver.webservice.lms.datafetch.validity.seconds=600
 
 sebserver.webservice.proctoring.resetBroadcastOnLeav=true
 sebserver.webservice.proctoring.zoom.enableWaitingRoom=false

src/main/resources/config/application.properties

@@ -19,7 +19,7 @@ server.tomcat.socket.soKeepAlive=true
 server.tomcat.socket.performanceConnectionTime=1
 server.tomcat.socket.performanceLatency=2
 server.tomcat.socket.performanceBandwidth=0
-server.tomcat.keepAliveTimeout(20000);
+server.tomcat.keepAliveTimeout(3000);
 server.tomcat.maxKeepAliveRequests(3000);
 server.tomcat.uri-encoding=UTF-8
 

src/main/resources/messages.properties

@@ -448,6 +448,7 @@ sebserver.quizdiscovery.list.column.endtime.tooltip=The end time of the LMS exam
 sebserver.quizdiscovery.info.pleaseSelect=At first please select an LMS exam from the list
 sebserver.quizdiscovery.list.action.no.modify.privilege=No Access: A LMS exam from other institution cannot be imported.
 sebserver.quizdiscovery.list.fetchnote=<b>Note:</b> This list is not complete yet since the service is still fetching data from LMS.<br/>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;Use the reload button on the left or the search icon from the list for update.
+sebserver.quizdiscovery.list.fetchnote.tooltip=Click to reload the list and get all currently fetched results.
 
 sebserver.quizdiscovery.action.list=LMS Exam Lookup
 sebserver.quizdiscovery.action.import=Import as Exam

src/test/java/ch/ethz/seb/sebserver/webservice/servicelayer/lms/impl/moodle/plugin/MoodlePluginCourseAccessTest.java

@@ -324,7 +324,8 @@ public class MoodlePluginCourseAccessTest {
                 asyncService,
                 moodleMockupRestTemplateFactory,
                 new NoOpCacheManager(),
-                mockEnvironment);
+                mockEnvironment,
+                false);
     }
 
 }