zervo/seb-server
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

SEBSERV-449 better timeouts and name search on moodle side

Browse source
This commit is contained in:
anhefti 2023-06-02 10:52:52 +02:00
parent 9d7ef0452f
commit 3ce025c4b1
5 changed files with 38 additions and 15 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
src/main/java/ch/ethz/seb/sebserver/ClientHttpRequestFactoryService.java
View file

@ -76,7 +76,7 @@ public class ClientHttpRequestFactoryService {
final ClientCredentialService clientCredentialService,
@Value("${sebserver.http.client.connect-timeout:15000}") final int connectTimeout,
@Value("${sebserver.http.client.connection-request-timeout:20000}") final int connectionRequestTimeout,
@Value("${sebserver.http.client.read-timeout:20000}") final int readTimeout) {
@Value("${sebserver.http.client.read-timeout:30000}") final int readTimeout) {
this.environment = environment;
this.clientCredentialService = clientCredentialService;

28
src/main/java/ch/ethz/seb/sebserver/webservice/servicelayer/lms/impl/QuizLookupServiceImpl.java
View file

@ -22,6 +22,7 @@ import java.util.stream.Collectors;
import org.joda.time.DateTime;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.annotation.Lazy;
import org.springframework.core.env.Environment;
import org.springframework.stereotype.Service;
@ -53,16 +54,19 @@ public class QuizLookupServiceImpl implements QuizLookupService {
private final UserService userService;
private final LmsSetupDAO lmsSetupDAO;
private final AsyncRunner asyncRunner;
private final long fetchedDataValiditySeconds;
public QuizLookupServiceImpl(
final UserService userService,
final LmsSetupDAO lmsSetupDAO,
final AsyncService asyncService,
final Environment environment) {
final Environment environment,
@Value("${sebserver.webservice.lms.datafetch.validity.seconds:600}") final long fetchedDataValiditySeconds) {
this.userService = userService;
this.lmsSetupDAO = lmsSetupDAO;
this.asyncRunner = asyncService.getAsyncRunner();
this.fetchedDataValiditySeconds = fetchedDataValiditySeconds;
}
@Override
@ -158,7 +162,10 @@ public class QuizLookupServiceImpl implements QuizLookupService {
}
if (!asyncLookup.isValid(filterMap)) {
this.lookups.remove(userId);
final AsyncLookup removed = this.lookups.remove(userId);
if (removed != null) {
removed.cancel();
}
this.createNewAsyncLookup(userId, filterMap, lmsAPITemplateSupplier);
}
@ -198,7 +205,12 @@ public class QuizLookupServiceImpl implements QuizLookupService {
}
final LookupFilterCriteria criteria = new LookupFilterCriteria(filterMap);
final AsyncLookup asyncLookup = new AsyncLookup(userInstitutionId, userId, criteria, buffers);
final AsyncLookup asyncLookup = new AsyncLookup(
userInstitutionId,
userId,
criteria,
buffers,
this.fetchedDataValiditySeconds);
if (log.isDebugEnabled()) {
log.debug("Create new AsyncLookup: user={} criteria={}", userId, criteria);
@ -278,18 +290,21 @@ public class QuizLookupServiceImpl implements QuizLookupService {
final Collection<AsyncQuizFetchBuffer> asyncBuffers;
final long timeCreated;
long timeCompleted = Long.MAX_VALUE;
private final long fetchedDataValiditySeconds;
public AsyncLookup(
final long institutionId,
final String userId,
final LookupFilterCriteria lookupFilterCriteria,
final Collection<AsyncQuizFetchBuffer> asyncBuffers) {
final Collection<AsyncQuizFetchBuffer> asyncBuffers,
final long fetchedDataValiditySeconds) {
this.institutionId = institutionId;
this.userId = userId;
this.lookupFilterCriteria = lookupFilterCriteria;
this.asyncBuffers = asyncBuffers;
this.timeCreated = Utils.getMillisecondsNow();
this.fetchedDataValiditySeconds = fetchedDataValiditySeconds;
}
LookupResult getAvailable() {
@ -307,10 +322,7 @@ public class QuizLookupServiceImpl implements QuizLookupService {
boolean isUpToDate() {
final long now = Utils.getMillisecondsNow();
if (now - this.timeCreated > 5 * Constants.MINUTE_IN_MILLIS) {
return false;
}
if (now - this.timeCompleted > Constants.MINUTE_IN_MILLIS) {
if (now - this.timeCreated > this.fetchedDataValiditySeconds * Constants.SECOND_IN_MILLIS) {
return false;
}
return true;

18
src/main/java/ch/ethz/seb/sebserver/webservice/servicelayer/lms/impl/moodle/plugin/MoodlePluginCourseAccess.java
View file

@ -61,6 +61,7 @@ import ch.ethz.seb.sebserver.webservice.servicelayer.lms.impl.moodle.MoodleUtils
import ch.ethz.seb.sebserver.webservice.servicelayer.lms.impl.moodle.MoodleUtils.Courses;
import ch.ethz.seb.sebserver.webservice.servicelayer.lms.impl.moodle.MoodleUtils.CoursesPlugin;
import ch.ethz.seb.sebserver.webservice.servicelayer.lms.impl.moodle.MoodleUtils.MoodleUserDetails;
import io.micrometer.core.instrument.util.StringUtils;
public class MoodlePluginCourseAccess extends AbstractCachedCourseAccess implements CourseAccessAPI {
@ -118,7 +119,7 @@ public class MoodlePluginCourseAccess extends AbstractCachedCourseAccess impleme
environment.getProperty(
"sebserver.webservice.circuitbreaker.moodleRestCall.blockingTime",
Long.class,
Constants.SECOND_IN_MILLIS * 20),
Constants.SECOND_IN_MILLIS * 30),
environment.getProperty(
"sebserver.webservice.circuitbreaker.moodleRestCall.timeToRecover",
Long.class,
@ -184,10 +185,11 @@ public class MoodlePluginCourseAccess extends AbstractCachedCourseAccess impleme
quizFromTime = DateTime.now(DateTimeZone.UTC).minusYears(this.cutoffTimeOffset);
}
final Predicate<QuizData> quizFilter = LmsAPIService.quizFilterPredicate(filterMap);
final String quizName = filterMap.getQuizName();
while (!asyncQuizFetchBuffer.finished && !asyncQuizFetchBuffer.canceled) {
try {
fetchQuizzesPage(page, quizFromTime, asyncQuizFetchBuffer, quizFilter);
fetchQuizzesPage(page, quizFromTime, quizName, asyncQuizFetchBuffer, quizFilter);
page++;
} catch (final Exception e) {
log.error("Unexpected error while trying to fetch moodle quiz page: {}", page, e);
@ -371,6 +373,7 @@ public class MoodlePluginCourseAccess extends AbstractCachedCourseAccess impleme
private void fetchQuizzesPage(
final int page,
final DateTime quizFromTime,
final String nameCondition,
final AsyncQuizFetchBuffer asyncQuizFetchBuffer,
final Predicate<QuizData> quizFilter) throws JsonParseException, JsonMappingException, IOException {
@ -382,7 +385,7 @@ public class MoodlePluginCourseAccess extends AbstractCachedCourseAccess impleme
: lmsSetup.lmsApiUrl + Constants.URL_PATH_SEPARATOR + MOODLE_QUIZ_START_URL_PATH;
final Collection<CourseData> fetchCoursesPage =
fetchCoursesPage(restTemplate, quizFromTime, page, this.pageSize);
fetchCoursesPage(restTemplate, quizFromTime, nameCondition, page, this.pageSize);
// finish if page is empty (no courses left
if (fetchCoursesPage.isEmpty()) {
asyncQuizFetchBuffer.finish();
@ -408,6 +411,7 @@ public class MoodlePluginCourseAccess extends AbstractCachedCourseAccess impleme
private Collection<CourseData> fetchCoursesPage(
final MoodleAPIRestTemplate restTemplate,
final DateTime quizFromTime,
final String nameCondition,
final int page,
final int size) throws JsonParseException, JsonMappingException, IOException {
@ -422,13 +426,19 @@ public class MoodlePluginCourseAccess extends AbstractCachedCourseAccess impleme
final long defaultCutOff = Utils.toUnixTimeInSeconds(
DateTime.now(DateTimeZone.UTC).minusYears(this.cutoffTimeOffset));
final long cutoffDate = (filterDate < defaultCutOff) ? filterDate : defaultCutOff;
final String sqlCondition = String.format(
String sqlCondition = String.format(
SQL_CONDITION_TEMPLATE,
String.valueOf(cutoffDate),
String.valueOf(filterDate));
final String fromElement = String.valueOf(page * size);
final LinkedMultiValueMap<String, String> attributes = new LinkedMultiValueMap<>();
if (StringUtils.isNotBlank(nameCondition)) {
sqlCondition = sqlCondition + " AND (m.name LIKE '" +
Utils.toSQLWildcard(nameCondition) +
"')";
}
// Note: courseid[]=0 means all courses. Moodle don't like empty parameter
attributes.add(PARAM_COURSE_ID_ARRAY, "0");
attributes.add(PARAM_SQL_CONDITIONS, sqlCondition);

4
src/main/resources/config/application-dev-ws.properties
View file

@ -18,7 +18,7 @@ spring.datasource.hikari.leakDetectionThreshold=2000
sebserver.http.client.connect-timeout=15000
sebserver.http.client.connection-request-timeout=10000
sebserver.http.client.read-timeout=20000
sebserver.http.client.read-timeout=30000
sebserver.webservice.distributed.updateInterval=1000
sebserver.webservice.distributed.connectionUpdate=2000
sebserver.webservice.clean-db-on-startup=false
@ -52,7 +52,7 @@ sebserver.webservice.api.pagination.maxPageSize=500
sebserver.webservice.lms.openedx.api.token.request.paths=/oauth2/access_token
sebserver.webservice.lms.moodle.api.token.request.paths=
sebserver.webservice.lms.address.alias=lms.mockup.com=lms.address.alias
sebserver.webservice.cache.moodle.course.pageSize=10
sebserver.webservice.cache.moodle.course.pageSize=250
springdoc.api-docs.enabled=true
springdoc.swagger-ui.enabled=true

1
src/main/resources/config/application-ws.properties
View file

@ -83,6 +83,7 @@ sebserver.webservice.lms.moodle.prependShortCourseName=true
sebserver.webservice.lms.moodle.fetch.cutoffdate.yearsBeforeNow=2
sebserver.webservice.lms.olat.sendAdditionalAttributesWithRestriction=false
sebserver.webservice.lms.address.alias=
sebserver.webservice.lms.datafetch.validity.seconds=600
sebserver.webservice.proctoring.resetBroadcastOnLeav=true
sebserver.webservice.proctoring.zoom.enableWaitingRoom=false