SEBSERV-233 better handling with circuit breaker

This commit is contained in:
anhefti 2022-04-27 17:03:22 +02:00
parent d2ea6eb316
commit 0dfde290ca
5 changed files with 59 additions and 14 deletions

View file

@ -733,7 +733,9 @@ public class ExamDAOImpl implements ExamDAO {
log.debug("Quizzes size mismatch detected by getting exams quiz data from LMS: {}", lmsSetup); log.debug("Quizzes size mismatch detected by getting exams quiz data from LMS: {}", lmsSetup);
} }
if (lmsSetup.testCourseAccessAPI().hasAnyError()) { try {
lmsSetup.checkCourseAPIAccess();
} catch (final Exception e) {
// No course access on the LMS. This means we can't get any quizzes from this LMSSetup at the moment // No course access on the LMS. This means we can't get any quizzes from this LMSSetup at the moment
// All exams are marked as corrupt because of LMS Setup failure // All exams are marked as corrupt because of LMS Setup failure

View file

@ -42,6 +42,14 @@ public interface CourseAccessAPI {
* @return {@link LmsSetupTestResult } instance with the test result report */ * @return {@link LmsSetupTestResult } instance with the test result report */
LmsSetupTestResult testCourseAccessAPI(); LmsSetupTestResult testCourseAccessAPI();
/** To make a quick course API access check without report that just throws an exception if not available */
default void checkCourseAPIAccess() {
final LmsSetupTestResult testCourseAccessAPI = this.testCourseAccessAPI();
if (!testCourseAccessAPI.isOk()) {
throw new RuntimeException("No course API Access: " + testCourseAccessAPI);
}
}
/** Get an unsorted List of filtered {@link QuizData } from the LMS course/quiz API /** Get an unsorted List of filtered {@link QuizData } from the LMS course/quiz API
* *
* @param filterMap the {@link FilterMap } to get a filtered result. Possible filter attributes are: * @param filterMap the {@link FilterMap } to get a filtered result. Possible filter attributes are:

View file

@ -43,6 +43,8 @@ public class LmsAPITemplateAdapter implements LmsAPITemplate {
private final SEBRestrictionAPI sebBestrictionAPI; private final SEBRestrictionAPI sebBestrictionAPI;
private final APITemplateDataSupplier apiTemplateDataSupplier; private final APITemplateDataSupplier apiTemplateDataSupplier;
/** CircuitBreaker for protected lmsTestRequest */
private final CircuitBreaker<LmsSetupTestResult> lmsTestRequest;
/** CircuitBreaker for protected quiz and course data requests */ /** CircuitBreaker for protected quiz and course data requests */
private final CircuitBreaker<List<QuizData>> allQuizzesRequest; private final CircuitBreaker<List<QuizData>> allQuizzesRequest;
/** CircuitBreaker for protected quiz and course data requests */ /** CircuitBreaker for protected quiz and course data requests */
@ -68,15 +70,29 @@ public class LmsAPITemplateAdapter implements LmsAPITemplate {
this.sebBestrictionAPI = sebBestrictionAPI; this.sebBestrictionAPI = sebBestrictionAPI;
this.apiTemplateDataSupplier = apiTemplateDataSupplier; this.apiTemplateDataSupplier = apiTemplateDataSupplier;
this.lmsTestRequest = asyncService.createCircuitBreaker(
environment.getProperty(
"sebserver.webservice.circuitbreaker.lmsTestRequest.attempts",
Integer.class,
2),
environment.getProperty(
"sebserver.webservice.circuitbreaker.lmsTestRequest.blockingTime",
Long.class,
Constants.SECOND_IN_MILLIS * 20),
environment.getProperty(
"sebserver.webservice.circuitbreaker.lmsTestRequest.timeToRecover",
Long.class,
Constants.MINUTE_IN_MILLIS));
this.allQuizzesRequest = asyncService.createCircuitBreaker( this.allQuizzesRequest = asyncService.createCircuitBreaker(
environment.getProperty( environment.getProperty(
"sebserver.webservice.circuitbreaker.quizzesRequest.attempts", "sebserver.webservice.circuitbreaker.quizzesRequest.attempts",
Integer.class, Integer.class,
3), 1),
environment.getProperty( environment.getProperty(
"sebserver.webservice.circuitbreaker.quizzesRequest.blockingTime", "sebserver.webservice.circuitbreaker.quizzesRequest.blockingTime",
Long.class, Long.class,
Constants.MINUTE_IN_MILLIS), Constants.SECOND_IN_MILLIS * 20),
environment.getProperty( environment.getProperty(
"sebserver.webservice.circuitbreaker.quizzesRequest.timeToRecover", "sebserver.webservice.circuitbreaker.quizzesRequest.timeToRecover",
Long.class, Long.class,
@ -86,7 +102,7 @@ public class LmsAPITemplateAdapter implements LmsAPITemplate {
environment.getProperty( environment.getProperty(
"sebserver.webservice.circuitbreaker.quizzesRequest.attempts", "sebserver.webservice.circuitbreaker.quizzesRequest.attempts",
Integer.class, Integer.class,
3), 1),
environment.getProperty( environment.getProperty(
"sebserver.webservice.circuitbreaker.quizzesRequest.blockingTime", "sebserver.webservice.circuitbreaker.quizzesRequest.blockingTime",
Long.class, Long.class,
@ -100,7 +116,7 @@ public class LmsAPITemplateAdapter implements LmsAPITemplate {
environment.getProperty( environment.getProperty(
"sebserver.webservice.circuitbreaker.quizzesRequest.attempts", "sebserver.webservice.circuitbreaker.quizzesRequest.attempts",
Integer.class, Integer.class,
3), 1),
environment.getProperty( environment.getProperty(
"sebserver.webservice.circuitbreaker.quizzesRequest.blockingTime", "sebserver.webservice.circuitbreaker.quizzesRequest.blockingTime",
Long.class, Long.class,
@ -177,14 +193,30 @@ public class LmsAPITemplateAdapter implements LmsAPITemplate {
return this.apiTemplateDataSupplier.getLmsSetup(); return this.apiTemplateDataSupplier.getLmsSetup();
} }
@Override
public void checkCourseAPIAccess() {
this.lmsTestRequest
.protectedRun(() -> {
final LmsSetupTestResult testCourseAccessAPI = this.courseAccessAPI.testCourseAccessAPI();
if (!testCourseAccessAPI.isOk()) {
throw new RuntimeException("No course API Access: " + testCourseAccessAPI);
}
return testCourseAccessAPI;
});
}
@Override @Override
public LmsSetupTestResult testCourseAccessAPI() { public LmsSetupTestResult testCourseAccessAPI() {
if (this.courseAccessAPI != null) { if (this.courseAccessAPI != null) {
return this.courseAccessAPI.testCourseAccessAPI(); if (log.isDebugEnabled()) {
} log.debug("Test Course Access API for LMSSetup: {}", lmsSetup());
}
if (log.isDebugEnabled()) { return this.lmsTestRequest.protectedRun(() -> this.courseAccessAPI.testCourseAccessAPI())
log.debug("Test Course Access API for LMSSetup: {}", lmsSetup()); .onError(error -> log.error(
"Failed to run protectedQuizzesRequest: {}",
error.getMessage()))
.getOrThrow();
} }
return LmsSetupTestResult.ofAPINotSupported(getType()); return LmsSetupTestResult.ofAPINotSupported(getType());

View file

@ -152,10 +152,12 @@ final class OpenEdxCourseAccess extends AbstractCachedCourseAccess implements Co
final LmsSetup lmsSetup = getApiTemplateDataSupplier().getLmsSetup(); final LmsSetup lmsSetup = getApiTemplateDataSupplier().getLmsSetup();
final String externalStartURI = getExternalLMSServerAddress(lmsSetup); final String externalStartURI = getExternalLMSServerAddress(lmsSetup);
final QuizData quizData = quizDataOf( final QuizData quizData = getRestTemplate()
lmsSetup, .map(template -> quizDataOf(
this.getOneCourse(id, this.restTemplate, id), lmsSetup,
externalStartURI); this.getOneCourse(id, template, id),
externalStartURI))
.getOrThrow();
if (quizData != null) { if (quizData != null) {
super.putToCache(quizData); super.putToCache(quizData);

View file

@ -10,6 +10,7 @@ package ch.ethz.seb.sebserver.webservice.servicelayer.session.impl;
import java.io.ByteArrayOutputStream; import java.io.ByteArrayOutputStream;
import org.apache.commons.lang3.BooleanUtils;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
import org.springframework.cache.annotation.CacheEvict; import org.springframework.cache.annotation.CacheEvict;
@ -114,7 +115,7 @@ public class ExamSessionCacheService {
} }
public boolean isRunning(final Exam exam) { public boolean isRunning(final Exam exam) {
if (exam == null || !exam.active) { if (exam == null || !exam.active || BooleanUtils.isFalse(exam.lmsDataAvailable)) {
return false; return false;
} }