SEBSERV-233 better handling with circuit breaker
This commit is contained in:
		
							parent
							
								
									d2ea6eb316
								
							
						
					
					
						commit
						0dfde290ca
					
				
					 5 changed files with 59 additions and 14 deletions
				
			
		|  | @ -733,7 +733,9 @@ public class ExamDAOImpl implements ExamDAO { | |||
|                     log.debug("Quizzes size mismatch detected by getting exams quiz data from LMS: {}", lmsSetup); | ||||
|                 } | ||||
| 
 | ||||
|                 if (lmsSetup.testCourseAccessAPI().hasAnyError()) { | ||||
|                 try { | ||||
|                     lmsSetup.checkCourseAPIAccess(); | ||||
|                 } catch (final Exception e) { | ||||
|                     // No course access on the LMS. This means we can't get any quizzes from this LMSSetup at the moment | ||||
|                     // All exams are marked as corrupt because of LMS Setup failure | ||||
| 
 | ||||
|  |  | |||
|  | @ -42,6 +42,14 @@ public interface CourseAccessAPI { | |||
|      * @return {@link LmsSetupTestResult } instance with the test result report */ | ||||
|     LmsSetupTestResult testCourseAccessAPI(); | ||||
| 
 | ||||
|     /** To make a quick course API access check without report that just throws an exception if not available */ | ||||
|     default void checkCourseAPIAccess() { | ||||
|         final LmsSetupTestResult testCourseAccessAPI = this.testCourseAccessAPI(); | ||||
|         if (!testCourseAccessAPI.isOk()) { | ||||
|             throw new RuntimeException("No course API Access: " + testCourseAccessAPI); | ||||
|         } | ||||
|     } | ||||
| 
 | ||||
|     /** Get an unsorted List of filtered {@link QuizData } from the LMS course/quiz API | ||||
|      * | ||||
|      * @param filterMap the {@link FilterMap } to get a filtered result. Possible filter attributes are: | ||||
|  |  | |||
|  | @ -43,6 +43,8 @@ public class LmsAPITemplateAdapter implements LmsAPITemplate { | |||
|     private final SEBRestrictionAPI sebBestrictionAPI; | ||||
|     private final APITemplateDataSupplier apiTemplateDataSupplier; | ||||
| 
 | ||||
|     /** CircuitBreaker for protected lmsTestRequest */ | ||||
|     private final CircuitBreaker<LmsSetupTestResult> lmsTestRequest; | ||||
|     /** CircuitBreaker for protected quiz and course data requests */ | ||||
|     private final CircuitBreaker<List<QuizData>> allQuizzesRequest; | ||||
|     /** CircuitBreaker for protected quiz and course data requests */ | ||||
|  | @ -68,15 +70,29 @@ public class LmsAPITemplateAdapter implements LmsAPITemplate { | |||
|         this.sebBestrictionAPI = sebBestrictionAPI; | ||||
|         this.apiTemplateDataSupplier = apiTemplateDataSupplier; | ||||
| 
 | ||||
|         this.lmsTestRequest = asyncService.createCircuitBreaker( | ||||
|                 environment.getProperty( | ||||
|                         "sebserver.webservice.circuitbreaker.lmsTestRequest.attempts", | ||||
|                         Integer.class, | ||||
|                         2), | ||||
|                 environment.getProperty( | ||||
|                         "sebserver.webservice.circuitbreaker.lmsTestRequest.blockingTime", | ||||
|                         Long.class, | ||||
|                         Constants.SECOND_IN_MILLIS * 20), | ||||
|                 environment.getProperty( | ||||
|                         "sebserver.webservice.circuitbreaker.lmsTestRequest.timeToRecover", | ||||
|                         Long.class, | ||||
|                         Constants.MINUTE_IN_MILLIS)); | ||||
| 
 | ||||
|         this.allQuizzesRequest = asyncService.createCircuitBreaker( | ||||
|                 environment.getProperty( | ||||
|                         "sebserver.webservice.circuitbreaker.quizzesRequest.attempts", | ||||
|                         Integer.class, | ||||
|                         3), | ||||
|                         1), | ||||
|                 environment.getProperty( | ||||
|                         "sebserver.webservice.circuitbreaker.quizzesRequest.blockingTime", | ||||
|                         Long.class, | ||||
|                         Constants.MINUTE_IN_MILLIS), | ||||
|                         Constants.SECOND_IN_MILLIS * 20), | ||||
|                 environment.getProperty( | ||||
|                         "sebserver.webservice.circuitbreaker.quizzesRequest.timeToRecover", | ||||
|                         Long.class, | ||||
|  | @ -86,7 +102,7 @@ public class LmsAPITemplateAdapter implements LmsAPITemplate { | |||
|                 environment.getProperty( | ||||
|                         "sebserver.webservice.circuitbreaker.quizzesRequest.attempts", | ||||
|                         Integer.class, | ||||
|                         3), | ||||
|                         1), | ||||
|                 environment.getProperty( | ||||
|                         "sebserver.webservice.circuitbreaker.quizzesRequest.blockingTime", | ||||
|                         Long.class, | ||||
|  | @ -100,7 +116,7 @@ public class LmsAPITemplateAdapter implements LmsAPITemplate { | |||
|                 environment.getProperty( | ||||
|                         "sebserver.webservice.circuitbreaker.quizzesRequest.attempts", | ||||
|                         Integer.class, | ||||
|                         3), | ||||
|                         1), | ||||
|                 environment.getProperty( | ||||
|                         "sebserver.webservice.circuitbreaker.quizzesRequest.blockingTime", | ||||
|                         Long.class, | ||||
|  | @ -177,14 +193,30 @@ public class LmsAPITemplateAdapter implements LmsAPITemplate { | |||
|         return this.apiTemplateDataSupplier.getLmsSetup(); | ||||
|     } | ||||
| 
 | ||||
|     @Override | ||||
|     public void checkCourseAPIAccess() { | ||||
|         this.lmsTestRequest | ||||
|                 .protectedRun(() -> { | ||||
|                     final LmsSetupTestResult testCourseAccessAPI = this.courseAccessAPI.testCourseAccessAPI(); | ||||
|                     if (!testCourseAccessAPI.isOk()) { | ||||
|                         throw new RuntimeException("No course API Access: " + testCourseAccessAPI); | ||||
|                     } | ||||
|                     return testCourseAccessAPI; | ||||
|                 }); | ||||
|     } | ||||
| 
 | ||||
|     @Override | ||||
|     public LmsSetupTestResult testCourseAccessAPI() { | ||||
|         if (this.courseAccessAPI != null) { | ||||
|             return this.courseAccessAPI.testCourseAccessAPI(); | ||||
|         } | ||||
|             if (log.isDebugEnabled()) { | ||||
|                 log.debug("Test Course Access API for LMSSetup: {}", lmsSetup()); | ||||
|             } | ||||
| 
 | ||||
|         if (log.isDebugEnabled()) { | ||||
|             log.debug("Test Course Access API for LMSSetup: {}", lmsSetup()); | ||||
|             return this.lmsTestRequest.protectedRun(() -> this.courseAccessAPI.testCourseAccessAPI()) | ||||
|                     .onError(error -> log.error( | ||||
|                             "Failed to run protectedQuizzesRequest: {}", | ||||
|                             error.getMessage())) | ||||
|                     .getOrThrow(); | ||||
|         } | ||||
| 
 | ||||
|         return LmsSetupTestResult.ofAPINotSupported(getType()); | ||||
|  |  | |||
|  | @ -152,10 +152,12 @@ final class OpenEdxCourseAccess extends AbstractCachedCourseAccess implements Co | |||
| 
 | ||||
|             final LmsSetup lmsSetup = getApiTemplateDataSupplier().getLmsSetup(); | ||||
|             final String externalStartURI = getExternalLMSServerAddress(lmsSetup); | ||||
|             final QuizData quizData = quizDataOf( | ||||
|                     lmsSetup, | ||||
|                     this.getOneCourse(id, this.restTemplate, id), | ||||
|                     externalStartURI); | ||||
|             final QuizData quizData = getRestTemplate() | ||||
|                     .map(template -> quizDataOf( | ||||
|                             lmsSetup, | ||||
|                             this.getOneCourse(id, template, id), | ||||
|                             externalStartURI)) | ||||
|                     .getOrThrow(); | ||||
| 
 | ||||
|             if (quizData != null) { | ||||
|                 super.putToCache(quizData); | ||||
|  |  | |||
|  | @ -10,6 +10,7 @@ package ch.ethz.seb.sebserver.webservice.servicelayer.session.impl; | |||
| 
 | ||||
| import java.io.ByteArrayOutputStream; | ||||
| 
 | ||||
| import org.apache.commons.lang3.BooleanUtils; | ||||
| import org.slf4j.Logger; | ||||
| import org.slf4j.LoggerFactory; | ||||
| import org.springframework.cache.annotation.CacheEvict; | ||||
|  | @ -114,7 +115,7 @@ public class ExamSessionCacheService { | |||
|     } | ||||
| 
 | ||||
|     public boolean isRunning(final Exam exam) { | ||||
|         if (exam == null || !exam.active) { | ||||
|         if (exam == null || !exam.active || BooleanUtils.isFalse(exam.lmsDataAvailable)) { | ||||
|             return false; | ||||
|         } | ||||
| 
 | ||||
|  |  | |||
		Loading…
	
	Add table
		
		Reference in a new issue
	
	 anhefti
						anhefti