SEBSERV-233 better handling with circuit breaker
This commit is contained in:
		
							parent
							
								
									d2ea6eb316
								
							
						
					
					
						commit
						0dfde290ca
					
				
					 5 changed files with 59 additions and 14 deletions
				
			
		|  | @ -733,7 +733,9 @@ public class ExamDAOImpl implements ExamDAO { | ||||||
|                     log.debug("Quizzes size mismatch detected by getting exams quiz data from LMS: {}", lmsSetup); |                     log.debug("Quizzes size mismatch detected by getting exams quiz data from LMS: {}", lmsSetup); | ||||||
|                 } |                 } | ||||||
| 
 | 
 | ||||||
|                 if (lmsSetup.testCourseAccessAPI().hasAnyError()) { |                 try { | ||||||
|  |                     lmsSetup.checkCourseAPIAccess(); | ||||||
|  |                 } catch (final Exception e) { | ||||||
|                     // No course access on the LMS. This means we can't get any quizzes from this LMSSetup at the moment |                     // No course access on the LMS. This means we can't get any quizzes from this LMSSetup at the moment | ||||||
|                     // All exams are marked as corrupt because of LMS Setup failure |                     // All exams are marked as corrupt because of LMS Setup failure | ||||||
| 
 | 
 | ||||||
|  |  | ||||||
|  | @ -42,6 +42,14 @@ public interface CourseAccessAPI { | ||||||
|      * @return {@link LmsSetupTestResult } instance with the test result report */ |      * @return {@link LmsSetupTestResult } instance with the test result report */ | ||||||
|     LmsSetupTestResult testCourseAccessAPI(); |     LmsSetupTestResult testCourseAccessAPI(); | ||||||
| 
 | 
 | ||||||
|  |     /** To make a quick course API access check without report that just throws an exception if not available */ | ||||||
|  |     default void checkCourseAPIAccess() { | ||||||
|  |         final LmsSetupTestResult testCourseAccessAPI = this.testCourseAccessAPI(); | ||||||
|  |         if (!testCourseAccessAPI.isOk()) { | ||||||
|  |             throw new RuntimeException("No course API Access: " + testCourseAccessAPI); | ||||||
|  |         } | ||||||
|  |     } | ||||||
|  | 
 | ||||||
|     /** Get an unsorted List of filtered {@link QuizData } from the LMS course/quiz API |     /** Get an unsorted List of filtered {@link QuizData } from the LMS course/quiz API | ||||||
|      * |      * | ||||||
|      * @param filterMap the {@link FilterMap } to get a filtered result. Possible filter attributes are: |      * @param filterMap the {@link FilterMap } to get a filtered result. Possible filter attributes are: | ||||||
|  |  | ||||||
|  | @ -43,6 +43,8 @@ public class LmsAPITemplateAdapter implements LmsAPITemplate { | ||||||
|     private final SEBRestrictionAPI sebBestrictionAPI; |     private final SEBRestrictionAPI sebBestrictionAPI; | ||||||
|     private final APITemplateDataSupplier apiTemplateDataSupplier; |     private final APITemplateDataSupplier apiTemplateDataSupplier; | ||||||
| 
 | 
 | ||||||
|  |     /** CircuitBreaker for protected lmsTestRequest */ | ||||||
|  |     private final CircuitBreaker<LmsSetupTestResult> lmsTestRequest; | ||||||
|     /** CircuitBreaker for protected quiz and course data requests */ |     /** CircuitBreaker for protected quiz and course data requests */ | ||||||
|     private final CircuitBreaker<List<QuizData>> allQuizzesRequest; |     private final CircuitBreaker<List<QuizData>> allQuizzesRequest; | ||||||
|     /** CircuitBreaker for protected quiz and course data requests */ |     /** CircuitBreaker for protected quiz and course data requests */ | ||||||
|  | @ -68,15 +70,29 @@ public class LmsAPITemplateAdapter implements LmsAPITemplate { | ||||||
|         this.sebBestrictionAPI = sebBestrictionAPI; |         this.sebBestrictionAPI = sebBestrictionAPI; | ||||||
|         this.apiTemplateDataSupplier = apiTemplateDataSupplier; |         this.apiTemplateDataSupplier = apiTemplateDataSupplier; | ||||||
| 
 | 
 | ||||||
|  |         this.lmsTestRequest = asyncService.createCircuitBreaker( | ||||||
|  |                 environment.getProperty( | ||||||
|  |                         "sebserver.webservice.circuitbreaker.lmsTestRequest.attempts", | ||||||
|  |                         Integer.class, | ||||||
|  |                         2), | ||||||
|  |                 environment.getProperty( | ||||||
|  |                         "sebserver.webservice.circuitbreaker.lmsTestRequest.blockingTime", | ||||||
|  |                         Long.class, | ||||||
|  |                         Constants.SECOND_IN_MILLIS * 20), | ||||||
|  |                 environment.getProperty( | ||||||
|  |                         "sebserver.webservice.circuitbreaker.lmsTestRequest.timeToRecover", | ||||||
|  |                         Long.class, | ||||||
|  |                         Constants.MINUTE_IN_MILLIS)); | ||||||
|  | 
 | ||||||
|         this.allQuizzesRequest = asyncService.createCircuitBreaker( |         this.allQuizzesRequest = asyncService.createCircuitBreaker( | ||||||
|                 environment.getProperty( |                 environment.getProperty( | ||||||
|                         "sebserver.webservice.circuitbreaker.quizzesRequest.attempts", |                         "sebserver.webservice.circuitbreaker.quizzesRequest.attempts", | ||||||
|                         Integer.class, |                         Integer.class, | ||||||
|                         3), |                         1), | ||||||
|                 environment.getProperty( |                 environment.getProperty( | ||||||
|                         "sebserver.webservice.circuitbreaker.quizzesRequest.blockingTime", |                         "sebserver.webservice.circuitbreaker.quizzesRequest.blockingTime", | ||||||
|                         Long.class, |                         Long.class, | ||||||
|                         Constants.MINUTE_IN_MILLIS), |                         Constants.SECOND_IN_MILLIS * 20), | ||||||
|                 environment.getProperty( |                 environment.getProperty( | ||||||
|                         "sebserver.webservice.circuitbreaker.quizzesRequest.timeToRecover", |                         "sebserver.webservice.circuitbreaker.quizzesRequest.timeToRecover", | ||||||
|                         Long.class, |                         Long.class, | ||||||
|  | @ -86,7 +102,7 @@ public class LmsAPITemplateAdapter implements LmsAPITemplate { | ||||||
|                 environment.getProperty( |                 environment.getProperty( | ||||||
|                         "sebserver.webservice.circuitbreaker.quizzesRequest.attempts", |                         "sebserver.webservice.circuitbreaker.quizzesRequest.attempts", | ||||||
|                         Integer.class, |                         Integer.class, | ||||||
|                         3), |                         1), | ||||||
|                 environment.getProperty( |                 environment.getProperty( | ||||||
|                         "sebserver.webservice.circuitbreaker.quizzesRequest.blockingTime", |                         "sebserver.webservice.circuitbreaker.quizzesRequest.blockingTime", | ||||||
|                         Long.class, |                         Long.class, | ||||||
|  | @ -100,7 +116,7 @@ public class LmsAPITemplateAdapter implements LmsAPITemplate { | ||||||
|                 environment.getProperty( |                 environment.getProperty( | ||||||
|                         "sebserver.webservice.circuitbreaker.quizzesRequest.attempts", |                         "sebserver.webservice.circuitbreaker.quizzesRequest.attempts", | ||||||
|                         Integer.class, |                         Integer.class, | ||||||
|                         3), |                         1), | ||||||
|                 environment.getProperty( |                 environment.getProperty( | ||||||
|                         "sebserver.webservice.circuitbreaker.quizzesRequest.blockingTime", |                         "sebserver.webservice.circuitbreaker.quizzesRequest.blockingTime", | ||||||
|                         Long.class, |                         Long.class, | ||||||
|  | @ -177,14 +193,30 @@ public class LmsAPITemplateAdapter implements LmsAPITemplate { | ||||||
|         return this.apiTemplateDataSupplier.getLmsSetup(); |         return this.apiTemplateDataSupplier.getLmsSetup(); | ||||||
|     } |     } | ||||||
| 
 | 
 | ||||||
|  |     @Override | ||||||
|  |     public void checkCourseAPIAccess() { | ||||||
|  |         this.lmsTestRequest | ||||||
|  |                 .protectedRun(() -> { | ||||||
|  |                     final LmsSetupTestResult testCourseAccessAPI = this.courseAccessAPI.testCourseAccessAPI(); | ||||||
|  |                     if (!testCourseAccessAPI.isOk()) { | ||||||
|  |                         throw new RuntimeException("No course API Access: " + testCourseAccessAPI); | ||||||
|  |                     } | ||||||
|  |                     return testCourseAccessAPI; | ||||||
|  |                 }); | ||||||
|  |     } | ||||||
|  | 
 | ||||||
|     @Override |     @Override | ||||||
|     public LmsSetupTestResult testCourseAccessAPI() { |     public LmsSetupTestResult testCourseAccessAPI() { | ||||||
|         if (this.courseAccessAPI != null) { |         if (this.courseAccessAPI != null) { | ||||||
|             return this.courseAccessAPI.testCourseAccessAPI(); |             if (log.isDebugEnabled()) { | ||||||
|         } |                 log.debug("Test Course Access API for LMSSetup: {}", lmsSetup()); | ||||||
|  |             } | ||||||
| 
 | 
 | ||||||
|         if (log.isDebugEnabled()) { |             return this.lmsTestRequest.protectedRun(() -> this.courseAccessAPI.testCourseAccessAPI()) | ||||||
|             log.debug("Test Course Access API for LMSSetup: {}", lmsSetup()); |                     .onError(error -> log.error( | ||||||
|  |                             "Failed to run protectedQuizzesRequest: {}", | ||||||
|  |                             error.getMessage())) | ||||||
|  |                     .getOrThrow(); | ||||||
|         } |         } | ||||||
| 
 | 
 | ||||||
|         return LmsSetupTestResult.ofAPINotSupported(getType()); |         return LmsSetupTestResult.ofAPINotSupported(getType()); | ||||||
|  |  | ||||||
|  | @ -152,10 +152,12 @@ final class OpenEdxCourseAccess extends AbstractCachedCourseAccess implements Co | ||||||
| 
 | 
 | ||||||
|             final LmsSetup lmsSetup = getApiTemplateDataSupplier().getLmsSetup(); |             final LmsSetup lmsSetup = getApiTemplateDataSupplier().getLmsSetup(); | ||||||
|             final String externalStartURI = getExternalLMSServerAddress(lmsSetup); |             final String externalStartURI = getExternalLMSServerAddress(lmsSetup); | ||||||
|             final QuizData quizData = quizDataOf( |             final QuizData quizData = getRestTemplate() | ||||||
|                     lmsSetup, |                     .map(template -> quizDataOf( | ||||||
|                     this.getOneCourse(id, this.restTemplate, id), |                             lmsSetup, | ||||||
|                     externalStartURI); |                             this.getOneCourse(id, template, id), | ||||||
|  |                             externalStartURI)) | ||||||
|  |                     .getOrThrow(); | ||||||
| 
 | 
 | ||||||
|             if (quizData != null) { |             if (quizData != null) { | ||||||
|                 super.putToCache(quizData); |                 super.putToCache(quizData); | ||||||
|  |  | ||||||
|  | @ -10,6 +10,7 @@ package ch.ethz.seb.sebserver.webservice.servicelayer.session.impl; | ||||||
| 
 | 
 | ||||||
| import java.io.ByteArrayOutputStream; | import java.io.ByteArrayOutputStream; | ||||||
| 
 | 
 | ||||||
|  | import org.apache.commons.lang3.BooleanUtils; | ||||||
| import org.slf4j.Logger; | import org.slf4j.Logger; | ||||||
| import org.slf4j.LoggerFactory; | import org.slf4j.LoggerFactory; | ||||||
| import org.springframework.cache.annotation.CacheEvict; | import org.springframework.cache.annotation.CacheEvict; | ||||||
|  | @ -114,7 +115,7 @@ public class ExamSessionCacheService { | ||||||
|     } |     } | ||||||
| 
 | 
 | ||||||
|     public boolean isRunning(final Exam exam) { |     public boolean isRunning(final Exam exam) { | ||||||
|         if (exam == null || !exam.active) { |         if (exam == null || !exam.active || BooleanUtils.isFalse(exam.lmsDataAvailable)) { | ||||||
|             return false; |             return false; | ||||||
|         } |         } | ||||||
| 
 | 
 | ||||||
|  |  | ||||||
		Loading…
	
	Add table
		
		Reference in a new issue
	
	 anhefti
						anhefti