SEBSERV-191 backend implementation

This commit is contained in:
anhefti 2021-11-03 10:31:15 +01:00
parent 458cc9486e
commit 2f2a3670b7
9 changed files with 532 additions and 29 deletions

View file

@ -68,6 +68,10 @@ public class ClientEvent implements Entity, IndicatorValueHolder {
}
}
public enum ExportType {
CSV
}
@JsonProperty(Domain.CLIENT_EVENT.ATTR_ID)
public final Long id;

View file

@ -77,6 +77,30 @@ public interface PaginationService {
final String tableName,
final Supplier<Result<Collection<T>>> delegate);
/** Fetches a paged batch of objects
*
* NOTE: Paging always depends on SQL level. It depends on the collection given by the SQL select statement
* that is executed within MyBatis by using the MyBatis page service.
* Be aware that if the delegate that is given here applies an additional filter to the filtering done
* on SQL level, this will lead to paging with not fully filled pages or even to empty pages if the filter
* filters a lot of the entries given by the SQL statement away.
* So we recommend to apply as much of the filtering as possible on the SQL level and only if necessary and
* not avoidable, apply a additional filter on software-level that eventually filter one or two entities
* for a page.
*
* @param pageNumber the current page number
* @param pageSize the (full) size of the page
* @param sort the name of the sort column with a leading '-' for descending sort order
* @param tableName the name of the SQL table on which the pagination is applying to
* @param delegate a collection supplier the does the underling SQL query with specified pagination attributes
* @return Result refers to a Collection of specified type of objects or to an exception on error case */
<T> Result<Collection<T>> fetch(
final int pageNumber,
final int pageSize,
final String sort,
final String tableName,
final Supplier<Result<Collection<T>>> delegate);
/** Use this to build a current Page from a given list of objects.
*
* @param <T> the Type if list entities

View file

@ -147,6 +147,20 @@ public class PaginationServiceImpl implements PaginationService {
});
}
@Override
public <T> Result<Collection<T>> fetch(
final int pageNumber,
final int pageSize,
final String sort,
final String tableName,
final Supplier<Result<Collection<T>>> delegate) {
return Result.tryCatch(() -> {
setPagination(pageNumber, pageSize, sort, tableName);
return delegate.get().getOrThrow();
});
}
private String verifySortColumnName(final String sort, final String columnName) {
if (StringUtils.isBlank(sort)) {

View file

@ -0,0 +1,38 @@
/*
* Copyright (c) 2021 ETH Zürich, Educational Development and Technology (LET)
*
* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/.
*/
package ch.ethz.seb.sebserver.webservice.servicelayer.exam;
import java.io.OutputStream;
import java.util.Collection;
import java.util.function.Predicate;
import org.springframework.scheduling.annotation.Async;
import ch.ethz.seb.sebserver.gbl.async.AsyncServiceSpringConfig;
import ch.ethz.seb.sebserver.gbl.model.EntityProcessingReport;
import ch.ethz.seb.sebserver.gbl.model.session.ClientEvent;
import ch.ethz.seb.sebserver.gbl.model.session.ClientEvent.ExportType;
import ch.ethz.seb.sebserver.gbl.util.Result;
import ch.ethz.seb.sebserver.webservice.servicelayer.dao.FilterMap;
public interface SEBClientEventAdminService {
Result<EntityProcessingReport> deleteAllClientEvents(Collection<String> ids);
@Async(AsyncServiceSpringConfig.EXECUTOR_BEAN_NAME)
void exportSEBClientLogs(
OutputStream output,
FilterMap filterMap,
String sort,
final Predicate<ClientEvent> predicate,
ExportType exportType,
boolean includeConnectionDetails,
boolean includeExamDetails);
}

View file

@ -0,0 +1,32 @@
/*
* Copyright (c) 2021 ETH Zürich, Educational Development and Technology (LET)
*
* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/.
*/
package ch.ethz.seb.sebserver.webservice.servicelayer.exam;
import java.io.OutputStream;
import ch.ethz.seb.sebserver.gbl.model.exam.Exam;
import ch.ethz.seb.sebserver.gbl.model.session.ClientEvent.ExportType;
import ch.ethz.seb.sebserver.webservice.datalayer.batis.model.ClientConnectionRecord;
import ch.ethz.seb.sebserver.webservice.datalayer.batis.model.ClientEventRecord;
public interface SEBClientEventExporter {
ExportType exportType();
void streamHeader(
OutputStream output,
boolean includeConnectionDetails,
boolean includeExamDetails);
void streamData(
OutputStream output,
ClientEventRecord eventData,
ClientConnectionRecord connectionData,
Exam examData);
}

View file

@ -0,0 +1,250 @@
/*
* Copyright (c) 2021 ETH Zürich, Educational Development and Technology (LET)
*
* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/.
*/
package ch.ethz.seb.sebserver.webservice.servicelayer.exam.impl;
import java.io.OutputStream;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.EnumMap;
import java.util.HashMap;
import java.util.Iterator;
import java.util.Map;
import java.util.Set;
import java.util.function.Predicate;
import java.util.stream.Collectors;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.context.annotation.Lazy;
import org.springframework.stereotype.Service;
import ch.ethz.seb.sebserver.gbl.api.APIMessage;
import ch.ethz.seb.sebserver.gbl.api.EntityType;
import ch.ethz.seb.sebserver.gbl.model.EntityKey;
import ch.ethz.seb.sebserver.gbl.model.EntityProcessingReport;
import ch.ethz.seb.sebserver.gbl.model.EntityProcessingReport.ErrorEntry;
import ch.ethz.seb.sebserver.gbl.model.exam.Exam;
import ch.ethz.seb.sebserver.gbl.model.session.ClientEvent;
import ch.ethz.seb.sebserver.gbl.model.session.ClientEvent.ExportType;
import ch.ethz.seb.sebserver.gbl.profile.WebServiceProfile;
import ch.ethz.seb.sebserver.gbl.util.Result;
import ch.ethz.seb.sebserver.webservice.datalayer.batis.mapper.ClientEventRecordDynamicSqlSupport;
import ch.ethz.seb.sebserver.webservice.datalayer.batis.model.ClientConnectionRecord;
import ch.ethz.seb.sebserver.webservice.datalayer.batis.model.ClientEventRecord;
import ch.ethz.seb.sebserver.webservice.servicelayer.PaginationService;
import ch.ethz.seb.sebserver.webservice.servicelayer.dao.ClientEventDAO;
import ch.ethz.seb.sebserver.webservice.servicelayer.dao.FilterMap;
import ch.ethz.seb.sebserver.webservice.servicelayer.exam.SEBClientEventAdminService;
import ch.ethz.seb.sebserver.webservice.servicelayer.exam.SEBClientEventExporter;
@Lazy
@Service
@WebServiceProfile
public class SEBClientEventAdminServiceImpl implements SEBClientEventAdminService {
private static final Logger log = LoggerFactory.getLogger(SEBClientEventAdminServiceImpl.class);
private final PaginationService paginationService;
private final ClientEventDAO clientEventDAO;
private final SEBClientEventExportTransactionHandler sebClientEventExportTransactionHandler;
private final EnumMap<ExportType, SEBClientEventExporter> exporter;
public SEBClientEventAdminServiceImpl(
final PaginationService paginationService,
final ClientEventDAO clientEventDAO,
final SEBClientEventExportTransactionHandler sebClientEventExportTransactionHandler,
final Collection<SEBClientEventExporter> exporter) {
this.paginationService = paginationService;
this.clientEventDAO = clientEventDAO;
this.sebClientEventExportTransactionHandler = sebClientEventExportTransactionHandler;
this.exporter = new EnumMap<>(ExportType.class);
exporter.forEach(exp -> this.exporter.putIfAbsent(exp.exportType(), exp));
}
@Override
public Result<EntityProcessingReport> deleteAllClientEvents(final Collection<String> ids) {
return Result.tryCatch(() -> {
if (ids == null || ids.isEmpty()) {
return EntityProcessingReport.ofEmptyError();
}
final Set<EntityKey> sources = ids.stream()
.map(id -> new EntityKey(id, EntityType.CLIENT_EVENT))
.collect(Collectors.toSet());
final Result<Collection<EntityKey>> delete = this.clientEventDAO.delete(sources);
if (delete.hasError()) {
return new EntityProcessingReport(
Collections.emptyList(),
Collections.emptyList(),
Arrays.asList(new ErrorEntry(null, APIMessage.ErrorMessage.UNEXPECTED.of(delete.getError()))));
} else {
return new EntityProcessingReport(
sources,
delete.get(),
Collections.emptyList());
}
});
}
@Override
public void exportSEBClientLogs(
final OutputStream output,
final FilterMap filterMap,
final String sort,
final Predicate<ClientEvent> predicate,
final ExportType exportType,
final boolean includeConnectionDetails,
final boolean includeExamDetails) {
new exportRunner(
this.exporter.get(exportType),
includeConnectionDetails,
includeExamDetails,
new Pager(filterMap, sort, predicate),
output)
.run();
}
private class exportRunner {
private final SEBClientEventExporter exporter;
private final boolean includeConnectionDetails;
private final boolean includeExamDetails;
private final Iterator<Collection<ClientEventRecord>> pager;
private final OutputStream output;
private final Map<Long, Exam> examCache;
private final Map<Long, ClientConnectionRecord> connectionCache;
public exportRunner(
final SEBClientEventExporter exporter,
final boolean includeConnectionDetails,
final boolean includeExamDetails,
final Iterator<Collection<ClientEventRecord>> pager,
final OutputStream output) {
this.exporter = exporter;
this.includeConnectionDetails = includeConnectionDetails;
this.includeExamDetails = includeExamDetails;
this.pager = pager;
this.output = output;
this.connectionCache = includeConnectionDetails ? new HashMap<>() : null;
this.examCache = includeExamDetails ? new HashMap<>() : null;
}
public void run() {
// first stream header line
this.exporter.streamHeader(this.output, this.includeConnectionDetails, this.includeExamDetails);
// then batch with the pager and stream line per line
while (this.pager.hasNext()) {
this.pager.next().forEach(rec -> {
this.exporter.streamData(
this.output,
rec,
this.includeConnectionDetails ? getConnection(rec.getClientConnectionId()) : null,
this.includeExamDetails ? getExam(rec.getClientConnectionId()) : null);
});
}
}
private ClientConnectionRecord getConnection(final Long connectionId) {
if (!this.connectionCache.containsKey(connectionId)) {
SEBClientEventAdminServiceImpl.this.sebClientEventExportTransactionHandler
.clientConnectionById(connectionId)
.map(e -> this.connectionCache.put(connectionId, e))
.onError(error -> log.error("Failed to get ClientConnectionRecord for id: {}",
connectionId,
error));
}
return this.connectionCache.get(connectionId);
}
private Exam getExam(final Long connectionId) {
final ClientConnectionRecord connection = getConnection(connectionId);
final Long examId = connection.getExamId();
if (!this.examCache.containsKey(examId)) {
SEBClientEventAdminServiceImpl.this.sebClientEventExportTransactionHandler
.examById(examId)
.map(e -> this.examCache.put(examId, e))
.onError(error -> log.error("Failed to get Exam for id: {}",
examId,
error));
}
return this.examCache.get(examId);
}
}
private class Pager implements Iterator<Collection<ClientEventRecord>> {
private final FilterMap filterMap;
private final String sort;
private final Predicate<ClientEvent> predicate;
private int pageNumber = 0;
private final int pageSize = 100;
private Collection<ClientEventRecord> nextRecords;
public Pager(
final FilterMap filterMap,
final String sort,
final Predicate<ClientEvent> predicate) {
this.filterMap = filterMap;
this.sort = sort;
this.predicate = predicate;
fetchNext();
}
@Override
public boolean hasNext() {
return this.nextRecords != null && !this.nextRecords.isEmpty();
}
@Override
public Collection<ClientEventRecord> next() {
final Collection<ClientEventRecord> result = this.nextRecords;
fetchNext();
return result;
}
private void fetchNext() {
try {
this.nextRecords = SEBClientEventAdminServiceImpl.this.paginationService.fetch(
this.pageNumber,
this.pageSize,
this.sort,
ClientEventRecordDynamicSqlSupport.clientEventRecord.name(),
() -> SEBClientEventAdminServiceImpl.this.sebClientEventExportTransactionHandler
.allMatching(this.filterMap, this.predicate))
.getOrThrow();
this.pageNumber++;
} catch (final Exception e) {
log.error("Failed to fetch next batch: ", e);
this.nextRecords = null;
}
}
}
}

View file

@ -0,0 +1,49 @@
/*
* Copyright (c) 2021 ETH Zürich, Educational Development and Technology (LET)
*
* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/.
*/
package ch.ethz.seb.sebserver.webservice.servicelayer.exam.impl;
import java.io.OutputStream;
import ch.ethz.seb.sebserver.gbl.model.exam.Exam;
import ch.ethz.seb.sebserver.gbl.model.session.ClientEvent.ExportType;
import ch.ethz.seb.sebserver.webservice.datalayer.batis.model.ClientConnectionRecord;
import ch.ethz.seb.sebserver.webservice.datalayer.batis.model.ClientEventRecord;
import ch.ethz.seb.sebserver.webservice.servicelayer.exam.SEBClientEventExporter;
public class SEBClientEventCSVExporter implements SEBClientEventExporter {
@Override
public ExportType exportType() {
return ExportType.CSV;
}
@Override
public void streamHeader(
final OutputStream output,
final boolean includeConnectionDetails,
final boolean includeExamDetails) {
// TODO
}
@Override
public void streamData(
final OutputStream output,
final ClientEventRecord eventData,
final ClientConnectionRecord connectionData,
final Exam examData) {
// TODO
}
private String toCSVString(final String text) {
}
}

View file

@ -0,0 +1,113 @@
/*
* Copyright (c) 2021 ETH Zürich, Educational Development and Technology (LET)
*
* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/.
*/
package ch.ethz.seb.sebserver.webservice.servicelayer.exam.impl;
import static org.mybatis.dynamic.sql.SqlBuilder.equalTo;
import static org.mybatis.dynamic.sql.SqlBuilder.isEqualToWhenPresent;
import java.util.Collection;
import java.util.function.Predicate;
import org.mybatis.dynamic.sql.SqlBuilder;
import org.springframework.context.annotation.Lazy;
import org.springframework.stereotype.Component;
import org.springframework.transaction.annotation.Transactional;
import ch.ethz.seb.sebserver.gbl.model.exam.Exam;
import ch.ethz.seb.sebserver.gbl.model.session.ClientConnection;
import ch.ethz.seb.sebserver.gbl.model.session.ClientEvent;
import ch.ethz.seb.sebserver.gbl.model.session.ClientEvent.EventType;
import ch.ethz.seb.sebserver.gbl.profile.WebServiceProfile;
import ch.ethz.seb.sebserver.gbl.util.Result;
import ch.ethz.seb.sebserver.webservice.datalayer.batis.mapper.ClientConnectionRecordDynamicSqlSupport;
import ch.ethz.seb.sebserver.webservice.datalayer.batis.mapper.ClientConnectionRecordMapper;
import ch.ethz.seb.sebserver.webservice.datalayer.batis.mapper.ClientEventRecordDynamicSqlSupport;
import ch.ethz.seb.sebserver.webservice.datalayer.batis.mapper.ClientEventRecordMapper;
import ch.ethz.seb.sebserver.webservice.datalayer.batis.model.ClientConnectionRecord;
import ch.ethz.seb.sebserver.webservice.datalayer.batis.model.ClientEventRecord;
import ch.ethz.seb.sebserver.webservice.servicelayer.dao.ExamDAO;
import ch.ethz.seb.sebserver.webservice.servicelayer.dao.FilterMap;
@Lazy
@Component
@WebServiceProfile
public class SEBClientEventExportTransactionHandler {
private final ClientEventRecordMapper clientEventRecordMapper;
private final ClientConnectionRecordMapper clientConnectionRecordMapper;
private final ExamDAO examDAO;
public SEBClientEventExportTransactionHandler(
final ClientEventRecordMapper clientEventRecordMapper,
final ClientConnectionRecordMapper clientConnectionRecordMapper,
final ExamDAO examDAO) {
this.clientEventRecordMapper = clientEventRecordMapper;
this.clientConnectionRecordMapper = clientConnectionRecordMapper;
this.examDAO = examDAO;
}
@Transactional(readOnly = true)
public Result<Collection<ClientEventRecord>> allMatching(
final FilterMap filterMap,
final Predicate<ClientEvent> predicate) {
return Result.tryCatch(() -> this.clientEventRecordMapper
.selectByExample()
.leftJoin(ClientConnectionRecordDynamicSqlSupport.clientConnectionRecord)
.on(
ClientConnectionRecordDynamicSqlSupport.id,
equalTo(ClientEventRecordDynamicSqlSupport.clientConnectionId))
.where(
ClientConnectionRecordDynamicSqlSupport.institutionId,
isEqualToWhenPresent(filterMap.getInstitutionId()))
.and(
ClientConnectionRecordDynamicSqlSupport.examId,
isEqualToWhenPresent(filterMap.getClientEventExamId()))
.and(
ClientConnectionRecordDynamicSqlSupport.examUserSessionId,
SqlBuilder.isLikeWhenPresent(filterMap.getSQLWildcard(ClientConnection.FILTER_ATTR_SESSION_ID)))
.and(
ClientEventRecordDynamicSqlSupport.clientConnectionId,
isEqualToWhenPresent(filterMap.getClientEventConnectionId()))
.and(
ClientEventRecordDynamicSqlSupport.type,
isEqualToWhenPresent(filterMap.getClientEventTypeId()))
.and(
ClientEventRecordDynamicSqlSupport.type,
SqlBuilder.isNotEqualTo(EventType.LAST_PING.id))
.and(
ClientEventRecordDynamicSqlSupport.clientTime,
SqlBuilder.isGreaterThanOrEqualToWhenPresent(filterMap.getClientEventClientTimeFrom()))
.and(
ClientEventRecordDynamicSqlSupport.clientTime,
SqlBuilder.isLessThanOrEqualToWhenPresent(filterMap.getClientEventClientTimeTo()))
.and(
ClientEventRecordDynamicSqlSupport.serverTime,
SqlBuilder.isGreaterThanOrEqualToWhenPresent(filterMap.getClientEventServerTimeFrom()))
.and(
ClientEventRecordDynamicSqlSupport.serverTime,
SqlBuilder.isLessThanOrEqualToWhenPresent(filterMap.getClientEventServerTimeTo()))
.and(
ClientEventRecordDynamicSqlSupport.text,
SqlBuilder.isLikeWhenPresent(filterMap.getClientEventText()))
.build()
.execute());
}
@Transactional(readOnly = true)
public Result<ClientConnectionRecord> clientConnectionById(final Long id) {
return Result.tryCatch(() -> this.clientConnectionRecordMapper.selectByPrimaryKey(id));
}
public Result<Exam> examById(final Long id) {
return this.examDAO.byPK(id);
}
}

View file

@ -8,13 +8,9 @@
package ch.ethz.seb.sebserver.webservice.weblayer.api;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.EnumSet;
import java.util.List;
import java.util.Set;
import java.util.stream.Collectors;
import javax.servlet.http.HttpServletRequest;
@ -28,13 +24,10 @@ import org.springframework.web.bind.annotation.RestController;
import ch.ethz.seb.sebserver.gbl.api.API;
import ch.ethz.seb.sebserver.gbl.api.API.BulkActionType;
import ch.ethz.seb.sebserver.gbl.api.APIMessage;
import ch.ethz.seb.sebserver.gbl.api.EntityType;
import ch.ethz.seb.sebserver.gbl.api.authorization.PrivilegeType;
import ch.ethz.seb.sebserver.gbl.model.EntityDependency;
import ch.ethz.seb.sebserver.gbl.model.EntityKey;
import ch.ethz.seb.sebserver.gbl.model.EntityProcessingReport;
import ch.ethz.seb.sebserver.gbl.model.EntityProcessingReport.ErrorEntry;
import ch.ethz.seb.sebserver.gbl.model.GrantEntity;
import ch.ethz.seb.sebserver.gbl.model.Page;
import ch.ethz.seb.sebserver.gbl.model.session.ClientEvent;
@ -53,6 +46,7 @@ import ch.ethz.seb.sebserver.webservice.servicelayer.dao.ClientEventDAO;
import ch.ethz.seb.sebserver.webservice.servicelayer.dao.ExamDAO;
import ch.ethz.seb.sebserver.webservice.servicelayer.dao.FilterMap;
import ch.ethz.seb.sebserver.webservice.servicelayer.dao.UserActivityLogDAO;
import ch.ethz.seb.sebserver.webservice.servicelayer.exam.SEBClientEventAdminService;
import ch.ethz.seb.sebserver.webservice.servicelayer.validation.BeanValidationService;
@WebServiceProfile
@ -62,6 +56,7 @@ public class ClientEventController extends ReadonlyEntityController<ClientEvent,
private final ExamDAO examDao;
private final ClientEventDAO clientEventDAO;
private final SEBClientEventAdminService sebClientEventAdminService;
protected ClientEventController(
final AuthorizationService authorization,
@ -70,7 +65,8 @@ public class ClientEventController extends ReadonlyEntityController<ClientEvent,
final UserActivityLogDAO userActivityLogDAO,
final PaginationService paginationService,
final BeanValidationService beanValidationService,
final ExamDAO examDao) {
final ExamDAO examDao,
final SEBClientEventAdminService sebClientEventAdminService) {
super(authorization,
bulkActionService,
@ -81,6 +77,7 @@ public class ClientEventController extends ReadonlyEntityController<ClientEvent,
this.examDao = examDao;
this.clientEventDAO = entityDAO;
this.sebClientEventAdminService = sebClientEventAdminService;
}
@RequestMapping(
@ -136,27 +133,9 @@ public class ClientEventController extends ReadonlyEntityController<ClientEvent,
this.checkWritePrivilege(institutionId);
if (ids == null || ids.isEmpty()) {
return EntityProcessingReport.ofEmptyError();
}
final Set<EntityKey> sources = ids.stream()
.map(id -> new EntityKey(id, EntityType.CLIENT_EVENT))
.collect(Collectors.toSet());
final Result<Collection<EntityKey>> delete = this.clientEventDAO.delete(sources);
if (delete.hasError()) {
return new EntityProcessingReport(
Collections.emptyList(),
Collections.emptyList(),
Arrays.asList(new ErrorEntry(null, APIMessage.ErrorMessage.UNEXPECTED.of(delete.getError()))));
} else {
return new EntityProcessingReport(
sources,
delete.get(),
Collections.emptyList());
}
return this.sebClientEventAdminService
.deleteAllClientEvents(ids)
.getOrThrow();
}
@Override