fixed import

This commit is contained in:
anhefti 2019-10-11 23:13:57 +02:00
parent 2bb0ae1c0d
commit dc7df0620c
10 changed files with 187 additions and 95 deletions

View file

@ -247,6 +247,8 @@ public class SebExamConfigPropForm implements TemplateComposer {
action.pageContext().getParent().getShell(),
widgetFactory);
dialog.setDialogWidth(500);
dialog.open(
CONFIG_KEY_TITLE_TEXT_KEY,
action.pageContext(),

View file

@ -8,9 +8,13 @@
package ch.ethz.seb.sebserver.webservice.servicelayer.sebconfig.impl;
import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.io.PipedInputStream;
import java.io.PipedOutputStream;
import java.io.SequenceInputStream;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
@ -44,6 +48,7 @@ import ch.ethz.seb.sebserver.webservice.servicelayer.dao.FilterMap;
import ch.ethz.seb.sebserver.webservice.servicelayer.sebconfig.AttributeValueConverter;
import ch.ethz.seb.sebserver.webservice.servicelayer.sebconfig.AttributeValueConverterService;
import ch.ethz.seb.sebserver.webservice.servicelayer.sebconfig.ConfigurationFormat;
import ch.ethz.seb.sebserver.webservice.servicelayer.sebconfig.ZipService;
@Lazy
@Component
@ -66,17 +71,20 @@ public class ExamConfigIO {
private final ConfigurationValueDAO configurationValueDAO;
private final ConfigurationDAO configurationDAO;
private final AttributeValueConverterService attributeValueConverterService;
private final ZipService zipService;
protected ExamConfigIO(
final ConfigurationAttributeDAO configurationAttributeDAO,
final ConfigurationValueDAO configurationValueDAO,
final ConfigurationDAO configurationDAO,
final AttributeValueConverterService attributeValueConverterService) {
final AttributeValueConverterService attributeValueConverterService,
final ZipService zipService) {
this.configurationAttributeDAO = configurationAttributeDAO;
this.configurationValueDAO = configurationValueDAO;
this.configurationDAO = configurationDAO;
this.attributeValueConverterService = attributeValueConverterService;
this.zipService = zipService;
}
@Async(AsyncServiceSpringConfig.EXECUTOR_BEAN_NAME)
@ -169,11 +177,13 @@ public class ExamConfigIO {
void importPlainXML(final InputStream in, final Long institutionId, final Long configurationId) {
try {
// get all attributes and map the names to ids
final Map<String, Long> attributeMap = this.configurationAttributeDAO
final Map<String, ConfigurationAttribute> attributeMap = this.configurationAttributeDAO
.allMatching(new FilterMap())
.getOrThrow()
.stream()
.collect(Collectors.toMap(attr -> attr.name, attr -> attr.id));
.collect(Collectors.toMap(
attr -> attr.name,
Function.identity()));
// the SAX handler with a ConfigValue sink that saves the values to DB
// and a attribute-name/id mapping function with pre-created mapping
@ -204,6 +214,31 @@ public class ExamConfigIO {
}
}
InputStream unzip(final InputStream input) throws Exception {
final byte[] zipHeader = new byte[4];
input.read(zipHeader);
final boolean isZipped = Byte.toUnsignedInt(zipHeader[0]) == Constants.GZIP_ID1
&& Byte.toUnsignedInt(zipHeader[1]) == Constants.GZIP_ID2
&& Byte.toUnsignedInt(zipHeader[2]) == Constants.GZIP_CM;
final InputStream sequencedInput = new SequenceInputStream(
new ByteArrayInputStream(zipHeader, 0, 4),
input);
if (isZipped) {
final PipedInputStream pipedIn = new PipedInputStream();
final PipedOutputStream pipedOut = new PipedOutputStream(pipedIn);
this.zipService.read(pipedOut, sequencedInput);
return pipedIn;
} else {
return sequencedInput;
}
}
private Predicate<ConfigurationAttribute> exportFormatBasedAttributeFilter(final ConfigurationFormat format) {
// Filter originatorVersion according to: https://www.safeexambrowser.org/developer/seb-config-key.html
return attr -> !("originatorVersion".equals(attr.getName()) && format == ConfigurationFormat.JSON);

View file

@ -23,6 +23,8 @@ import org.xml.sax.SAXException;
import org.xml.sax.helpers.DefaultHandler;
import ch.ethz.seb.sebserver.gbl.Constants;
import ch.ethz.seb.sebserver.gbl.model.sebconfig.AttributeType;
import ch.ethz.seb.sebserver.gbl.model.sebconfig.ConfigurationAttribute;
import ch.ethz.seb.sebserver.gbl.model.sebconfig.ConfigurationValue;
import ch.ethz.seb.sebserver.webservice.servicelayer.sebconfig.impl.ExamConfigImportHandler.PListNode.Type;
@ -38,7 +40,7 @@ public class ExamConfigImportHandler extends DefaultHandler {
Constants.XML_PLIST_INTEGER));
private final Consumer<ConfigurationValue> valueConsumer;
private final Function<String, Long> attributeNameIdResolver;
private final Function<String, ConfigurationAttribute> attributeResolver;
private final Long institutionId;
private final Long configId;
@ -48,11 +50,11 @@ public class ExamConfigImportHandler extends DefaultHandler {
final Long institutionId,
final Long configId,
final Consumer<ConfigurationValue> valueConsumer,
final Function<String, Long> attributeNameIdResolver) {
final Function<String, ConfigurationAttribute> attributeResolver) {
super();
this.valueConsumer = valueConsumer;
this.attributeNameIdResolver = attributeNameIdResolver;
this.attributeResolver = attributeResolver;
this.institutionId = institutionId;
this.configId = configId;
}
@ -218,7 +220,6 @@ public class ExamConfigImportHandler extends DefaultHandler {
} else {
parent.value += "," + top.value;
}
return;
}
@ -226,39 +227,56 @@ public class ExamConfigImportHandler extends DefaultHandler {
? parent.name + "." + top.name
: top.name;
final Long attributeId = this.attributeNameIdResolver.apply(attrName);
if (attributeId == null) {
if (log.isDebugEnabled()) {
log.debug("Skip unknown configuration attribute: {}", attrName);
}
} else {
saveValue(attrName, attributeId, top.listIndex, top.value);
}
final ConfigurationAttribute attribute = this.attributeResolver.apply(attrName);
saveValue(attrName, attribute, top.listIndex, top.value);
}
} else if (top.type == Type.ARRAY && StringUtils.isNoneBlank(top.value)) {
} else if (top.type == Type.ARRAY) {
this.stack.pop();
final PListNode parent = this.stack.pop();
final PListNode grandParent = this.stack.peek();
this.stack.push(parent);
final String attrName = (parent.type == Type.DICT && grandParent.type == Type.ARRAY)
? parent.name + "." + top.name
: top.name;
final Long attributeId = this.attributeNameIdResolver.apply(attrName);
final ConfigurationAttribute attribute = this.attributeResolver.apply(attrName);
saveValue(attrName, attributeId, top.listIndex, top.value);
// check if we have a simple values array
if (attribute.type == AttributeType.MULTI_CHECKBOX_SELECTION
|| attribute.type == AttributeType.MULTI_SELECTION) {
saveValue(attrName, attribute, top.listIndex, (top.value == null) ? "" : top.value);
}
} else if (!Constants.XML_PLIST_KEY_NAME.equals(qName)) {
this.stack.pop();
}
}
private void saveValue(final String name, final Long attributeId, final int listIndex, final String value) {
// TODO use AttributeValueConverterService here. Extend the converters with fromXML functionality
private void saveValue(
final String name,
final ConfigurationAttribute attribute,
final int listIndex,
final String value) {
if (attribute == null) {
log.warn("Import of unknown attribute. name={} value={}", name, value);
return;
}
if (value == null) {
log.warn("*********************** Save null value: {}", name);
} else if (StringUtils.isBlank(value)) {
log.warn("*********************** Save blank value: {}", name);
} else {
log.warn("*********************** Save value value: {} : {}", name, value);
}
final ConfigurationValue configurationValue = new ConfigurationValue(
null,
this.institutionId,
this.configId,
attributeId,
attribute.id,
listIndex,
value);
@ -326,6 +344,7 @@ public class ExamConfigImportHandler extends DefaultHandler {
int arrayCounter = 0;
int listIndex = 0;
String value;
boolean saveNullValueAsBlank = false;
protected PListNode(final Type type) {
this.type = type;

View file

@ -21,7 +21,6 @@ import org.cryptonode.jncryptor.AES256JNCryptor;
import org.cryptonode.jncryptor.AES256JNCryptorInputStream;
import org.cryptonode.jncryptor.AES256JNCryptorOutputStream;
import org.cryptonode.jncryptor.CryptorException;
import org.cryptonode.jncryptor.PasswordKey;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.context.annotation.Lazy;
@ -99,7 +98,7 @@ public class PasswordEncryptor implements SebConfigCryptor {
final CharSequence password = context.getPassword();
try {
final byte[] version = new byte[1];
final byte[] version = new byte[4];
input.read(version);
final SequenceInputStream sequenceInputStream = new SequenceInputStream(
@ -134,12 +133,10 @@ public class PasswordEncryptor implements SebConfigCryptor {
try {
final ByteArrayOutputStream out = new ByteArrayOutputStream();
IOUtils.copyLarge(sequenceInputStream, out);
IOUtils.copy(sequenceInputStream, out);
final byte[] ciphertext = out.toByteArray();
final AES256JNCryptor cryptor = new AES256JNCryptor();
cryptor.setPBKDFIterations(10000);
final PasswordKey passwordKey = cryptor.getPasswordKey(Utils.toCharArray(password));
final int versionNumber = cryptor.getVersionNumber();
final byte[] decryptData = cryptor.decryptData(ciphertext, Utils.toCharArray(password));
final ByteArrayInputStream decryptedIn = new ByteArrayInputStream(decryptData);
IOUtils.copyLarge(decryptedIn, output);

View file

@ -49,14 +49,15 @@ public final class SebConfigEncryptionServiceImpl implements SebConfigEncryption
private final Map<Strategy, SebConfigCryptor> encryptors;
public SebConfigEncryptionServiceImpl(final Collection<SebConfigCryptor> encryptors) {
public SebConfigEncryptionServiceImpl(
final Collection<SebConfigCryptor> encryptors) {
this.encryptors = encryptors
.stream()
.flatMap(e -> e.strategies()
.stream()
.map(s -> new ImmutablePair<>(s, e)))
.collect(Collectors.toMap(p -> p.left, p -> p.right));
}
@Override
@ -135,15 +136,12 @@ public final class SebConfigEncryptionServiceImpl implements SebConfigEncryption
newIn = input;
}
if (log.isDebugEnabled()) {
log.debug("Password decryption with strategy: {}", strategy);
}
if ((strategy == Strategy.PASSWORD_PSWD || strategy == Strategy.PASSWORD_PWCC)
&& StringUtils.isBlank(context.getPassword())) {
return new AsyncResult<>(new IllegalArgumentException("Missing Password"));
}
// then decrypt stream
getEncryptor(strategy)
.getOrThrow()
.decrypt(pout, newIn, context);
@ -151,7 +149,6 @@ public final class SebConfigEncryptionServiceImpl implements SebConfigEncryption
IOUtils.copyLarge(pin, output);
return new AsyncResult<>(null);
} catch (final IOException e) {
log.error("Error while stream decrypted data: ", e);
return new AsyncResult<>(e);

View file

@ -8,13 +8,11 @@
package ch.ethz.seb.sebserver.webservice.servicelayer.sebconfig.impl;
import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.io.PipedInputStream;
import java.io.PipedOutputStream;
import java.io.SequenceInputStream;
import java.util.Collection;
import java.util.List;
import java.util.concurrent.Future;
@ -29,7 +27,6 @@ import org.slf4j.LoggerFactory;
import org.springframework.context.annotation.Lazy;
import org.springframework.stereotype.Service;
import ch.ethz.seb.sebserver.gbl.Constants;
import ch.ethz.seb.sebserver.gbl.api.APIMessage;
import ch.ethz.seb.sebserver.gbl.api.APIMessage.APIMessageException;
import ch.ethz.seb.sebserver.gbl.api.APIMessage.FieldValidationException;
@ -336,17 +333,22 @@ public class SebExamConfigServiceImpl implements SebExamConfigService {
Future<Exception> streamDecrypted = null;
try {
final InputStream cryptIn = this.unzip(input);
final InputStream cryptIn = this.examConfigIO.unzip(input);
final PipedInputStream plainIn = new PipedInputStream();
final PipedOutputStream cryptOut = new PipedOutputStream(plainIn);
// decrypt
streamDecrypted = this.sebConfigEncryptionService.streamDecrypted(
cryptOut,
cryptIn,
EncryptionContext.contextOf(password));
// if zipped, unzip attach unzip stream first
final InputStream _plainIn = this.examConfigIO.unzip(plainIn);
// parse XML and import
this.examConfigIO.importPlainXML(
plainIn,
_plainIn,
newConfig.institutionId,
newConfig.id);
@ -371,33 +373,6 @@ public class SebExamConfigServiceImpl implements SebExamConfigService {
});
}
private InputStream unzip(final InputStream input) throws Exception {
final byte[] zipHeader = new byte[Constants.GZIP_HEADER_LENGTH];
input.read(zipHeader);
//final int zipType = ByteBuffer.wrap(zipHeader).getInt();
final boolean isZipped = Byte.toUnsignedInt(zipHeader[0]) == Constants.GZIP_ID1
&& Byte.toUnsignedInt(zipHeader[1]) == Constants.GZIP_ID2
&& Byte.toUnsignedInt(zipHeader[2]) == Constants.GZIP_CM;
if (isZipped) {
final InputStream sequencedInput = new SequenceInputStream(
new ByteArrayInputStream(zipHeader),
input);
final PipedInputStream pipedIn = new PipedInputStream();
final PipedOutputStream pipedOut = new PipedOutputStream(pipedIn);
this.zipService.read(pipedOut, sequencedInput);
return pipedIn;
} else {
return new SequenceInputStream(
new ByteArrayInputStream(zipHeader),
input);
}
}
private void exportPlainOnly(
final ConfigurationFormat exportFormat,
final OutputStream out,

View file

@ -41,7 +41,6 @@ public class ZipServiceImpl implements ZipService {
try {
zipOutputStream = new GZIPOutputStream(out);
IOUtils.copyLarge(in, zipOutputStream);
} catch (final IOException e) {
@ -75,7 +74,6 @@ public class ZipServiceImpl implements ZipService {
try {
zipInputStream = new GZIPInputStream(in);
IOUtils.copyLarge(zipInputStream, out);
} catch (final IOException e) {

View file

@ -15,11 +15,12 @@ import java.util.Collections;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.function.Function;
import java.util.stream.Collectors;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.context.annotation.Lazy;
import org.springframework.stereotype.Component;
import org.springframework.util.CollectionUtils;
@ -41,6 +42,8 @@ import ch.ethz.seb.sebserver.webservice.servicelayer.sebconfig.AttributeValueCon
@WebServiceProfile
public class TableConverter implements AttributeValueConverter {
private static final Logger log = LoggerFactory.getLogger(TableConverter.class);
public static final Set<AttributeType> SUPPORTED_TYPES = Collections.unmodifiableSet(
new HashSet<>(Arrays.asList(
AttributeType.TABLE,
@ -139,6 +142,7 @@ public class TableConverter implements AttributeValueConverter {
}
writeRows(
value,
out,
getSortedChildAttributes(attribute),
values,
@ -153,47 +157,99 @@ public class TableConverter implements AttributeValueConverter {
}
private void writeRows(
final ConfigurationValue tableValue,
final OutputStream out,
final Map<Long, ConfigurationAttribute> attributeMap,
final List<ConfigurationAttribute> sortedAttributes,
final List<List<ConfigurationValue>> values,
final AttributeValueConverterService attributeValueConverterService,
final boolean xml) throws IOException {
final Iterator<List<ConfigurationValue>> irows = values.iterator();
for (int index = 0; index < values.size(); index++) {
final List<ConfigurationValue> rowValues = values.get(index);
while (irows.hasNext()) {
final List<ConfigurationValue> rowValues = irows.next();
out.write((xml) ? XML_DICT_START : JSON_DICT_START);
final Iterator<ConfigurationValue> ivalue = rowValues.iterator();
final Iterator<ConfigurationAttribute> attrItr = sortedAttributes.iterator();
while (attrItr.hasNext()) {
final ConfigurationAttribute attr = attrItr.next();
ConfigurationValue value = rowValues.stream()
.filter(val -> attr.id.equals(val.attributeId))
.findFirst()
.orElse(null);
if (value == null) {
log.warn("Missing AttributeValue for ConfigurationAttribute: {}. Create ad-hoc attribute", attr);
value = new ConfigurationValue(
-1L,
tableValue.institutionId,
tableValue.configurationId,
attr.id,
index,
attr.defaultValue);
}
final ConfigurationValue _value = value;
while (ivalue.hasNext()) {
final ConfigurationValue value = ivalue.next();
final ConfigurationAttribute attr = attributeMap.get(value.attributeId);
final AttributeValueConverter converter =
attributeValueConverterService.getAttributeValueConverter(attr);
if (xml) {
converter.convertToXML(out, attr, a -> value);
converter.convertToXML(out, attr, a -> _value);
} else {
converter.convertToJSON(out, attr, a -> value);
converter.convertToJSON(out, attr, a -> _value);
}
if (!xml && ivalue.hasNext()) {
if (!xml && attrItr.hasNext()) {
out.write(Utils.toByteArray(Constants.LIST_SEPARATOR));
}
}
out.write((xml) ? XML_DICT_END : JSON_DICT_END);
if (!xml && irows.hasNext()) {
if (!xml && index < values.size() - 1) {
out.write(Utils.toByteArray(Constants.LIST_SEPARATOR));
}
out.flush();
}
// final Iterator<List<ConfigurationValue>> irows = values.iterator();
//
// while (irows.hasNext()) {
// final List<ConfigurationValue> rowValues = irows.next();
// out.write((xml) ? XML_DICT_START : JSON_DICT_START);
//
// final Iterator<ConfigurationValue> ivalue = rowValues.iterator();
//
// while (ivalue.hasNext()) {
// final ConfigurationValue value = ivalue.next();
// final ConfigurationAttribute attr = attributeMap.get(value.attributeId);
// final AttributeValueConverter converter =
// attributeValueConverterService.getAttributeValueConverter(attr);
//
// if (xml) {
// converter.convertToXML(out, attr, a -> value);
// } else {
// converter.convertToJSON(out, attr, a -> value);
// }
//
// if (!xml && ivalue.hasNext()) {
// out.write(Utils.toByteArray(Constants.LIST_SEPARATOR));
// }
// }
// out.write((xml) ? XML_DICT_END : JSON_DICT_END);
//
// if (!xml && irows.hasNext()) {
// out.write(Utils.toByteArray(Constants.LIST_SEPARATOR));
// }
//
// out.flush();
// }
}
private Map<Long, ConfigurationAttribute> getSortedChildAttributes(final ConfigurationAttribute attribute) {
private List<ConfigurationAttribute> getSortedChildAttributes(final ConfigurationAttribute attribute) {
return this.configurationAttributeDAO
.allMatching(new FilterMap().putIfAbsent(
ConfigurationAttribute.FILTER_ATTR_PARENT_ID,
@ -201,9 +257,7 @@ public class TableConverter implements AttributeValueConverter {
.getOrThrow()
.stream()
.sorted()
.collect(Collectors.toMap(
attr -> attr.id,
Function.identity()));
.collect(Collectors.toList());
}
}

View file

@ -158,6 +158,7 @@ INSERT IGNORE INTO configuration_attribute VALUES
(98, 'prohibitedProcesses.originalName', 'TEXT_FIELD', 93, null, null, null, ''),
(99, 'prohibitedProcesses.identifier', 'TEXT_FIELD', 93, null, null, null, ''),
(100, 'prohibitedProcesses.strongKill', 'CHECKBOX', 93, null, null, null, 'false'),
(101, 'prohibitedProcesses.currentUser', 'CHECKBOX', 93, null, null, null, 'false'),
(200, 'URLFilterEnable', 'CHECKBOX', null, null, null, null, 'false'),
(201, 'URLFilterEnableContentFilter', 'CHECKBOX', null, null, null, null, 'false'),

View file

@ -17,10 +17,26 @@ import java.util.function.Function;
import org.junit.Test;
import ch.ethz.seb.sebserver.gbl.model.sebconfig.AttributeType;
import ch.ethz.seb.sebserver.gbl.model.sebconfig.ConfigurationAttribute;
import ch.ethz.seb.sebserver.gbl.model.sebconfig.ConfigurationValue;
public class ExamConfigImportHandlerTest {
private static final Function<String, ConfigurationAttribute> attributeResolver =
name -> new ConfigurationAttribute(
getId(name),
null, name, (name.contains("array")) ? AttributeType.MULTI_SELECTION : null, null, null, null,
null);
private static final Long getId(final String name) {
try {
return Long.parseLong(String.valueOf(name.charAt(name.length() - 1)));
} catch (final Exception e) {
return -1L;
}
}
@Test
public void simpleStringValueTest() throws Exception {
final ValueCollector valueCollector = new ValueCollector();
@ -28,7 +44,7 @@ public class ExamConfigImportHandlerTest {
1L,
1L,
valueCollector,
name -> Long.parseLong(String.valueOf(name.charAt(name.length() - 1))));
attributeResolver);
final String attribute = "param1";
final String value = "value1";
@ -60,7 +76,7 @@ public class ExamConfigImportHandlerTest {
1L,
1L,
valueCollector,
name -> Long.parseLong(String.valueOf(name.charAt(name.length() - 1))));
attributeResolver);
final String attribute = "param2";
final String value = "22";
@ -92,7 +108,7 @@ public class ExamConfigImportHandlerTest {
1L,
1L,
valueCollector,
name -> Long.parseLong(String.valueOf(name.charAt(name.length() - 1))));
attributeResolver);
final String attribute = "param3";
final String value = "true";
@ -123,7 +139,7 @@ public class ExamConfigImportHandlerTest {
1L,
1L,
valueCollector,
name -> Long.parseLong(String.valueOf(name.charAt(name.length() - 1))));
attributeResolver);
final String attribute = "array1";
final String value1 = "val1";
@ -166,9 +182,9 @@ public class ExamConfigImportHandlerTest {
public void dictOfValuesTest() throws Exception {
final ValueCollector valueCollector = new ValueCollector();
final List<String> attrNamesCollector = new ArrayList<>();
final Function<String, Long> attrConverter = attrName -> {
final Function<String, ConfigurationAttribute> attrConverter = attrName -> {
attrNamesCollector.add(attrName);
return Long.parseLong(String.valueOf(attrName.charAt(attrName.length() - 1)));
return attributeResolver.apply(attrName);
};
final ExamConfigImportHandler candidate = new ExamConfigImportHandler(
1L,
@ -235,9 +251,9 @@ public class ExamConfigImportHandlerTest {
public void arrayOfDictOfValuesTest() throws Exception {
final ValueCollector valueCollector = new ValueCollector();
final List<String> attrNamesCollector = new ArrayList<>();
final Function<String, Long> attrConverter = attrName -> {
final Function<String, ConfigurationAttribute> attrConverter = attrName -> {
attrNamesCollector.add(attrName);
return Long.parseLong(String.valueOf(attrName.charAt(attrName.length() - 1)));
return attributeResolver.apply(attrName);
};
final ExamConfigImportHandler candidate = new ExamConfigImportHandler(
1L,
@ -308,9 +324,7 @@ public class ExamConfigImportHandlerTest {
valueCollector.values.toString());
assertEquals(
"[attribute.attr1, attribute.attr2, attribute.attr3, "
+ "attribute.attr1, attribute.attr2, attribute.attr3, "
+ "attribute.attr1, attribute.attr2, attribute.attr3]",
"[attribute.attr1, attribute.attr2, attribute.attr3, attribute.attr1, attribute.attr2, attribute.attr3, attribute.attr1, attribute.attr2, attribute.attr3, attribute]",
attrNamesCollector.toString());
}