diff --git a/src/main/java/fr/inra/oresing/checker/CheckerOnOneVariableComponentLineChecker.java b/src/main/java/fr/inra/oresing/checker/CheckerOnOneVariableComponentLineChecker.java
index d85a994f8866a577cb77f9b39e14ffac09450062..4c3a9420dd19a2557503c4c1f80f7c48370a136f 100644
--- a/src/main/java/fr/inra/oresing/checker/CheckerOnOneVariableComponentLineChecker.java
+++ b/src/main/java/fr/inra/oresing/checker/CheckerOnOneVariableComponentLineChecker.java
@@ -6,8 +6,8 @@ import fr.inra.oresing.model.ReferenceColumn;
import fr.inra.oresing.model.ReferenceDatum;
import fr.inra.oresing.model.VariableComponentKey;
import fr.inra.oresing.persistence.SqlPrimitiveType;
-import fr.inra.oresing.rest.DefaultValidationCheckResult;
import fr.inra.oresing.rest.ValidationCheckResult;
+import fr.inra.oresing.rest.validationcheckresults.DefaultValidationCheckResult;
import fr.inra.oresing.transformer.LineTransformer;
import org.assertj.core.util.Strings;
diff --git a/src/main/java/fr/inra/oresing/checker/DateLineChecker.java b/src/main/java/fr/inra/oresing/checker/DateLineChecker.java
index cd6d06289ca0e1e3602abf3e69b277dfb9ee2902..b1ba76f25661e6f063252ca02f68063ead7c9ae4 100644
--- a/src/main/java/fr/inra/oresing/checker/DateLineChecker.java
+++ b/src/main/java/fr/inra/oresing/checker/DateLineChecker.java
@@ -4,6 +4,7 @@ import com.fasterxml.jackson.annotation.JsonIgnore;
import com.google.common.collect.ImmutableMap;
import fr.inra.oresing.persistence.SqlPrimitiveType;
import fr.inra.oresing.rest.ValidationCheckResult;
+import fr.inra.oresing.rest.validationcheckresults.DateValidationCheckResult;
import fr.inra.oresing.transformer.LineTransformer;
import java.time.format.DateTimeFormatter;
diff --git a/src/main/java/fr/inra/oresing/checker/FloatChecker.java b/src/main/java/fr/inra/oresing/checker/FloatChecker.java
index 5e86a8adef9655e6c9b8654192d5f4fd44ae3c45..7038e6b8b89949f0f40b126cace2589b94ab01d5 100644
--- a/src/main/java/fr/inra/oresing/checker/FloatChecker.java
+++ b/src/main/java/fr/inra/oresing/checker/FloatChecker.java
@@ -2,8 +2,8 @@ package fr.inra.oresing.checker;
import com.fasterxml.jackson.annotation.JsonIgnore;
import com.google.common.collect.ImmutableMap;
+import fr.inra.oresing.rest.validationcheckresults.DefaultValidationCheckResult;
import fr.inra.oresing.persistence.SqlPrimitiveType;
-import fr.inra.oresing.rest.DefaultValidationCheckResult;
import fr.inra.oresing.rest.ValidationCheckResult;
import fr.inra.oresing.transformer.LineTransformer;
@@ -53,4 +53,4 @@ public class FloatChecker implements CheckerOnOneVariableComponentLineChecker<Fl
public SqlPrimitiveType getSqlType() {
return SqlPrimitiveType.NUMERIC;
}
-}
\ No newline at end of file
+}
diff --git a/src/main/java/fr/inra/oresing/checker/GroovyLineChecker.java b/src/main/java/fr/inra/oresing/checker/GroovyLineChecker.java
index fb29c552e884e62e8096407007e6bc74b20290e6..fa2444058c1aec46020c4faa78498f166f411b81 100644
--- a/src/main/java/fr/inra/oresing/checker/GroovyLineChecker.java
+++ b/src/main/java/fr/inra/oresing/checker/GroovyLineChecker.java
@@ -6,7 +6,7 @@ import fr.inra.oresing.groovy.GroovyExpression;
import fr.inra.oresing.model.Datum;
import fr.inra.oresing.model.ReferenceDatum;
import fr.inra.oresing.model.SomethingThatCanProvideEvaluationContext;
-import fr.inra.oresing.rest.DefaultValidationCheckResult;
+import fr.inra.oresing.rest.validationcheckresults.DefaultValidationCheckResult;
import fr.inra.oresing.rest.ValidationCheckResult;
import java.util.Optional;
diff --git a/src/main/java/fr/inra/oresing/checker/IntegerChecker.java b/src/main/java/fr/inra/oresing/checker/IntegerChecker.java
index b442755d1c10959b4923f87c668435c922e95236..d5693c1efe45573b0daac0ed9a4aa019b1eac3b2 100644
--- a/src/main/java/fr/inra/oresing/checker/IntegerChecker.java
+++ b/src/main/java/fr/inra/oresing/checker/IntegerChecker.java
@@ -3,8 +3,8 @@ package fr.inra.oresing.checker;
import com.fasterxml.jackson.annotation.JsonIgnore;
import com.google.common.collect.ImmutableMap;
import fr.inra.oresing.persistence.SqlPrimitiveType;
-import fr.inra.oresing.rest.DefaultValidationCheckResult;
import fr.inra.oresing.rest.ValidationCheckResult;
+import fr.inra.oresing.rest.validationcheckresults.DefaultValidationCheckResult;
import fr.inra.oresing.transformer.LineTransformer;
public class IntegerChecker implements CheckerOnOneVariableComponentLineChecker<IntegerCheckerConfiguration> {
diff --git a/src/main/java/fr/inra/oresing/checker/InvalidDatasetContentException.java b/src/main/java/fr/inra/oresing/checker/InvalidDatasetContentException.java
index c783e9079ca3189260759aa17f24d8e7d6c8caea..a730d38767af31754b51ee90163803f893af4332 100644
--- a/src/main/java/fr/inra/oresing/checker/InvalidDatasetContentException.java
+++ b/src/main/java/fr/inra/oresing/checker/InvalidDatasetContentException.java
@@ -3,7 +3,7 @@ package fr.inra.oresing.checker;
import com.google.common.collect.*;
import fr.inra.oresing.OreSiTechnicalException;
import fr.inra.oresing.rest.CsvRowValidationCheckResult;
-import fr.inra.oresing.rest.DefaultValidationCheckResult;
+import fr.inra.oresing.rest.validationcheckresults.DefaultValidationCheckResult;
import fr.inra.oresing.rest.ValidationCheckResult;
import java.util.List;
diff --git a/src/main/java/fr/inra/oresing/checker/ReferenceLineChecker.java b/src/main/java/fr/inra/oresing/checker/ReferenceLineChecker.java
index 48573745ade35044a4d92c062ebe2989c03a1e1b..421221db31b01fbbb6254715cd04dcff8990da82 100644
--- a/src/main/java/fr/inra/oresing/checker/ReferenceLineChecker.java
+++ b/src/main/java/fr/inra/oresing/checker/ReferenceLineChecker.java
@@ -3,6 +3,7 @@ package fr.inra.oresing.checker;
import com.fasterxml.jackson.annotation.JsonIgnore;
import com.google.common.collect.ImmutableMap;
import fr.inra.oresing.persistence.SqlPrimitiveType;
+import fr.inra.oresing.rest.validationcheckresults.ReferenceValidationCheckResult;
import fr.inra.oresing.transformer.LineTransformer;
import java.util.UUID;
@@ -70,4 +71,4 @@ public class ReferenceLineChecker implements CheckerOnOneVariableComponentLineCh
public SqlPrimitiveType getSqlType() {
return SqlPrimitiveType.TEXT;
}
-}
\ No newline at end of file
+}
diff --git a/src/main/java/fr/inra/oresing/checker/RegularExpressionChecker.java b/src/main/java/fr/inra/oresing/checker/RegularExpressionChecker.java
index 2efb0ab79c29b6dd7a455ec59fef5d1c5788744f..e0cd07418f62ee69bfc9b2ed5c29a374a2221b65 100644
--- a/src/main/java/fr/inra/oresing/checker/RegularExpressionChecker.java
+++ b/src/main/java/fr/inra/oresing/checker/RegularExpressionChecker.java
@@ -2,8 +2,8 @@ package fr.inra.oresing.checker;
import com.fasterxml.jackson.annotation.JsonIgnore;
import com.google.common.collect.ImmutableMap;
+import fr.inra.oresing.rest.validationcheckresults.DefaultValidationCheckResult;
import fr.inra.oresing.persistence.SqlPrimitiveType;
-import fr.inra.oresing.rest.DefaultValidationCheckResult;
import fr.inra.oresing.rest.ValidationCheckResult;
import fr.inra.oresing.transformer.LineTransformer;
@@ -61,4 +61,4 @@ public class RegularExpressionChecker implements CheckerOnOneVariableComponentLi
public SqlPrimitiveType getSqlType() {
return SqlPrimitiveType.TEXT;
}
-}
\ No newline at end of file
+}
diff --git a/src/main/java/fr/inra/oresing/model/ReferenceDatum.java b/src/main/java/fr/inra/oresing/model/ReferenceDatum.java
index 8ac944ecee78acc8b0259fbd03b54576f970796e..8c912fa63a185b3a64847ca0791fd4dc7413aefc 100644
--- a/src/main/java/fr/inra/oresing/model/ReferenceDatum.java
+++ b/src/main/java/fr/inra/oresing/model/ReferenceDatum.java
@@ -6,7 +6,6 @@ import java.util.LinkedHashMap;
import java.util.Map;
public class ReferenceDatum implements SomethingThatCanProvideEvaluationContext {
-
private final Map<ReferenceColumn, String> values;
public ReferenceDatum() {
@@ -46,4 +45,4 @@ public class ReferenceDatum implements SomethingThatCanProvideEvaluationContext
public ImmutableMap<String, Object> getEvaluationContext() {
return ImmutableMap.of("datum", asMap());
}
-}
+}
\ No newline at end of file
diff --git a/src/main/java/fr/inra/oresing/rest/ConfigurationParsingResult.java b/src/main/java/fr/inra/oresing/rest/ConfigurationParsingResult.java
index 6c6cea221dc1e8b94ad06288c627b09c080f012d..8d5cf59e91d01476ed9a5c8370ef12b2b3a6cd01 100644
--- a/src/main/java/fr/inra/oresing/rest/ConfigurationParsingResult.java
+++ b/src/main/java/fr/inra/oresing/rest/ConfigurationParsingResult.java
@@ -5,6 +5,7 @@ import com.google.common.collect.ImmutableSet;
import fr.inra.oresing.groovy.GroovyExpression;
import fr.inra.oresing.model.Configuration;
import fr.inra.oresing.model.VariableComponentKey;
+import fr.inra.oresing.rest.validationcheckresults.DefaultValidationCheckResult;
import lombok.Value;
import javax.annotation.Nullable;
diff --git a/src/main/java/fr/inra/oresing/rest/OreSiService.java b/src/main/java/fr/inra/oresing/rest/OreSiService.java
index d2d00b2d2b5c211cfdf74ccc1eaa59a9e3e50ff2..a1dec03be21af32daca88252cfce4ce2ef6df025 100644
--- a/src/main/java/fr/inra/oresing/rest/OreSiService.java
+++ b/src/main/java/fr/inra/oresing/rest/OreSiService.java
@@ -3,66 +3,23 @@ package fr.inra.oresing.rest;
import com.google.common.base.Charsets;
import com.google.common.base.Preconditions;
import com.google.common.base.Predicate;
-import com.google.common.collect.BiMap;
-import com.google.common.collect.HashBiMap;
-import com.google.common.collect.HashMultimap;
-import com.google.common.collect.ImmutableList;
-import com.google.common.collect.ImmutableMap;
-import com.google.common.collect.ImmutableMultiset;
-import com.google.common.collect.ImmutableSet;
-import com.google.common.collect.ImmutableSetMultimap;
-import com.google.common.collect.ImmutableSortedSet;
-import com.google.common.collect.Iterables;
-import com.google.common.collect.Iterators;
-import com.google.common.collect.Maps;
-import com.google.common.collect.MoreCollectors;
-import com.google.common.collect.Ordering;
-import com.google.common.collect.SetMultimap;
-import com.google.common.collect.Sets;
+import com.google.common.collect.*;
import com.google.common.primitives.Ints;
import fr.inra.oresing.OreSiTechnicalException;
-import fr.inra.oresing.checker.CheckerFactory;
-import fr.inra.oresing.checker.DateLineChecker;
-import fr.inra.oresing.checker.DateValidationCheckResult;
-import fr.inra.oresing.checker.FloatChecker;
-import fr.inra.oresing.checker.IntegerChecker;
-import fr.inra.oresing.checker.InvalidDatasetContentException;
-import fr.inra.oresing.checker.LineChecker;
-import fr.inra.oresing.checker.ReferenceLineChecker;
-import fr.inra.oresing.checker.ReferenceLineCheckerConfiguration;
-import fr.inra.oresing.checker.ReferenceValidationCheckResult;
+import fr.inra.oresing.ValidationLevel;
+import fr.inra.oresing.checker.*;
import fr.inra.oresing.groovy.CommonExpression;
import fr.inra.oresing.groovy.Expression;
import fr.inra.oresing.groovy.GroovyContextHelper;
import fr.inra.oresing.groovy.StringGroovyExpression;
-import fr.inra.oresing.model.Application;
-import fr.inra.oresing.model.Authorization;
-import fr.inra.oresing.model.BinaryFile;
-import fr.inra.oresing.model.BinaryFileDataset;
-import fr.inra.oresing.model.Configuration;
-import fr.inra.oresing.model.Data;
-import fr.inra.oresing.model.Datum;
-import fr.inra.oresing.model.LocalDateTimeRange;
-import fr.inra.oresing.model.ReferenceColumn;
-import fr.inra.oresing.model.ReferenceDatum;
-import fr.inra.oresing.model.ReferenceValue;
-import fr.inra.oresing.model.VariableComponentKey;
+import fr.inra.oresing.model.*;
import fr.inra.oresing.model.internationalization.Internationalization;
import fr.inra.oresing.model.internationalization.InternationalizationDisplay;
import fr.inra.oresing.model.internationalization.InternationalizationReferenceMap;
-import fr.inra.oresing.persistence.AuthenticationService;
-import fr.inra.oresing.persistence.BinaryFileInfos;
-import fr.inra.oresing.persistence.DataRepository;
-import fr.inra.oresing.persistence.DataRow;
-import fr.inra.oresing.persistence.Ltree;
-import fr.inra.oresing.persistence.OreSiRepository;
-import fr.inra.oresing.persistence.ReferenceValueRepository;
-import fr.inra.oresing.persistence.SqlPolicy;
-import fr.inra.oresing.persistence.SqlSchema;
-import fr.inra.oresing.persistence.SqlSchemaForApplication;
-import fr.inra.oresing.persistence.SqlService;
+import fr.inra.oresing.persistence.*;
import fr.inra.oresing.persistence.roles.OreSiRightOnApplicationRole;
import fr.inra.oresing.persistence.roles.OreSiUserRole;
+import fr.inra.oresing.rest.validationcheckresults.*;
import fr.inra.oresing.transformer.TransformerFactory;
import lombok.Value;
import lombok.extern.slf4j.Slf4j;
@@ -94,20 +51,7 @@ import java.time.LocalDateTime;
import java.time.ZoneOffset;
import java.time.format.DateTimeFormatter;
import java.time.temporal.ChronoUnit;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.Comparator;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Iterator;
-import java.util.LinkedHashMap;
-import java.util.LinkedHashSet;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
-import java.util.Optional;
-import java.util.Set;
-import java.util.UUID;
+import java.util.*;
import java.util.function.BiFunction;
import java.util.function.Consumer;
import java.util.function.Function;
@@ -370,11 +314,13 @@ public class OreSiService {
}
public UUID addReference(Application app, String refType, MultipartFile file) throws IOException {
+ ReferenceValueRepository referenceValueRepository = repo.getRepository(app).referenceValue();
authenticationService.setRoleForClient();
UUID fileId = storeFile(app, file,"");
Configuration conf = app.getConfiguration();
Configuration.ReferenceDescription ref = conf.getReferences().get(refType);
+ final ImmutableMap<String, UUID> storedReferences = referenceValueRepository.getReferenceIdPerKeys(refType);
ImmutableSet<LineChecker> lineCheckers = checkerFactory.getReferenceValidationLineCheckers(app, refType);
Optional<ReferenceLineChecker> selfLineChecker = lineCheckers.stream()
@@ -390,6 +336,7 @@ public class OreSiService {
Function<Ltree, Ltree> getHierarchicalReferenceFn;
Map<Ltree, Ltree> buildedHierarchicalKeys = new HashMap<>();
Map<Ltree, Ltree> parentreferenceMap = new HashMap<>();
+ ListMultimap<Ltree, Integer> hierarchicalKeys = LinkedListMultimap.create();
if (toUpdateCompositeReference.isPresent()) {
Configuration.CompositeReferenceDescription compositeReferenceDescription = toUpdateCompositeReference.get();
boolean root = Iterables.get(compositeReferenceDescription.getComponents(), 0).getReference().equals(refType);
@@ -413,7 +360,6 @@ public class OreSiService {
getHierarchicalReferenceFn = (reference) -> reference;
}
- ReferenceValueRepository referenceValueRepository = repo.getRepository(app).referenceValue();
CSVFormat csvFormat = CSVFormat.DEFAULT
.withDelimiter(ref.getSeparator())
@@ -423,7 +369,7 @@ public class OreSiService {
Iterator<CSVRecord> linesIterator = csvParser.iterator();
CSVRecord headerRow = linesIterator.next();
ImmutableList<String> columns = Streams.stream(headerRow).collect(ImmutableList.toImmutableList());
- Function<CSVRecord, ReferenceDatum> csvRecordToLineAsMapFn = line -> {
+ Function<CSVRecord, RowWithReferenceDatum> csvRecordToLineAsMapFn = line -> {
Iterator<String> currentHeader = columns.iterator();
ReferenceDatum referenceDatum = new ReferenceDatum();
line.forEach(value -> {
@@ -431,15 +377,16 @@ public class OreSiService {
ReferenceColumn referenceColumn = new ReferenceColumn(header);
referenceDatum.put(referenceColumn, value);
});
- return referenceDatum;
+ int lineNumber = Ints.checkedCast(line.getRecordNumber());
+ return new RowWithReferenceDatum(lineNumber, referenceDatum);
};
List<CsvRowValidationCheckResult> rowErrors = new LinkedList<>();
Stream<CSVRecord> recordStream = Streams.stream(csvParser);
if (isRecursive) {
- recordStream = addMissingReferences(recordStream, selfLineChecker, recursiveComponentDescription, columns, ref, parentreferenceMap);
+ recordStream = addMissingReferences(recordStream, selfLineChecker, recursiveComponentDescription, columns, ref, parentreferenceMap, rowErrors, refType);
+ InvalidDatasetContentException.checkErrorsIsEmpty(rowErrors);
}
- List<Ltree> hierarchicalKeys = new LinkedList<>();
Optional<InternationalizationReferenceMap> internationalizationReferenceMap = Optional.ofNullable(conf)
.map(configuration -> conf.getInternationalization())
.map(inter -> inter.getReferences())
@@ -452,7 +399,8 @@ public class OreSiService {
.map(internationalizationDisplay -> internationalizationDisplay.getPattern());
Stream<ReferenceValue> referenceValuesStream = recordStream
.map(csvRecordToLineAsMapFn)
- .map(referenceDatum -> {
+ .map(rowWithReferenceDatum -> {
+ ReferenceDatum referenceDatum = rowWithReferenceDatum.getReferenceDatum();
Map<String, Set<UUID>> refsLinkedTo = new LinkedHashMap<>();
lineCheckers.forEach(lineChecker -> {
ValidationCheckResult validationCheckResult = lineChecker.checkReference(referenceDatum);
@@ -467,18 +415,24 @@ public class OreSiService {
.add(referenceId);
}
} else {
- rowErrors.add(new CsvRowValidationCheckResult(validationCheckResult, csvParser.getCurrentLineNumber()));
+ rowErrors.add(new CsvRowValidationCheckResult(validationCheckResult, rowWithReferenceDatum.getLineNumber()));
}
});
- ReferenceValue e = new ReferenceValue();
+ final ReferenceValue e = new ReferenceValue();
Preconditions.checkState(!ref.getColumns().isEmpty(), "aucune colonne désignée comme clé naturelle pour le référentiel " + refType);
- String naturalKeyAsString = ref.getKeyColumns().stream()
- .map(ReferenceColumn::new)
- .map(referenceDatum::get)
- .filter(StringUtils::isNotEmpty)
- .map(Ltree::escapeToLabel)
- .collect(Collectors.joining(KEYCOLUMN_SEPARATOR));
- Ltree naturalKey = Ltree.fromSql(naturalKeyAsString);
+ Ltree naturalKey;
+ if (ref.getKeyColumns().isEmpty()) {
+ UUID technicalId = e.getId();
+ naturalKey = Ltree.fromUuid(technicalId);
+ } else {
+ String naturalKeyAsString = ref.getKeyColumns().stream()
+ .map(ReferenceColumn::new)
+ .map(referenceDatum::get)
+ .filter(StringUtils::isNotEmpty)
+ .map(Ltree::escapeToLabel)
+ .collect(Collectors.joining(KEYCOLUMN_SEPARATOR));
+ naturalKey = Ltree.fromSql(naturalKeyAsString);
+ }
Ltree recursiveNaturalKey = naturalKey;
final Ltree refTypeAsLabel = Ltree.fromUnescapedString(refType);
if (isRecursive) {
@@ -486,7 +440,9 @@ public class OreSiService {
.map(referenceLineChecker -> referenceLineChecker.getReferenceValues())
.map(values -> values.get(naturalKey.getSql()))
.filter(key -> key != null)
- .ifPresent(key -> e.setId(key));
+ .ifPresent(key -> {
+ e.setId(key);
+ });
Ltree parentKey = parentreferenceMap.getOrDefault(recursiveNaturalKey, null);
while (parentKey != null) {
recursiveNaturalKey = Ltree.join(parentKey, recursiveNaturalKey);
@@ -508,6 +464,15 @@ public class OreSiService {
getHierarchicalReferenceFn.apply(selfHierarchicalReference);
referenceDatum.putAll(InternationalizationDisplay.getDisplays(displayPattern, displayColumns, referenceDatum));
buildedHierarchicalKeys.put(naturalKey, hierarchicalKey);
+
+ /**
+ * on remplace l'id par celle en base si elle existe
+ * a noter que pour les references récursives on récupère l'id depuis referenceLineChecker.getReferenceValues() ce qui revient au même
+ */
+
+ if (storedReferences.containsKey(hierarchicalKey.getSql())) {
+ e.setId(storedReferences.get(hierarchicalKey.getSql()));
+ }
e.setBinaryFile(fileId);
e.setReferenceType(refType);
e.setHierarchicalKey(hierarchicalKey);
@@ -516,22 +481,18 @@ public class OreSiService {
e.setNaturalKey(naturalKey);
e.setApplication(app.getId());
e.setRefValues(referenceDatum.asMap());
- return e;
- })
- .sorted(Comparator.comparing(a -> a.getHierarchicalKey().getSql()))
- .map(e -> {
- if (hierarchicalKeys.contains(e.getHierarchicalKey())) {
- /*envoyer un message de warning : le refType avec la clef e.getNaturalKey existe en plusieurs exemplaires
- dans le fichier. Seule la première ligne est enregistrée
- */
-// ValidationCheckResult validationCheckResult = new ValidationCheckResult()
-// rowErrors.add(new CsvRowValidationCheckResult(validationCheckResult, csvParser.getCurrentLineNumber()));
+ if (hierarchicalKeys.containsKey(e.getHierarchicalKey())) {
+ ValidationCheckResult validationCheckResult = new DuplicationLineValidationCheckResult(DuplicationLineValidationCheckResult.FileType.REFERENCES, refType, ValidationLevel.ERROR, e.getHierarchicalKey(), rowWithReferenceDatum.getLineNumber(), hierarchicalKeys.get(e.getHierarchicalKey()));
+ rowErrors.add(new CsvRowValidationCheckResult(validationCheckResult, rowWithReferenceDatum.getLineNumber()));
+ hierarchicalKeys.put(e.getHierarchicalKey(), rowWithReferenceDatum.getLineNumber());
+ return null;
} else {
- hierarchicalKeys.add(e.getHierarchicalKey());
+ hierarchicalKeys.put(e.getHierarchicalKey(), rowWithReferenceDatum.getLineNumber());
+ return e;
}
- return e;
})
- .filter(e -> e != null);
+ .filter(e -> e != null)
+ .sorted(Comparator.comparing(a -> a.getHierarchicalKey().getSql()));
referenceValueRepository.storeAll(referenceValuesStream);
InvalidDatasetContentException.checkErrorsIsEmpty(rowErrors);
}
@@ -539,11 +500,12 @@ public class OreSiService {
return fileId;
}
- private Stream<CSVRecord> addMissingReferences(Stream<CSVRecord> recordStream, Optional<ReferenceLineChecker> selfLineChecker, Optional<Configuration.CompositeReferenceComponentDescription> recursiveComponentDescription, ImmutableList<String> columns, Configuration.ReferenceDescription ref, Map<Ltree, Ltree> referenceMap) {
+ private Stream<CSVRecord> addMissingReferences(Stream<CSVRecord> recordStream, Optional<ReferenceLineChecker> selfLineChecker, Optional<Configuration.CompositeReferenceComponentDescription> recursiveComponentDescription, ImmutableList<String> columns, Configuration.ReferenceDescription ref, Map<Ltree, Ltree> referenceMap, List<CsvRowValidationCheckResult> rowErrors, String refType) {
Integer parentRecursiveIndex = recursiveComponentDescription
.map(rcd -> rcd.getParentRecursiveKey())
.map(rck -> columns.indexOf(rck))
.orElse(null);
+ ListMultimap<String, Long> missingParentReferences = LinkedListMultimap.create();
if (parentRecursiveIndex == null || parentRecursiveIndex < 0) {
return recordStream;
}
@@ -554,31 +516,44 @@ public class OreSiService {
List<CSVRecord> collect = recordStream
.peek(csvrecord -> {
String s = csvrecord.get(parentRecursiveIndex);
+ String naturalKey = ref.getKeyColumns()
+ .stream()
+ .map(kc -> columns.indexOf(kc))
+ .map(k -> Strings.isNullOrEmpty(csvrecord.get(k))?null:Ltree.escapeToLabel(csvrecord.get(k)))
+ .filter(k->k!=null)
+ .collect(Collectors.joining("__"));
+ if (!referenceUUIDs.containsKey(naturalKey)) {
+ referenceUUIDs.put(naturalKey, UUID.randomUUID());
+ }
if (!Strings.isNullOrEmpty(s)) {
- String naturalKey;
try {
s = Ltree.escapeToLabel(s);
- naturalKey = ref.getKeyColumns()
- .stream()
- .map(kc -> columns.indexOf(kc))
- .map(k -> Ltree.escapeToLabel(csvrecord.get(k)))
- .collect(Collectors.joining("__"));
} catch (IllegalArgumentException e) {
return;
}
referenceMap.put(Ltree.fromSql(naturalKey), Ltree.fromUnescapedString(s));
if (!referenceUUIDs.containsKey(s)) {
- referenceUUIDs.put(s, UUID.randomUUID());
- }
- if (!referenceUUIDs.containsKey(naturalKey)) {
- referenceUUIDs.put(naturalKey, UUID.randomUUID());
+ final UUID uuid = UUID.randomUUID();
+ referenceUUIDs.put(s, uuid);
+ missingParentReferences.put(s, csvrecord.getRecordNumber());
}
}
+ missingParentReferences.removeAll(naturalKey);
return;
})
.collect(Collectors.toList());
selfLineChecker
.ifPresent(slc -> slc.setReferenceValues(ImmutableMap.copyOf(referenceUUIDs)));
+
+ if (!missingParentReferences.isEmpty()) {
+ missingParentReferences.asMap().entrySet().stream()
+ .forEach(entry -> {
+ final String missingParentReference = entry.getKey();
+ entry.getValue().stream().forEach(
+ lineNumber -> rowErrors.add(new CsvRowValidationCheckResult(new MissingParentLineValidationCheckResult(lineNumber, refType, missingParentReference, referenceUUIDs.keySet()), lineNumber))
+ );
+ });
+ }
return collect.stream();
}
@@ -1525,9 +1500,15 @@ public class OreSiService {
Datum datum;
}
+ @Value
+ private static class RowWithReferenceDatum {
+ int lineNumber;
+ ReferenceDatum referenceDatum;
+ }
+
@Value
private static class ParsedCsvRow {
int lineNumber;
List<Map.Entry<String, String>> columns;
}
-}
\ No newline at end of file
+}
diff --git a/src/main/java/fr/inra/oresing/checker/DateValidationCheckResult.java b/src/main/java/fr/inra/oresing/rest/validationcheckresults/DateValidationCheckResult.java
similarity index 94%
rename from src/main/java/fr/inra/oresing/checker/DateValidationCheckResult.java
rename to src/main/java/fr/inra/oresing/rest/validationcheckresults/DateValidationCheckResult.java
index 66d533df785a7bf97c607f7f82ff6a58368d2471..8bf361a4f9a4d3445b4af60f24e80c6c43b32a5a 100644
--- a/src/main/java/fr/inra/oresing/checker/DateValidationCheckResult.java
+++ b/src/main/java/fr/inra/oresing/rest/validationcheckresults/DateValidationCheckResult.java
@@ -1,7 +1,9 @@
-package fr.inra.oresing.checker;
+package fr.inra.oresing.rest.validationcheckresults;
import com.google.common.collect.ImmutableMap;
import fr.inra.oresing.ValidationLevel;
+import fr.inra.oresing.checker.CheckerTarget;
+import fr.inra.oresing.checker.DateLineChecker;
import fr.inra.oresing.rest.ValidationCheckResult;
import lombok.Value;
diff --git a/src/main/java/fr/inra/oresing/rest/DefaultValidationCheckResult.java b/src/main/java/fr/inra/oresing/rest/validationcheckresults/DefaultValidationCheckResult.java
similarity index 90%
rename from src/main/java/fr/inra/oresing/rest/DefaultValidationCheckResult.java
rename to src/main/java/fr/inra/oresing/rest/validationcheckresults/DefaultValidationCheckResult.java
index cbb109f950a4fa3fc5756d5887744e40d307384c..66f227a2d934647598cc3fc4a2822de0d1fc0029 100644
--- a/src/main/java/fr/inra/oresing/rest/DefaultValidationCheckResult.java
+++ b/src/main/java/fr/inra/oresing/rest/validationcheckresults/DefaultValidationCheckResult.java
@@ -1,7 +1,8 @@
-package fr.inra.oresing.rest;
+package fr.inra.oresing.rest.validationcheckresults;
import com.google.common.collect.ImmutableMap;
import fr.inra.oresing.ValidationLevel;
+import fr.inra.oresing.rest.ValidationCheckResult;
import lombok.Value;
import java.util.Map;
diff --git a/src/main/java/fr/inra/oresing/rest/validationcheckresults/DuplicationLineValidationCheckResult.java b/src/main/java/fr/inra/oresing/rest/validationcheckresults/DuplicationLineValidationCheckResult.java
new file mode 100644
index 0000000000000000000000000000000000000000..75f3606e5c759277b0e3c9a140f21440cc2404a1
--- /dev/null
+++ b/src/main/java/fr/inra/oresing/rest/validationcheckresults/DuplicationLineValidationCheckResult.java
@@ -0,0 +1,52 @@
+package fr.inra.oresing.rest.validationcheckresults;
+
+import com.google.common.collect.ImmutableMap;
+import fr.inra.oresing.ValidationLevel;
+import fr.inra.oresing.persistence.Ltree;
+import fr.inra.oresing.rest.ValidationCheckResult;
+
+import java.util.List;
+import java.util.Map;
+
+public class DuplicationLineValidationCheckResult implements ValidationCheckResult {
+
+ public static String MESSAGE_FOR_REFERENCES ="duplicatedLineInReference";
+ public static String MESSAGE_FOR_DATATYPES ="duplicatedLineInDatatype";
+ ValidationLevel level;
+ String message;
+
+ Map<String, Object> messageParams;
+
+ public DuplicationLineValidationCheckResult(FileType filetype, String file, ValidationLevel level, Ltree hierarchicalKey, int currentLineNumber, List<Integer> otherLines) {
+ this.level = level;
+ this.message = FileType.DATATYPE.message;
+ this.messageParams = ImmutableMap.of(
+ "file", file,
+ "lineNumber", currentLineNumber,
+ "otherLines", otherLines,
+ "duplicateKey", hierarchicalKey.getSql()
+ );
+ }
+
+ @Override
+ public ValidationLevel getLevel() {
+ return level;
+ }
+
+ @Override
+ public String getMessage() {
+ return message;
+ }
+
+ @Override
+ public Map<String, Object> getMessageParams() {
+ return this.messageParams;
+ }
+ public enum FileType{
+ DATATYPE(MESSAGE_FOR_DATATYPES),REFERENCES(MESSAGE_FOR_REFERENCES);
+ String message;
+ FileType(String message) {
+ this.message = message;
+ }
+ };
+}
\ No newline at end of file
diff --git a/src/main/java/fr/inra/oresing/rest/validationcheckresults/MissingParentLineValidationCheckResult.java b/src/main/java/fr/inra/oresing/rest/validationcheckresults/MissingParentLineValidationCheckResult.java
new file mode 100644
index 0000000000000000000000000000000000000000..a4723a3ef0b1adaf3add28e88182260882fc8d7f
--- /dev/null
+++ b/src/main/java/fr/inra/oresing/rest/validationcheckresults/MissingParentLineValidationCheckResult.java
@@ -0,0 +1,38 @@
+package fr.inra.oresing.rest.validationcheckresults;
+
+import com.google.common.collect.ImmutableMap;
+import fr.inra.oresing.ValidationLevel;
+import fr.inra.oresing.rest.ValidationCheckResult;
+
+import java.util.Map;
+import java.util.Set;
+
+public class MissingParentLineValidationCheckResult implements ValidationCheckResult {
+ public static String MISSING_PARENT_LINE_IN_RECURSIVE_REFERENCE ="missingParentLineInRecursiveReference";
+ Map<String, Object> messageParams;
+
+ public MissingParentLineValidationCheckResult(long lineNumber, String refType, String missingReferencesKey, Set<String> knownReferences ) {
+ super();
+ this.messageParams = ImmutableMap.of(
+ "lineNumber", lineNumber,
+ "references", refType,
+ "missingReferencesKey", missingReferencesKey,
+ "knownReferences", knownReferences
+ );
+ }
+
+ @Override
+ public ValidationLevel getLevel() {
+ return ValidationLevel.ERROR;
+ }
+
+ @Override
+ public String getMessage() {
+ return MISSING_PARENT_LINE_IN_RECURSIVE_REFERENCE;
+ }
+
+ @Override
+ public Map<String, Object> getMessageParams() {
+ return this.messageParams;
+ }
+}
\ No newline at end of file
diff --git a/src/main/java/fr/inra/oresing/checker/ReferenceValidationCheckResult.java b/src/main/java/fr/inra/oresing/rest/validationcheckresults/ReferenceValidationCheckResult.java
similarity index 90%
rename from src/main/java/fr/inra/oresing/checker/ReferenceValidationCheckResult.java
rename to src/main/java/fr/inra/oresing/rest/validationcheckresults/ReferenceValidationCheckResult.java
index a565f433151a73e799169c543951b3ca55bc0eed..1af149b85acefbe1b65fb4cb0c0157a7fc282929 100644
--- a/src/main/java/fr/inra/oresing/checker/ReferenceValidationCheckResult.java
+++ b/src/main/java/fr/inra/oresing/rest/validationcheckresults/ReferenceValidationCheckResult.java
@@ -1,7 +1,8 @@
-package fr.inra.oresing.checker;
+package fr.inra.oresing.rest.validationcheckresults;
import com.google.common.collect.ImmutableMap;
import fr.inra.oresing.ValidationLevel;
+import fr.inra.oresing.checker.CheckerTarget;
import fr.inra.oresing.rest.ValidationCheckResult;
import lombok.Value;
diff --git a/src/test/java/fr/inra/oresing/rest/Fixtures.java b/src/test/java/fr/inra/oresing/rest/Fixtures.java
index 377615bba489b574fd5e6a7053c65884feefccf1..f1cfa32968db2920aea8c83833fe859e7f8bf159 100644
--- a/src/test/java/fr/inra/oresing/rest/Fixtures.java
+++ b/src/test/java/fr/inra/oresing/rest/Fixtures.java
@@ -468,6 +468,26 @@ public class Fixtures {
return "/data/pros/donnees_prelevement_pro.csv";
}
+ public String getDuplicatedApplicationConfigurationResourceName() {
+ return "/data/duplication/duplication.yaml";
+ }
+
+ public Map<String, String> getDuplicatedReferentielFiles() {
+ Map<String, String> referentielFiles = new LinkedHashMap<>();
+ referentielFiles.put("typezonewithoutduplication", "/data/duplication/typezone.csv");
+ referentielFiles.put("typezonewithduplication", "/data/duplication/typezoneduplique.csv");
+ referentielFiles.put("zonewithoutduplication", "/data/duplication/zone_etude.csv");
+ referentielFiles.put("zonewithduplication", "/data/duplication/zone_etude_dupliqué.csv");
+ referentielFiles.put("zonewithmissingparent", "/data/duplication/zone_etude_missing_parent.csv");
+ return referentielFiles;
+ }
+
+ public Map<String, String> getDuplicatedDataFiles() {
+ Map<String, String> referentielFiles = new LinkedHashMap<>();
+ referentielFiles.put("data_without_duplicateds", "/data/duplication/data.csv");
+ return referentielFiles;
+ }
+
public String getPhysicoChimieSolsProDataResourceName() {
return "/data/pros/physico_chimie_sols.csv";
}
diff --git a/src/test/java/fr/inra/oresing/rest/OreSiResourcesTest.java b/src/test/java/fr/inra/oresing/rest/OreSiResourcesTest.java
index e645b42ddfc22a6c293f3b85912a3252f5474f04..4f329573cbe8d46c995d7892fcd0c14966c9f10e 100644
--- a/src/test/java/fr/inra/oresing/rest/OreSiResourcesTest.java
+++ b/src/test/java/fr/inra/oresing/rest/OreSiResourcesTest.java
@@ -6,6 +6,8 @@ import com.google.common.io.Resources;
import com.jayway.jsonpath.JsonPath;
import fr.inra.oresing.OreSiNg;
import fr.inra.oresing.OreSiTechnicalException;
+import fr.inra.oresing.ValidationLevel;
+import fr.inra.oresing.checker.InvalidDatasetContentException;
import fr.inra.oresing.persistence.AuthenticationService;
import fr.inra.oresing.persistence.JsonRowMapper;
import lombok.extern.slf4j.Slf4j;
@@ -13,6 +15,7 @@ import org.apache.commons.io.IOUtils;
import org.apache.commons.lang3.StringUtils;
import org.hamcrest.Matchers;
import org.hamcrest.core.Is;
+import org.hamcrest.core.IsEqual;
import org.hamcrest.core.IsNull;
import org.json.JSONArray;
import org.junit.Assert;
@@ -36,6 +39,7 @@ import org.springframework.test.web.servlet.MockMvc;
import org.springframework.test.web.servlet.ResultActions;
import org.springframework.test.web.servlet.request.MockMvcRequestBuilders;
import org.springframework.test.web.servlet.result.MockMvcResultMatchers;
+import org.springframework.web.util.NestedServletException;
import javax.servlet.http.Cookie;
import java.io.IOException;
@@ -899,6 +903,233 @@ public class OreSiResourcesTest {
}
}
+ @Test
+ public void addDuplicatedTest() throws Exception {
+ authenticationService.addUserRightCreateApplication(userId);
+ try (InputStream configurationFile = fixtures.getClass().getResourceAsStream(fixtures.getDuplicatedApplicationConfigurationResourceName())) {
+ MockMultipartFile configuration = new MockMultipartFile("file", "duplicated.yaml", "text/plain", configurationFile);
+ mockMvc.perform(MockMvcRequestBuilders.multipart("/api/v1/applications/duplicated")
+ .file(configuration)
+ .cookie(authCookie))
+ .andExpect(MockMvcResultMatchers.status().is2xxSuccessful());
+ }
+ String message;
+
+ //on charge le fichier de type zone d'étude
+ final String typezonewithoutduplicationDuplication = fixtures.getDuplicatedReferentielFiles().get("typezonewithoutduplication");
+ try (InputStream refStream = fixtures.getClass().getResourceAsStream(typezonewithoutduplicationDuplication)) {
+ MockMultipartFile refFile = new MockMultipartFile("file", "type_zone_etude.csv", "text/plain", refStream);
+ message = mockMvc.perform(MockMvcRequestBuilders.multipart("/api/v1/applications/duplicated/references/{refType}", "types_de_zones_etudes")
+ .file(refFile)
+ .cookie(authCookie))
+ .andExpect(status().is2xxSuccessful())
+ .andReturn().getResponse().getContentAsString();
+ }
+
+ // on vérifie le nombre de ligne
+ message = mockMvc.perform(MockMvcRequestBuilders.get("/api/v1/applications/duplicated/references/{refType}", "types_de_zones_etudes")
+ .cookie(authCookie))
+ .andExpect(status().is2xxSuccessful())
+ .andExpect(jsonPath("$.referenceValues.length()", IsEqual.equalTo(2)))
+ .andReturn().getResponse().getContentAsString();
+
+
+ //on recharge le fichier de type zone d'étude
+ try (InputStream refStream = fixtures.getClass().getResourceAsStream(typezonewithoutduplicationDuplication)) {
+ MockMultipartFile refFile = new MockMultipartFile("file", "type_zone_etude2.csv", "text/plain", refStream);
+ message = mockMvc.perform(MockMvcRequestBuilders.multipart("/api/v1/applications/duplicated/references/{refType}", "types_de_zones_etudes")
+ .file(refFile)
+ .cookie(authCookie))
+ .andExpect(status().is2xxSuccessful())
+ .andReturn().getResponse().getContentAsString();
+ }
+
+ //il doit toujours y avoir le même nombre de ligne
+ message = mockMvc.perform(MockMvcRequestBuilders.get("/api/v1/applications/duplicated/references/{refType}", "types_de_zones_etudes")
+ .cookie(authCookie))
+ .andExpect(status().is2xxSuccessful())
+ .andExpect(jsonPath("$.referenceValues.length()", IsEqual.equalTo(2)))
+ .andReturn().getResponse().getContentAsString();
+
+
+ //on charge le fichier de zone type d'étude avec une duplication
+ final String typezonewithduplicationDuplication = fixtures.getDuplicatedReferentielFiles().get("typezonewithduplication");
+ try (InputStream refStream = fixtures.getClass().getResourceAsStream(typezonewithduplicationDuplication)) {
+ MockMultipartFile refFile = new MockMultipartFile("file", "type_zone_etude_duplicate.csv", "text/plain", refStream);
+ final ResultActions error = mockMvc
+ .perform(MockMvcRequestBuilders.multipart("/api/v1/applications/duplicated/references/{refType}", "types_de_zones_etudes")
+ .file(refFile)
+ .cookie(authCookie));
+ Assert.fail();
+ } catch (NestedServletException e) {
+ Assert.assertTrue(e.getCause() instanceof InvalidDatasetContentException);
+ final InvalidDatasetContentException invalidDatasetContentException = (InvalidDatasetContentException) e.getCause();
+ final List<CsvRowValidationCheckResult> errors = invalidDatasetContentException.getErrors();
+ Assert.assertEquals(1, errors.size());
+ Assert.assertEquals(4, errors.get(0).getLineNumber());
+ final ValidationCheckResult validationCheckResult = errors.get(0).getValidationCheckResult();
+ Assert.assertEquals(ValidationLevel.ERROR, validationCheckResult.getLevel());
+ Assert.assertEquals("duplicatedLineInDatatype", validationCheckResult.getMessage());
+ final Map<String, Object> messageParams = validationCheckResult.getMessageParams();
+ Assert.assertEquals("types_de_zones_etudes", messageParams.get("file"));
+ Assert.assertEquals(4, messageParams.get("lineNumber"));
+ Assert.assertArrayEquals(new Integer[]{3, 4}, ((List) messageParams.get("otherLines")).toArray());
+ Assert.assertEquals("zone20", messageParams.get("duplicateKey"));
+ }
+
+ //il doit toujours y avoir le même nombre de ligne
+ message = mockMvc.perform(MockMvcRequestBuilders.get("/api/v1/applications/duplicated/references/{refType}", "types_de_zones_etudes")
+ .cookie(authCookie))
+ .andExpect(status().is2xxSuccessful())
+ .andExpect(jsonPath("$.referenceValues.length()", IsEqual.equalTo(2)))
+ .andReturn().getResponse().getContentAsString();
+/*
+on test le dépôt d'un fichier récursif
+ */
+
+
+//on charge le fichier de zone d'étude
+ final String zonewithoutduplicationDuplication = fixtures.getDuplicatedReferentielFiles().get("zonewithoutduplication");
+ try (InputStream refStream = fixtures.getClass().getResourceAsStream(zonewithoutduplicationDuplication)) {
+ MockMultipartFile refFile = new MockMultipartFile("file", "zone_etude.csv", "text/plain", refStream);
+ message = mockMvc.perform(MockMvcRequestBuilders.multipart("/api/v1/applications/duplicated/references/{refType}", "zones_etudes")
+ .file(refFile)
+ .cookie(authCookie))
+ .andExpect(status().is2xxSuccessful())
+ .andReturn().getResponse().getContentAsString();
+ }
+
+ // on vérifie le nombre de ligne
+ message = mockMvc.perform(MockMvcRequestBuilders.get("/api/v1/applications/duplicated/references/{refType}", "zones_etudes")
+ .cookie(authCookie))
+ .andExpect(status().is2xxSuccessful())
+ .andExpect(jsonPath("$.referenceValues.length()", IsEqual.equalTo(2)))
+ .andReturn().getResponse().getContentAsString();
+
+
+ //on recharge le fichier de zone d'étude
+ try (InputStream refStream = fixtures.getClass().getResourceAsStream(zonewithoutduplicationDuplication)) {
+ MockMultipartFile refFile = new MockMultipartFile("file", "zone_etude2.csv", "text/plain", refStream);
+ message = mockMvc.perform(MockMvcRequestBuilders.multipart("/api/v1/applications/duplicated/references/{refType}", "zones_etudes")
+ .file(refFile)
+ .cookie(authCookie))
+ .andExpect(status().is2xxSuccessful())
+ .andReturn().getResponse().getContentAsString();
+ }
+
+ //il doit toujours y avoir le même nombre de ligne
+ message = mockMvc.perform(MockMvcRequestBuilders.get("/api/v1/applications/duplicated/references/{refType}", "zones_etudes")
+ .cookie(authCookie))
+ .andExpect(status().is2xxSuccessful())
+ .andExpect(jsonPath("$.referenceValues.length()", IsEqual.equalTo(2)))
+ .andReturn().getResponse().getContentAsString();
+
+
+ //on charge le fichier de zone d'étudeavec une duplication
+ final String zonewithduplicationDuplication = fixtures.getDuplicatedReferentielFiles().get("zonewithduplication");
+ try (InputStream refStream = fixtures.getClass().getResourceAsStream(zonewithduplicationDuplication)) {
+ MockMultipartFile refFile = new MockMultipartFile("file", "zone_etude_duplicated.csv", "text/plain", refStream);
+ final ResultActions error = mockMvc
+ .perform(MockMvcRequestBuilders.multipart("/api/v1/applications/duplicated/references/{refType}", "zones_etudes")
+ .file(refFile)
+ .cookie(authCookie));
+ Assert.fail();
+ } catch (NestedServletException e) {
+ Assert.assertTrue(e.getCause() instanceof InvalidDatasetContentException);
+ final InvalidDatasetContentException invalidDatasetContentException = (InvalidDatasetContentException) e.getCause();
+ final List<CsvRowValidationCheckResult> errors = invalidDatasetContentException.getErrors();
+ Assert.assertEquals(1, errors.size());
+ Assert.assertEquals(4, errors.get(0).getLineNumber());
+ final ValidationCheckResult validationCheckResult = errors.get(0).getValidationCheckResult();
+ Assert.assertEquals(ValidationLevel.ERROR, validationCheckResult.getLevel());
+ Assert.assertEquals("duplicatedLineInDatatype", validationCheckResult.getMessage());
+ final Map<String, Object> messageParams = validationCheckResult.getMessageParams();
+ Assert.assertEquals("zones_etudes", messageParams.get("file"));
+ Assert.assertEquals(4, messageParams.get("lineNumber"));
+ Assert.assertArrayEquals(new Integer[]{2, 4}, ((List) messageParams.get("otherLines")).toArray());
+ Assert.assertEquals("site1", messageParams.get("duplicateKey"));
+ }
+
+ //il doit toujours y avoir le même nombre de ligne
+ message = mockMvc.perform(MockMvcRequestBuilders.get("/api/v1/applications/duplicated/references/{refType}", "zones_etudes")
+ .cookie(authCookie))
+ .andExpect(status().is2xxSuccessful())
+ .andExpect(jsonPath("$.referenceValues.length()", IsEqual.equalTo(2)))
+ .andReturn().getResponse().getContentAsString();
+
+ //on charge le fichier de zone d'étudeavec une duplication
+ final String zonewithmissingParent = fixtures.getDuplicatedReferentielFiles().get("zonewithmissingparent");
+ try (InputStream refStream = fixtures.getClass().getResourceAsStream(zonewithmissingParent)) {
+ MockMultipartFile refFile = new MockMultipartFile("file", "zone_etude_missing_parent.csv", "text/plain", refStream);
+ final ResultActions error = mockMvc
+ .perform(MockMvcRequestBuilders.multipart("/api/v1/applications/duplicated/references/{refType}", "zones_etudes")
+ .file(refFile)
+ .cookie(authCookie));
+ Assert.fail();
+ } catch (NestedServletException e) {
+ Assert.assertTrue(e.getCause() instanceof InvalidDatasetContentException);
+ final InvalidDatasetContentException invalidDatasetContentException = (InvalidDatasetContentException) e.getCause();
+ final List<CsvRowValidationCheckResult> errors = invalidDatasetContentException.getErrors();
+ Assert.assertEquals(1, errors.size());
+ Assert.assertEquals(3, errors.get(0).getLineNumber());
+ final ValidationCheckResult validationCheckResult = errors.get(0).getValidationCheckResult();
+ Assert.assertEquals(ValidationLevel.ERROR, validationCheckResult.getLevel());
+ Assert.assertEquals("missingParentLineInRecursiveReference", validationCheckResult.getMessage());
+ final Map<String, Object> messageParams = validationCheckResult.getMessageParams();
+ Assert.assertEquals("zones_etudes", messageParams.get("references"));
+ Assert.assertEquals(3L, messageParams.get("lineNumber"));
+ Assert.assertEquals("site3", messageParams.get("missingReferencesKey"));
+ Assert.assertTrue(Set.of("site3","site1.site2","site1","site2").containsAll((Set) messageParams.get("knownReferences")));
+ }
+
+ //il doit toujours y avoir le même nombre de ligne
+ message = mockMvc.perform(MockMvcRequestBuilders.get("/api/v1/applications/duplicated/references/{refType}", "zones_etudes")
+ .cookie(authCookie))
+ .andExpect(status().is2xxSuccessful())
+ .andExpect(jsonPath("$.referenceValues.length()", IsEqual.equalTo(2)))
+ .andReturn().getResponse().getContentAsString();
+
+ //on teste un dépot de fichier de données
+ final String dataWithoutDuplicateds = fixtures.getDuplicatedDataFiles().get("data_without_duplicateds");
+ try (InputStream refStream = fixtures.getClass().getResourceAsStream(dataWithoutDuplicateds)) {
+ MockMultipartFile refFile = new MockMultipartFile("file", "data_without_duplicateds.csv", "text/plain", refStream);
+ mockMvc.perform(MockMvcRequestBuilders.multipart("/api/v1/applications/duplicated/data/dty")
+ .file(refFile)
+ .cookie(authCookie))
+ .andExpect(MockMvcResultMatchers.status().is2xxSuccessful());
+ }
+ //on teste le nombre de ligne
+ try (InputStream refStream = fixtures.getClass().getResourceAsStream(dataWithoutDuplicateds)) {
+ final String response = mockMvc.perform(get("/api/v1/applications/duplicated/data/dty")
+ .cookie(authCookie))
+ .andExpect(status().is2xxSuccessful())
+ .andExpect(jsonPath("$.totalRows", IsEqual.equalTo(4
+ )))
+ .andReturn().getResponse().getContentAsString();
+ log.debug(response);
+ }
+
+ // on redepose le fichier
+
+ try (InputStream refStream = fixtures.getClass().getResourceAsStream(dataWithoutDuplicateds)) {
+ MockMultipartFile refFile = new MockMultipartFile("file", "data_without_duplicateds.csv", "text/plain", refStream);
+ mockMvc.perform(MockMvcRequestBuilders.multipart("/api/v1/applications/duplicated/data/dty")
+ .file(refFile)
+ .cookie(authCookie))
+ .andExpect(MockMvcResultMatchers.status().is2xxSuccessful());
+ }
+ // le nombre de ligne est inchangé
+ /*try (InputStream refStream = fixtures.getClass().getResourceAsStream(dataWithoutDuplicateds)) {
+ final String response = mockMvc.perform(get("/api/v1/applications/duplicated/data/dty")
+ .cookie(authCookie))
+ .andExpect(status().is2xxSuccessful())
+ .andExpect(jsonPath("$.totalRows", IsEqual.equalTo(4
+ )))
+ .andReturn().getResponse().getContentAsString();
+ log.debug(response);
+ }*/
+ }
+
@Test
public void addApplicationOLAC() throws Exception {
authenticationService.addUserRightCreateApplication(userId);
@@ -1012,7 +1243,7 @@ public class OreSiResourcesTest {
for (Map.Entry<String, String> entry : fixtures.getFluxMeteoForetEssaiDataResourceName().entrySet()) {
try (InputStream refStream = fixtures.getClass().getResourceAsStream(entry.getValue())) {
MockMultipartFile refFile = new MockMultipartFile("file", "flux_meteo_dataResult.csv", "text/plain", refStream);
- mockMvc.perform(MockMvcRequestBuilders.multipart("/api/v1/applications/foret/data/"+entry.getKey())
+ mockMvc.perform(MockMvcRequestBuilders.multipart("/api/v1/applications/foret/data/" + entry.getKey())
.file(refFile)
.cookie(authCookie))
.andExpect(MockMvcResultMatchers.status().is2xxSuccessful());
diff --git a/src/test/resources/data/duplication/data.csv b/src/test/resources/data/duplication/data.csv
new file mode 100644
index 0000000000000000000000000000000000000000..250c8f3114d79f262d32b7c155ebfe71ee647b46
--- /dev/null
+++ b/src/test/resources/data/duplication/data.csv
@@ -0,0 +1,5 @@
+site;date
+site1.site2;23/02/1980
+site1.site2;24/02/1980
+site1;23/02/1980
+site1;24/02/1980
\ No newline at end of file
diff --git a/src/test/resources/data/duplication/duplication.yaml b/src/test/resources/data/duplication/duplication.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..cae0e427ef5054a09cbd3f3306a02cd446a6b8fd
--- /dev/null
+++ b/src/test/resources/data/duplication/duplication.yaml
@@ -0,0 +1,72 @@
+version: 0
+application:
+ name: duplication
+ version: 1
+compositeReferences:
+ localizations:
+ components:
+ - reference: zones_etudes
+ parentRecursiveKey: parent
+references:
+ types_de_zones_etudes:
+ keyColumns: [nom]
+ columns:
+ nom:
+ zones_etudes:
+ validations:
+ parent_ref:
+ description: référence au parent
+ checker:
+ name: Reference
+ params:
+ refType: zones_etudes
+ columns: parent
+ required: false
+ codify: true
+ keyColumns: [nom]
+ columns:
+ nom:
+ parent:
+dataTypes:
+ dty:
+ authorization:
+ dataGroups:
+ reference:
+ data:
+ - localization
+ - Date
+ label: "Reference"
+ authorizationScopes:
+ authorization_zoneEtude:
+ component: zones_etudes
+ variable: localization
+ timeScope:
+ component: day
+ variable: Date
+ data:
+ Date:
+ components:
+ day:
+ checker:
+ name: Date
+ params:
+ pattern: dd/MM/yyyy
+ localization:
+ components:
+ zones_etudes:
+ checker:
+ name: Reference
+ params:
+ refType: zones_etudes
+ format:
+ headerLine: 1
+ firstRowLine: 2
+ columns:
+ - header: "site"
+ boundTo:
+ variable: localization
+ component: zones_etudes
+ - header: "date"
+ boundTo:
+ variable: Date
+ component: day
\ No newline at end of file
diff --git a/src/test/resources/data/duplication/typezone.csv b/src/test/resources/data/duplication/typezone.csv
new file mode 100644
index 0000000000000000000000000000000000000000..7372a66a40d3f56f2ef919953758b872436ab8ab
--- /dev/null
+++ b/src/test/resources/data/duplication/typezone.csv
@@ -0,0 +1,3 @@
+nom;
+zone1;
+zone2;
\ No newline at end of file
diff --git a/src/test/resources/data/duplication/typezoneduplique.csv b/src/test/resources/data/duplication/typezoneduplique.csv
new file mode 100644
index 0000000000000000000000000000000000000000..a9be259d5e6302c06f5437d7da6170c844dcc0a1
--- /dev/null
+++ b/src/test/resources/data/duplication/typezoneduplique.csv
@@ -0,0 +1,5 @@
+nom;
+zone10;
+zone20;
+zone20;
+zone30;
\ No newline at end of file
diff --git a/src/test/resources/data/duplication/zone_etude.csv b/src/test/resources/data/duplication/zone_etude.csv
new file mode 100644
index 0000000000000000000000000000000000000000..10a42bca849fc8e95f8ac3fe8460af7b681b9100
--- /dev/null
+++ b/src/test/resources/data/duplication/zone_etude.csv
@@ -0,0 +1,3 @@
+nom;parent;
+site1;;
+site2;site1;
\ No newline at end of file
diff --git "a/src/test/resources/data/duplication/zone_etude_dupliqu\303\251.csv" "b/src/test/resources/data/duplication/zone_etude_dupliqu\303\251.csv"
new file mode 100644
index 0000000000000000000000000000000000000000..cc4f7bf7b09df0d3e5d54c74e956fdd406f15b16
--- /dev/null
+++ "b/src/test/resources/data/duplication/zone_etude_dupliqu\303\251.csv"
@@ -0,0 +1,4 @@
+nom;parent;
+site1;;
+site2;site1;
+site1;;
\ No newline at end of file
diff --git a/src/test/resources/data/duplication/zone_etude_missing_parent.csv b/src/test/resources/data/duplication/zone_etude_missing_parent.csv
new file mode 100644
index 0000000000000000000000000000000000000000..b0bdde17e050dd6f9cc2c2f557063c37898b2683
--- /dev/null
+++ b/src/test/resources/data/duplication/zone_etude_missing_parent.csv
@@ -0,0 +1,4 @@
+nom;parent;
+site1;;
+site1;site3;
+site2;site1;
\ No newline at end of file
diff --git a/ui2/src/locales/en.json b/ui2/src/locales/en.json
index bf7cfbfe2f1172afc8863e2dd3743186fee822df..4084364098bac84b069d968719c8f9662ec3f846 100644
--- a/ui2/src/locales/en.json
+++ b/ui2/src/locales/en.json
@@ -158,7 +158,10 @@
"requiredValueWithColumn": "For column: <code> {target} </code> the value cannot be zero.",
"timerangeoutofinterval":"The date <code> {value} </code> is incompatible with the date range of the deposit. It must be between <code> {from} </code> and <code> {to} </code>. ",
"badauthorizationscopeforrepository":"Authorization <code> {authorization} </code>) must be <code> {expectedValue} / code> and not <code> {givenValue} </code>",
- "overlappingpublishedversion":"There is a deposited version in the deposit dates have an overlapping period with the deposited version. <code> {overlapingFiles] </code>"
+ "overlappingpublishedversion":"There is a deposited version in the deposit dates have an overlapping period with the deposited version. <code> {overlapingFiles] </code>",
+ "duplicateLineInReference": "In the repository file {file}, line {lineNumber} has the same id {duplicateKey} as lines {otherLines}",
+ "duplicateLineInDatatype": "In data file {file}, line {lineNumber} has the same identifier {duplicateKey} as lines {otherLines}",
+ "missingParentLineInRecursiveReference": "In repository file {references} the id value {missingReferencesKey} for parent does not exist. Accepted values ${knownReferences}"
},
"referencesManagement":{
"actions":"Actions",
@@ -245,4 +248,4 @@
"regEx": ".*",
"star": "*"
}
-}
\ No newline at end of file
+}
diff --git a/ui2/src/locales/fr.json b/ui2/src/locales/fr.json
index ea2ef42d1c8892af8392eb88f03a8d60010da83d..48ea91b1b508fd3c070fb6335560d090db303f5a 100644
--- a/ui2/src/locales/fr.json
+++ b/ui2/src/locales/fr.json
@@ -158,7 +158,10 @@
"requiredValueWithColumn": "Pour la colonne : <code>{target}</code> la valeur ne peut pas être nulle.",
"timerangeoutofinterval": "La date <code>{value}</code> est incompatible avec l'intervale de dates du dépôt. Elle doit être comprise entre <code>{from}</code> et <code>{to}</code>.",
"badauthorizationscopeforrepository": "L'autorisation <code>{authorization}</code>) doit être <code>{expectedValue}/code> et non <code>{givenValue}</code>",
- "overlappingpublishedversion": "Il existe une version déposée dans les dates de dépôt ont une période chevauchante avec la version déposée. <code>{overlapingFiles]</code>"
+ "overlappingpublishedversion": "Il existe une version déposée dans les dates de dépôt ont une période chevauchante avec la version déposée. <code>{overlapingFiles]</code>",
+ "duplicatedLineInReference": "Dans le fichier du référentiel {file}, la ligne {lineNumber} a le même identifiant {duplicateKey} que les lignes {otherLines}",
+ "duplicatedLineInDatatype": "Dans le fichier de données {file}, la ligne {lineNumber} a le même identifiant {duplicateKey} que les lignes {otherLines}",
+ "missingParentLineInRecursiveReference": "Dans le fichier du référentiel {references} la valeur d'identifiant {missingReferencesKey} pour le parent n'existe pas. Valeurs acceptées {knownReferences}"
},
"referencesManagement": {
"actions": "Actions",
@@ -244,4 +247,4 @@
"slash": "/",
"regEx": ".*"
}
-}
\ No newline at end of file
+}