add volumetry distinct interrigationId column

src/main/java/fr/insee/genesis/domain/ports/api/LunaticJsonRawDataApiPort.java

@@ -28,6 +28,7 @@ public interface LunaticJsonRawDataApiPort {
     void updateProcessDates(List<SurveyUnitModel> surveyUnitModels);
     Set<String> findDistinctQuestionnaireIds();
     long countRawResponsesByQuestionnaireId(String campaignId);
+    long countDistinctInterrogationIdsByQuestionnaireId(String questionnaireId);
     Page<LunaticJsonRawDataModel> findRawDataByCampaignIdAndDate(String campaignId, Instant  startDt, Instant endDt, Pageable pageable);
 
     List<LunaticJsonRawDataModel> getRawDataByInterrogationId(String interrogationId);

src/main/java/fr/insee/genesis/domain/ports/api/RawResponseApiPort.java

@@ -24,6 +24,7 @@ public interface RawResponseApiPort {
     List<String> getUnprocessedCollectionInstrumentIds();
     void updateProcessDates(List<SurveyUnitModel> surveyUnitModels);
     Page<RawResponseModel> findRawResponseDataByCampaignIdAndDate(String campaignId, Instant startDate, Instant endDate, Pageable pageable);
+    long countDistinctInterrogationIdsByCollectionInstrumentId(String collectionInstrumentId);
 
     long countByCollectionInstrumentId(String collectionInstrumentId);
 

src/main/java/fr/insee/genesis/domain/ports/api/SurveyUnitApiPort.java

@@ -92,4 +92,6 @@ List<SurveyUnitModel> parseEditedVariables(SurveyUnitInputDto surveyUnitInputDto
     Set<String> findCampaignIdsFrom(SurveyUnitInputDto dto);
 
     long countResponsesByQuestionnaireId(String questionnaireId);
+
+    long countDistinctInterrogationIdsByQuestionnaireAndCollectionInstrumentId(String id);
 }

src/main/java/fr/insee/genesis/domain/ports/spi/LunaticJsonRawDataPersistencePort.java

@@ -28,4 +28,5 @@ public interface LunaticJsonRawDataPersistencePort {
     List<GroupedInterrogation> findProcessedIdsGroupedByQuestionnaireSince(LocalDateTime since);
     List<GroupedInterrogation> findUnprocessedIds();
     Set<String> findUnprocessedInterrogationIdsByCollectionInstrumentId(String collectionInstrumentId);
+    long countDistinctInterrogationIdsByQuestionnaireId(String questionnaireId);
 }

src/main/java/fr/insee/genesis/domain/ports/spi/RawResponsePersistencePort.java

@@ -21,6 +21,6 @@ public interface RawResponsePersistencePort {
     Page<RawResponseModel> findByCampaignIdAndDate(String campaignId, Instant startDate, Instant endDate, Pageable pageable);
     long countByCollectionInstrumentId(String collectionInstrumentId);
     Set<String> findDistinctCollectionInstrumentIds();
-
+    long countDistinctInterrogationIdsByCollectionInstrumentId(String collectionInstrumentId);
     Page<RawResponseModel> findByCollectionInstrumentId(String collectionInstrumentId, Pageable pageable);
 }

src/main/java/fr/insee/genesis/domain/ports/spi/SurveyUnitPersistencePort.java

@@ -72,4 +72,6 @@ List<SurveyUnitModel> findInterrogationIdsByCollectionInstrumentIdAndRecordDateB
     Set<String> findCampaignIdsByQuestionnaireId(String questionnaireId);
 
     long countByQuestionnaireId(String questionnaireId);
+
+    long countDistinctInterrogationIdsByQuestionnaireAndCollectionInstrumentId(String id);
 }

src/main/java/fr/insee/genesis/domain/service/rawdata/LunaticJsonRawDataService.java

@@ -548,6 +548,11 @@ public long countRawResponsesByQuestionnaireId(String campaignId) {
         return lunaticJsonRawDataPersistencePort.countRawResponsesByQuestionnaireId(campaignId);
     }
 
+    @Override
+    public long countDistinctInterrogationIdsByQuestionnaireId(String questionnaireId) {
+        return lunaticJsonRawDataPersistencePort.countDistinctInterrogationIdsByQuestionnaireId(questionnaireId);
+    }
+
     @Override
     public Map<String, List<String>> findProcessedIdsgroupedByQuestionnaireSince(LocalDateTime since) {
         List<GroupedInterrogation> idsByQuestionnaire = lunaticJsonRawDataPersistencePort.findProcessedIdsGroupedByQuestionnaireSince(since);

src/main/java/fr/insee/genesis/domain/service/rawdata/RawResponseService.java

@@ -536,6 +536,11 @@ public Page<RawResponseModel> findRawResponseDataByCampaignIdAndDate(String camp
         return rawResponsePersistencePort.findByCampaignIdAndDate(campaignId,startDate, endDate,pageable);
     }
 
+    @Override
+    public long countDistinctInterrogationIdsByCollectionInstrumentId(String collectionInstrumentId) {
+        return rawResponsePersistencePort.countByCollectionInstrumentId(collectionInstrumentId);
+    }
+
     @Override
     public long countByCollectionInstrumentId(String collectionInstrumentId) {
         return rawResponsePersistencePort.countByCollectionInstrumentId(collectionInstrumentId);

src/main/java/fr/insee/genesis/domain/service/surveyunit/SurveyUnitService.java

@@ -689,4 +689,9 @@ private boolean isLastVariableState(SurveyUnitModel surveyUnitModel, VariableDto
         }
         return true;
     }
+
+    @Override
+    public long countDistinctInterrogationIdsByQuestionnaireAndCollectionInstrumentId(String id) {
+        return surveyUnitPersistencePort.countDistinctInterrogationIdsByQuestionnaireAndCollectionInstrumentId(id);
+    }
 }

src/main/java/fr/insee/genesis/domain/service/volumetry/VolumetryLogService.java

@@ -16,8 +16,11 @@
 import java.time.LocalDateTime;
 import java.time.format.DateTimeFormatter;
 import java.time.format.DateTimeParseException;
+import java.util.ArrayList;
+import java.util.Collections;
 import java.util.HashMap;
 import java.util.HashSet;
+import java.util.List;
 import java.util.Map;
 import java.util.Set;
 import java.util.stream.Stream;
@@ -44,17 +47,23 @@ public Map<String, Long> writeVolumetries(SurveyUnitApiPort surveyUnitApiPort) t
         if (Files.exists(logFilePath)){
             Files.delete(logFilePath);
         }
-        Files.writeString(logFilePath, "campaign;volumetry\n");
+        Files.writeString(logFilePath, "campaign;volumetry;distinctInterrogationIds\n");
 
         //Write lines
-        Set<String> collectionInstrumentIds = surveyUnitApiPort.findDistinctQuestionnairesAndCollectionInstrumentIds();
-        for (String collectionInstrumentId : collectionInstrumentIds) {
+        Set<String> collectionInstrumentIds =
+                surveyUnitApiPort.findDistinctQuestionnairesAndCollectionInstrumentIds();
+
+        List<String> sortedIds = new ArrayList<>(collectionInstrumentIds);
+        Collections.sort(sortedIds);
+
+        for (String collectionInstrumentId : sortedIds) {
             long countResult = surveyUnitApiPort.countResponsesByCollectionInstrumentId(collectionInstrumentId);
             countResult += surveyUnitApiPort.countResponsesByQuestionnaireId(collectionInstrumentId);
 
+            long distinctInterrogationIds =
+                    surveyUnitApiPort.countDistinctInterrogationIdsByQuestionnaireAndCollectionInstrumentId(collectionInstrumentId);
 
-            String line = collectionInstrumentId + ";" + countResult + "\n";
-
+            String line = collectionInstrumentId + ";" + countResult + ";" + distinctInterrogationIds + "\n";
             Files.writeString(logFilePath, line, StandardOpenOption.APPEND);
             responseVolumetricsByQuestionnaireMap.put(collectionInstrumentId, countResult);
         }
@@ -66,54 +75,82 @@ public Map<String, Map<String, Long>> writeRawDataVolumetries(
             LunaticJsonRawDataApiPort lunaticJsonRawDataApiPort,
             RawResponseApiPort rawResponseApiPort
     ) throws IOException {
+
         Map<String, Map<String, Long>> rawDataVolumetricsMap = new HashMap<>();
         rawDataVolumetricsMap.put(Constants.MONGODB_LUNATIC_RAWDATA_COLLECTION_NAME, new HashMap<>());
         rawDataVolumetricsMap.put(Constants.MONGODB_RAW_RESPONSES_COLLECTION_NAME, new HashMap<>());
         rawDataVolumetricsMap.put(Constants.VOLUMETRY_RAW_TOTAL, new HashMap<>());
 
-        Path logFilePath = Path.of(config.getLogFolder()).resolve(Constants.VOLUMETRY_FOLDER_NAME)
+        Path logFilePath = Path.of(config.getLogFolder())
+                .resolve(Constants.VOLUMETRY_FOLDER_NAME)
                 .resolve(
-                        LocalDateTime.now().format(DateTimeFormatter.ofPattern(Constants.VOLUMETRY_FILE_DATE_FORMAT))
-                                + Constants.VOLUMETRY_RAW_FILE_SUFFIX + ".csv");
+                        LocalDateTime.now()
+                                .format(DateTimeFormatter.ofPattern(Constants.VOLUMETRY_FILE_DATE_FORMAT))
+                                + Constants.VOLUMETRY_RAW_FILE_SUFFIX + ".csv"
+                );
+
         Files.createDirectories(logFilePath.getParent());
-        //Overwrite log file with header if exists
-        if (Files.exists(logFilePath)){
+
+        // Overwrite file if exists
+        if (Files.exists(logFilePath)) {
             Files.delete(logFilePath);
         }
 
-        //Write header
-        Files.writeString(logFilePath, "questionnaireId;%s;%s;%s\n"
-                .formatted(
-                        Constants.MONGODB_LUNATIC_RAWDATA_COLLECTION_NAME,
-                        Constants.MONGODB_RAW_RESPONSES_COLLECTION_NAME,
-                        Constants.VOLUMETRY_RAW_TOTAL
-                )
+        Files.writeString(
+                logFilePath,
+                "questionnaireId;%s;%s;%s;distinctInterrogationIds\n"
+                        .formatted(
+                                Constants.MONGODB_LUNATIC_RAWDATA_COLLECTION_NAME,
+                                Constants.MONGODB_RAW_RESPONSES_COLLECTION_NAME,
+                                Constants.VOLUMETRY_RAW_TOTAL
+                        )
         );
 
-        //Write lines
-        Set<String> oldRawDataQuestionnaires = lunaticJsonRawDataApiPort.findDistinctQuestionnaireIds();
-        Set<String> rawDataQuestionnaires = new HashSet<>(rawResponseApiPort.getDistinctCollectionInstrumentIds());
-        rawDataQuestionnaires.addAll(oldRawDataQuestionnaires);
-        for (String questionnaireId : rawDataQuestionnaires) {
-            long oldRawDataCountResult = lunaticJsonRawDataApiPort.countRawResponsesByQuestionnaireId(questionnaireId);
-            long rawDataCountResult = rawResponseApiPort.countByCollectionInstrumentId(questionnaireId);
-            long total = oldRawDataCountResult + rawDataCountResult;
-
-            String delimiter = ";";
-            String line = questionnaireId + delimiter
-                    + oldRawDataCountResult + delimiter
-                    + rawDataCountResult + delimiter
-                    + total
-                    +"\n";
+        // Merge questionnaire ids from both sources
+        Set<String> lunaticQuestionnaires = lunaticJsonRawDataApiPort.findDistinctQuestionnaireIds();
+        Set<String> rawQuestionnaires = new HashSet<>(rawResponseApiPort.getDistinctCollectionInstrumentIds());
+        rawQuestionnaires.addAll(lunaticQuestionnaires);
+
+        List<String> sortedQuestionnaires = new ArrayList<>(rawQuestionnaires);
+        Collections.sort(sortedQuestionnaires);
+
+        for (String questionnaireId : sortedQuestionnaires) {
+
+            long lunaticCount =
+                    lunaticJsonRawDataApiPort.countRawResponsesByQuestionnaireId(questionnaireId);
+
+            long rawCount =
+                    rawResponseApiPort.countByCollectionInstrumentId(questionnaireId);
+
+            long total = lunaticCount + rawCount;
+
+            long lunaticDistinct =
+                    lunaticJsonRawDataApiPort.countDistinctInterrogationIdsByQuestionnaireId(questionnaireId);
+
+            long rawDistinct =
+                    rawResponseApiPort.countDistinctInterrogationIdsByCollectionInstrumentId(questionnaireId);
+
+            long distinctTotal = lunaticDistinct + rawDistinct;
+
+            String line = questionnaireId + ";"
+                    + lunaticCount + ";"
+                    + rawCount + ";"
+                    + total + ";"
+                    + distinctTotal
+                    + "\n";
 
             Files.writeString(logFilePath, line, StandardOpenOption.APPEND);
+
             rawDataVolumetricsMap.get(Constants.MONGODB_LUNATIC_RAWDATA_COLLECTION_NAME)
-                    .put(questionnaireId, oldRawDataCountResult);
+                    .put(questionnaireId, lunaticCount);
+
             rawDataVolumetricsMap.get(Constants.MONGODB_RAW_RESPONSES_COLLECTION_NAME)
-                    .put(questionnaireId, rawDataCountResult);
+                    .put(questionnaireId, rawCount);
+
             rawDataVolumetricsMap.get(Constants.VOLUMETRY_RAW_TOTAL)
                     .put(questionnaireId, total);
         }
+
         return rawDataVolumetricsMap;
     }
 

src/main/java/fr/insee/genesis/infrastructure/adapter/LunaticJsonRawDataMongoAdapter.java

@@ -136,4 +136,10 @@ public Set<String> findUnprocessedInterrogationIdsByCollectionInstrumentId(Strin
 
         return interrogationIds;
     }
+
+    @Override
+    public long countDistinctInterrogationIdsByQuestionnaireId(String questionnaireId) {
+        Long count = repository.countDistinctInterrogationIdsByQuestionnaireId(questionnaireId);
+        return count != null ? count : 0;
+    }
 }

src/main/java/fr/insee/genesis/infrastructure/adapter/RawResponseMongoAdapter.java

@@ -91,7 +91,13 @@ public long countByCollectionInstrumentId(String collectionInstrumentId) {
     public Set<String> findDistinctCollectionInstrumentIds() {
         return new HashSet<>(repository.findDistinctCollectionInstrumentId());
     }
-
+
+    @Override
+    public long countDistinctInterrogationIdsByCollectionInstrumentId(String collectionInstrumentId) {
+        Long count = repository.countDistinctInterrogationIdsByCollectionInstrumentId(collectionInstrumentId);
+        return count != null ? count : 0;
+    }
+
     @Override
     public Page<RawResponseModel> findByCollectionInstrumentId(String collectionInstrumentId, Pageable pageable) {
         Page<RawResponseDocument> rawDataDocs = repository.findByCollectionInstrumentId(collectionInstrumentId, pageable);

src/main/java/fr/insee/genesis/infrastructure/adapter/SurveyUnitMongoAdapter.java

@@ -255,4 +255,17 @@ public Set<String> findCampaignIdsByQuestionnaireId(String questionnaireId) {
 	public long countByQuestionnaireId(String questionnaireId) {
 		return mongoRepository.countByQuestionnaireId(questionnaireId);
 	}
+
+    @Override
+    public long countDistinctInterrogationIdsByQuestionnaireAndCollectionInstrumentId(String id) {
+        Set<String> distinct = new HashSet<>();
+
+        mongoRepository.findInterrogationIdsByQuestionnaireId(id)
+                .forEach(d -> distinct.add(d.getInterrogationId()));
+
+        mongoRepository.findInterrogationIdsByCollectionInstrumentId(id)
+                .forEach(d -> distinct.add(d.getInterrogationId()));
+
+        return distinct.size();
+    }
 }

src/main/java/fr/insee/genesis/infrastructure/repository/LunaticJsonMongoDBRepository.java

@@ -45,6 +45,13 @@ public interface LunaticJsonMongoDBRepository extends MongoRepository<LunaticJso
     @Query(value = "{ 'interrogationId': ?0}")
     List<LunaticJsonRawDataDocument> findByInterrogationId(String interrogationId);
 
+    @Aggregation(pipeline = {
+            "{ '$match': { 'questionnaireId': ?0 } }",
+            "{ '$group': { '_id': '$interrogationId' } }",
+            "{ '$count': 'count' }"
+    })
+    Long countDistinctInterrogationIdsByQuestionnaireId(String questionnaireId);
+
     Page<LunaticJsonRawDataDocument> findByCampaignIdAndRecordDateBetween(String campagneId, Instant start, Instant  end, Pageable pageable);
     long countByQuestionnaireId(String questionnaireId);
     @Aggregation(pipeline = {

src/main/java/fr/insee/genesis/infrastructure/repository/RawResponseRepository.java

@@ -52,4 +52,12 @@ public interface RawResponseRepository extends MongoRepository<RawResponseDocume
     })
     List<String> findDistinctCollectionInstrumentId();
     Page<RawResponseDocument> findByCollectionInstrumentId(String collectionInstrumentId, Pageable pageable);
+
+    @Aggregation(pipeline = {
+            "{ '$match': { 'collectionInstrumentId': ?0 } }",
+            "{ '$group': { '_id': '$interrogationId' } }",
+            "{ '$count': 'count' }"
+    })
+    Long countDistinctInterrogationIdsByCollectionInstrumentId(String collectionInstrumentId);
+
 }

src/main/java/fr/insee/genesis/infrastructure/repository/SurveyUnitMongoDBRepository.java

@@ -128,4 +128,19 @@ List<SurveyUnitDocument> findInterrogationIdsQuestionnaireIdAndRecordDateBetween
 
 	@Query(value = "{ 'questionnaireId' : ?0 }", fields = "{ _id : 0, 'campaignId' : 1 }")
 	Set<String> findCampaignIdsByQuestionnaireId(String questionnaireId);
+
+
+    @Aggregation(pipeline = {
+            "{ '$match': { 'questionnaireId': ?0 } }",
+            "{ '$group': { '_id': '$interrogationId' } }",
+            "{ '$count': 'count' }"
+    })
+    Long countDistinctInterrogationIdsByQuestionnaireId(String questionnaireId);
+
+    @Aggregation(pipeline = {
+            "{ '$match': { 'collectionInstrumentId': ?0 } }",
+            "{ '$group': { '_id': '$interrogationId' } }",
+            "{ '$count': 'count' }"
+    })
+    Long countDistinctInterrogationIdsByCollectionInstrumentId(String collectionInstrumentId);
 }

src/test/java/cucumber/functional_tests/RawDataDefinitions.java

@@ -149,6 +149,11 @@ public Page<RawResponseModel> findRawResponseDataByCampaignIdAndDate(String camp
             return null;
         }
 
+        @Override
+        public long countDistinctInterrogationIdsByCollectionInstrumentId(String collectionInstrumentId) {
+            return 0;
+        }
+
         @Override
         public long countByCollectionInstrumentId(String collectionInstrumentId) {
             return 0;

src/test/java/fr/insee/genesis/controller/rest/responses/RawResponseControllerTest.java

@@ -123,6 +123,11 @@ public Page<RawResponseModel> findRawResponseDataByCampaignIdAndDate(String camp
             return rawResponseDataPersistanceStub.findByCampaignIdAndDate(campaignId, startDate, endDate, pageable);
         }
 
+        @Override
+        public long countDistinctInterrogationIdsByCollectionInstrumentId(String collectionInstrumentId) {
+            return 0;
+        }
+
         @Override
         public long countByCollectionInstrumentId(String collectionInstrumentId) {
             return 0;

src/test/java/fr/insee/genesis/stubs/LunaticJsonMongoDBRepositoryStub.java

@@ -87,6 +87,11 @@ public List<LunaticJsonRawDataDocument> findByInterrogationId(String interrogati
         ;
     }
 
+    @Override
+    public Long countDistinctInterrogationIdsByQuestionnaireId(String questionnaireId) {
+        return 0L;
+    }
+
     @Override
     public Page<LunaticJsonRawDataDocument> findByCampaignIdAndRecordDateBetween(String campagneId, Instant  start, Instant end, Pageable pageable){
         return Page.empty(pageable);

src/test/java/fr/insee/genesis/stubs/LunaticJsonRawDataPersistanceStub.java

@@ -234,4 +234,9 @@ public Set<String> findUnprocessedInterrogationIdsByCollectionInstrumentId(Strin
        unprocessedDocuments.forEach(doc -> interrogationIds.add(doc.interrogationId()));
        return interrogationIds;
     }
+
+    @Override
+    public long countDistinctInterrogationIdsByQuestionnaireId(String questionnaireId) {
+        return 0;
+    }
 }

src/test/java/fr/insee/genesis/stubs/RawResponseDataPersistanceStub.java

@@ -97,4 +97,9 @@ public long countByCollectionInstrumentId(String collectionInstrumentId) {
     public Set<String> findDistinctCollectionInstrumentIds() {
         return new HashSet<>();
     }
+
+    @Override
+    public long countDistinctInterrogationIdsByCollectionInstrumentId(String collectionInstrumentId) {
+        return 0;
+    }
 }

src/test/java/fr/insee/genesis/stubs/SurveyUnitPersistencePortStub.java

@@ -238,4 +238,9 @@ public Set<String> findCampaignIdsByQuestionnaireId(String questionnaireId) {
     public long countByQuestionnaireId(String questionnaireId) {
         return 0;
     }
+
+    @Override
+    public long countDistinctInterrogationIdsByQuestionnaireAndCollectionInstrumentId(String id) {
+        return 0;
+    }
 }