From 8053aa51a77de5932e2bdb2ad80b080bc3278df0 Mon Sep 17 00:00:00 2001 From: Leonid Andreev Date: Thu, 5 Jun 2025 15:09:35 -0400 Subject: [PATCH 01/25] early mods of the new export data provider framework #11405 --- .../gdcc/spi/export/ExportDataProvider.java | 27 +++++++++++++++++- .../export/InternalExportDataProvider.java | 28 +++++++++++++++---- .../iq/dataverse/util/json/JsonPrinter.java | 12 +++++++- 3 files changed, 60 insertions(+), 7 deletions(-) diff --git a/modules/dataverse-spi/src/main/java/io/gdcc/spi/export/ExportDataProvider.java b/modules/dataverse-spi/src/main/java/io/gdcc/spi/export/ExportDataProvider.java index d039ac39e8f..79e78b3e2f2 100644 --- a/modules/dataverse-spi/src/main/java/io/gdcc/spi/export/ExportDataProvider.java +++ b/modules/dataverse-spi/src/main/java/io/gdcc/spi/export/ExportDataProvider.java @@ -23,6 +23,22 @@ public interface ExportDataProvider { * in the dataset. */ JsonObject getDatasetJson(); + + /** + * @return - dataset metadata in the standard Dataverse JSON format used in the + * API and available as the JSON metadata export via the user interface. + * Same as above, but does not include any information about the files + * in the dataset. In a situation where we need to generate a format + * like DC, that has no use for this information, it makes sense to + * skip retrieving and formatting it, since there can be quite a few + * files in a dataset. + * @apiNote - there is no JSON schema defining this output, but the format is + * well documented in the Dataverse online guides. This, and the + * OAI_ORE export are the only two that provide 'complete' + * dataset-level metadata along with basic file metadata for each file + * in the dataset. + */ + JsonObject getDatasetOnlyJson(); /** * @@ -39,7 +55,7 @@ public interface ExportDataProvider { * Dataverse is capable of extracting DDI-centric metadata from tabular * datafiles. This detailed metadata, which is only available for successfully * "ingested" tabular files, is not included in the output of any other methods - * in this interface. + * in this interface. * * @return - a JSONArray with one entry per ingested tabular dataset file. * @apiNote - there is no JSON schema available for this output and the format @@ -50,6 +66,15 @@ public interface ExportDataProvider { */ JsonArray getDatasetFileDetails(); + /** + * Same as above, but gives an option for retrieving this stuff in batches, + * for datasets with massive numbers of files. + * @param offset + * @param length + * @return + */ + JsonArray getDatasetFileDetails(Long offset, Integer length); + /** * * @return - the subset of metadata conforming to the schema.org standard as diff --git a/src/main/java/edu/harvard/iq/dataverse/export/InternalExportDataProvider.java b/src/main/java/edu/harvard/iq/dataverse/export/InternalExportDataProvider.java index f0d77eb8b52..ab094a9cebd 100644 --- a/src/main/java/edu/harvard/iq/dataverse/export/InternalExportDataProvider.java +++ b/src/main/java/edu/harvard/iq/dataverse/export/InternalExportDataProvider.java @@ -26,8 +26,10 @@ public class InternalExportDataProvider implements ExportDataProvider { private DatasetVersion dv; private JsonObject jsonRepresentation = null; + private JsonObject jsonRepresentationNoFiles = null; private JsonObject schemaDotOrgRepresentation = null; private JsonObject oreRepresentation = null; + private JsonArray fileAndDataDetails = null; private InputStream is = null; InternalExportDataProvider(DatasetVersion dv) { @@ -47,6 +49,19 @@ public JsonObject getDatasetJson() { } return jsonRepresentation; } + + @Override + public JsonObject getDatasetOnlyJson() { + // If we already have the "full" Json representation (with files) + // generated, should we return it (potentially moving MUCH more json + // than the client needs, or spend extra cycles generating the short + // form from scratch? - I'm choosing to go with latter. + if (jsonRepresentationNoFiles == null) { + final JsonObjectBuilder datasetAsJsonBuilder = JsonPrinter.jsonAsDatasetDto(dv, false); + jsonRepresentationNoFiles = datasetAsJsonBuilder.build(); + } + return jsonRepresentationNoFiles; + } @Override public JsonObject getDatasetSchemaDotOrg() { @@ -73,12 +88,15 @@ public String getDataCiteXml() { @Override public JsonArray getDatasetFileDetails() { - JsonArrayBuilder jab = Json.createArrayBuilder(); - for (FileMetadata fileMetadata : dv.getFileMetadatas()) { - DataFile dataFile = fileMetadata.getDataFile(); - jab.add(JsonPrinter.json(dataFile, fileMetadata, true)); + if (fileAndDataDetails == null) { + JsonArrayBuilder jab = Json.createArrayBuilder(); + for (FileMetadata fileMetadata : dv.getFileMetadatas()) { + DataFile dataFile = fileMetadata.getDataFile(); + jab.add(JsonPrinter.json(dataFile, fileMetadata, true)); + } + fileAndDataDetails = jab.build(); } - return jab.build(); + return fileAndDataDetails; } @Override diff --git a/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java b/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java index 0b5803d75d1..6c090cfb330 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java @@ -547,8 +547,18 @@ public static JsonObjectBuilder jsonWithCitation(DatasetVersion dsv, boolean inc * should the method be renamed? */ public static JsonObjectBuilder jsonAsDatasetDto(DatasetVersion dsv) { + return jsonAsDatasetDto(dsv, true); + } + + /** + * Same as above, but gives an option to skip the file-level info + * @param dsv + * @param includeFiles + * @return + */ + public static JsonObjectBuilder jsonAsDatasetDto(DatasetVersion dsv, boolean includeFiles) { JsonObjectBuilder datasetDtoAsJson = JsonPrinter.json(dsv.getDataset()); - datasetDtoAsJson.add("datasetVersion", jsonWithCitation(dsv, true)); + datasetDtoAsJson.add("datasetVersion", jsonWithCitation(dsv, includeFiles)); return datasetDtoAsJson; } From ebbde8ba9fac1ddfea6e81aa9d2add8997aba33c Mon Sep 17 00:00:00 2001 From: Leonid Andreev Date: Thu, 5 Jun 2025 15:35:50 -0400 Subject: [PATCH 02/25] ... and had to change it back temporarily --- .../src/main/java/io/gdcc/spi/export/ExportDataProvider.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/dataverse-spi/src/main/java/io/gdcc/spi/export/ExportDataProvider.java b/modules/dataverse-spi/src/main/java/io/gdcc/spi/export/ExportDataProvider.java index 79e78b3e2f2..06abe6e7ece 100644 --- a/modules/dataverse-spi/src/main/java/io/gdcc/spi/export/ExportDataProvider.java +++ b/modules/dataverse-spi/src/main/java/io/gdcc/spi/export/ExportDataProvider.java @@ -38,7 +38,7 @@ public interface ExportDataProvider { * dataset-level metadata along with basic file metadata for each file * in the dataset. */ - JsonObject getDatasetOnlyJson(); + //JsonObject getDatasetOnlyJson(); /** * @@ -73,7 +73,7 @@ public interface ExportDataProvider { * @param length * @return */ - JsonArray getDatasetFileDetails(Long offset, Integer length); + //JsonArray getDatasetFileDetails(Long offset, Integer length); /** * From dcae3fcc87e1cba7349d94ca65968d22f95661f6 Mon Sep 17 00:00:00 2001 From: Leonid Andreev Date: Thu, 12 Jun 2025 14:52:44 -0400 Subject: [PATCH 03/25] re-checking in the updated data provider interface. #11405 --- .../src/main/java/io/gdcc/spi/export/ExportDataProvider.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/dataverse-spi/src/main/java/io/gdcc/spi/export/ExportDataProvider.java b/modules/dataverse-spi/src/main/java/io/gdcc/spi/export/ExportDataProvider.java index 06abe6e7ece..79e78b3e2f2 100644 --- a/modules/dataverse-spi/src/main/java/io/gdcc/spi/export/ExportDataProvider.java +++ b/modules/dataverse-spi/src/main/java/io/gdcc/spi/export/ExportDataProvider.java @@ -38,7 +38,7 @@ public interface ExportDataProvider { * dataset-level metadata along with basic file metadata for each file * in the dataset. */ - //JsonObject getDatasetOnlyJson(); + JsonObject getDatasetOnlyJson(); /** * @@ -73,7 +73,7 @@ public interface ExportDataProvider { * @param length * @return */ - //JsonArray getDatasetFileDetails(Long offset, Integer length); + JsonArray getDatasetFileDetails(Long offset, Integer length); /** * From be3603ca08723e7be601978386f5dbb06a48ad14 Mon Sep 17 00:00:00 2001 From: Leonid Andreev Date: Tue, 26 Aug 2025 14:44:34 -0400 Subject: [PATCH 04/25] intermediate state of the export data provider implementation using the extended interface. --- pom.xml | 2 +- .../export/InternalExportDataProvider.java | 105 +++++++++++++++++- 2 files changed, 102 insertions(+), 5 deletions(-) diff --git a/pom.xml b/pom.xml index c8971fb6587..38f38a431a8 100644 --- a/pom.xml +++ b/pom.xml @@ -671,7 +671,7 @@ io.gdcc dataverse-spi - 2.0.0 + 2.1.0 javax.cache diff --git a/src/main/java/edu/harvard/iq/dataverse/export/InternalExportDataProvider.java b/src/main/java/edu/harvard/iq/dataverse/export/InternalExportDataProvider.java index ab094a9cebd..19078707df4 100644 --- a/src/main/java/edu/harvard/iq/dataverse/export/InternalExportDataProvider.java +++ b/src/main/java/edu/harvard/iq/dataverse/export/InternalExportDataProvider.java @@ -11,12 +11,20 @@ import edu.harvard.iq.dataverse.DataCitation; import edu.harvard.iq.dataverse.DataFile; import edu.harvard.iq.dataverse.DatasetVersion; +import edu.harvard.iq.dataverse.DatasetVersionFilesServiceBean; import edu.harvard.iq.dataverse.FileMetadata; +import edu.harvard.iq.dataverse.FileSearchCriteria; import edu.harvard.iq.dataverse.pidproviders.doi.datacite.DOIDataCiteRegisterService; +import edu.harvard.iq.dataverse.util.BundleUtil; import io.gdcc.spi.export.ExportDataProvider; import edu.harvard.iq.dataverse.util.bagit.OREMap; import edu.harvard.iq.dataverse.util.json.JsonPrinter; import edu.harvard.iq.dataverse.util.json.JsonUtil; +import static edu.harvard.iq.dataverse.util.FileUtil.MIME_TYPE_INGESTED_FILE; +import io.gdcc.spi.export.ExportException; +import java.util.List; +import java.util.logging.Level; +import java.util.logging.Logger; /** * Provides all data necessary to create an export @@ -24,13 +32,14 @@ */ public class InternalExportDataProvider implements ExportDataProvider { - private DatasetVersion dv; + private final DatasetVersion dv; private JsonObject jsonRepresentation = null; private JsonObject jsonRepresentationNoFiles = null; private JsonObject schemaDotOrgRepresentation = null; private JsonObject oreRepresentation = null; - private JsonArray fileAndDataDetails = null; + private JsonArray fileAndDataDetails = null; private InputStream is = null; + private DatasetVersionFilesServiceBean datasetVersionFilesService; InternalExportDataProvider(DatasetVersion dv) { this.dv = dv; @@ -40,9 +49,26 @@ public class InternalExportDataProvider implements ExportDataProvider { this.dv = dv; this.is=is; } + + InternalExportDataProvider(DatasetVersion dv, DatasetVersionFilesServiceBean datasetVersionFilesService) { + this.dv = dv; + this.datasetVersionFilesService = datasetVersionFilesService; + } @Override - public JsonObject getDatasetJson() { + public JsonObject getDatasetJson(ExportDataOption... options) { + if (isOnlyDatasetLevelMetadataRequested(options)) { + // If we already have the "full" Json representation (with files) + // generated, should we return it (potentially moving MUCH more json + // than the client needs, or spend extra cycles generating the short + // form from scratch? - I'm choosing to go with latter. + if (jsonRepresentationNoFiles == null) { + final JsonObjectBuilder datasetAsJsonBuilder = JsonPrinter.jsonAsDatasetDto(dv, false); + jsonRepresentationNoFiles = datasetAsJsonBuilder.build(); + } + return jsonRepresentationNoFiles; + } + if (jsonRepresentation == null) { final JsonObjectBuilder datasetAsJsonBuilder = JsonPrinter.jsonAsDatasetDto(dv); jsonRepresentation = datasetAsJsonBuilder.build(); @@ -50,7 +76,7 @@ public JsonObject getDatasetJson() { return jsonRepresentation; } - @Override + /*@Override*/ public JsonObject getDatasetOnlyJson() { // If we already have the "full" Json representation (with files) // generated, should we return it (potentially moving MUCH more json @@ -99,12 +125,83 @@ public JsonArray getDatasetFileDetails() { return fileAndDataDetails; } + @Override + public JsonArray getTabularDataDetails(Integer offset, Integer length, ExportDataOption... options) throws ExportException { + JsonArrayBuilder jab = Json.createArrayBuilder(); + + List fileMetadatas = null; + Optional datasetVersionFilesServiceOptional = getDatasetVersionFilesService(); + + if (datasetVersionFilesServiceOptional.isPresent()) { + + FileSearchCriteria fileSearchCriteria; + try { + fileSearchCriteria = new FileSearchCriteria( + MIME_TYPE_INGESTED_FILE, + isOnlyPublicMetadataRequested(options) ? FileSearchCriteria.FileAccessStatus.Public : null, // this is optional + null, + null, + null + ); + } catch (IllegalArgumentException e) { + throw new ExportException("Failed to build a retrieval query for tabular file metadata"); + } + + + fileMetadatas = datasetVersionFilesServiceOptional.get().getFileMetadatas(dv, length, offset, fileSearchCriteria, DatasetVersionFilesServiceBean.FileOrderCriteria.NameAZ); + + } else { + throw new ExportException("EJB DatasetVersionFilesService is not available"); + } + + for (FileMetadata fileMetadata : dv.getFileMetadatas()) { + DataFile dataFile = fileMetadata.getDataFile(); + jab.add(JsonPrinter.json(dataFile, fileMetadata, true)); + } + return jab.build(); + } + @Override public Optional getPrerequisiteInputStream() { return Optional.ofNullable(is); } + + public Optional getDatasetVersionFilesService() { + return Optional.ofNullable(datasetVersionFilesService); + } public void setPrerequisiteInputStream(InputStream prereqStream) { this.is=prereqStream; } + + public void setDatasetVersionFilesService(DatasetVersionFilesServiceBean datasetVersionFilesService) { + this.datasetVersionFilesService = datasetVersionFilesService; + } + + private boolean isOnlyDatasetLevelMetadataRequested(ExportDataOption... options) { + for (ExportDataOption option : options) { + + if (option == ExportDataOption.DatasetOnly) { + return true; + } + } + + // By default, we pack both the Dataset, and the File-level metadata in that Json + return false; + } + + private boolean isOnlyPublicMetadataRequested(ExportDataOption... options) throws ExportException { + + for (ExportDataOption option : options) { + + if (option == ExportDataOption.PublicFilesOnly) { + return true; + } else { + throw new ExportException("Unsupported data export option"); + } + } + + // By default, we return the metadata for all files - embargoed, restricted, etc.: + return false; + } } From 90757da5e762436985763913120589e7ead161bf Mon Sep 17 00:00:00 2001 From: Leonid Andreev Date: Tue, 26 Aug 2025 14:47:31 -0400 Subject: [PATCH 05/25] removed some experimental code. #11405 --- .../export/InternalExportDataProvider.java | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/export/InternalExportDataProvider.java b/src/main/java/edu/harvard/iq/dataverse/export/InternalExportDataProvider.java index 19078707df4..2e691c51388 100644 --- a/src/main/java/edu/harvard/iq/dataverse/export/InternalExportDataProvider.java +++ b/src/main/java/edu/harvard/iq/dataverse/export/InternalExportDataProvider.java @@ -76,19 +76,6 @@ public JsonObject getDatasetJson(ExportDataOption... options) { return jsonRepresentation; } - /*@Override*/ - public JsonObject getDatasetOnlyJson() { - // If we already have the "full" Json representation (with files) - // generated, should we return it (potentially moving MUCH more json - // than the client needs, or spend extra cycles generating the short - // form from scratch? - I'm choosing to go with latter. - if (jsonRepresentationNoFiles == null) { - final JsonObjectBuilder datasetAsJsonBuilder = JsonPrinter.jsonAsDatasetDto(dv, false); - jsonRepresentationNoFiles = datasetAsJsonBuilder.build(); - } - return jsonRepresentationNoFiles; - } - @Override public JsonObject getDatasetSchemaDotOrg() { if (schemaDotOrgRepresentation == null) { From fbd49985d502d37b20f8daf8a5fcbd189b82a232 Mon Sep 17 00:00:00 2001 From: Leonid Andreev Date: Wed, 27 Aug 2025 17:11:06 -0400 Subject: [PATCH 06/25] work in progress --- .../export/InternalExportDataProvider.java | 23 +++++++++---------- 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/export/InternalExportDataProvider.java b/src/main/java/edu/harvard/iq/dataverse/export/InternalExportDataProvider.java index 2e691c51388..9c4e84f28e3 100644 --- a/src/main/java/edu/harvard/iq/dataverse/export/InternalExportDataProvider.java +++ b/src/main/java/edu/harvard/iq/dataverse/export/InternalExportDataProvider.java @@ -15,16 +15,14 @@ import edu.harvard.iq.dataverse.FileMetadata; import edu.harvard.iq.dataverse.FileSearchCriteria; import edu.harvard.iq.dataverse.pidproviders.doi.datacite.DOIDataCiteRegisterService; -import edu.harvard.iq.dataverse.util.BundleUtil; import io.gdcc.spi.export.ExportDataProvider; import edu.harvard.iq.dataverse.util.bagit.OREMap; import edu.harvard.iq.dataverse.util.json.JsonPrinter; import edu.harvard.iq.dataverse.util.json.JsonUtil; import static edu.harvard.iq.dataverse.util.FileUtil.MIME_TYPE_INGESTED_FILE; import io.gdcc.spi.export.ExportException; +import io.gdcc.spi.export.ExportDataOption; import java.util.List; -import java.util.logging.Level; -import java.util.logging.Logger; /** * Provides all data necessary to create an export @@ -77,7 +75,7 @@ public JsonObject getDatasetJson(ExportDataOption... options) { } @Override - public JsonObject getDatasetSchemaDotOrg() { + public JsonObject getDatasetSchemaDotOrg(ExportDataOption... options) { if (schemaDotOrgRepresentation == null) { String jsonLdAsString = dv.getJsonLd(); schemaDotOrgRepresentation = JsonUtil.getJsonObject(jsonLdAsString); @@ -86,7 +84,7 @@ public JsonObject getDatasetSchemaDotOrg() { } @Override - public JsonObject getDatasetORE() { + public JsonObject getDatasetORE(ExportDataOption... options) { if (oreRepresentation == null) { oreRepresentation = new OREMap(dv).getOREMap(); } @@ -94,13 +92,13 @@ public JsonObject getDatasetORE() { } @Override - public String getDataCiteXml() { + public String getDataCiteXml(ExportDataOption... options) { return DOIDataCiteRegisterService.getMetadataFromDvObject( dv.getDataset().getGlobalId().asString(), new DataCitation(dv).getDataCiteMetadata(), dv.getDataset()); } @Override - public JsonArray getDatasetFileDetails() { + public JsonArray getDatasetFileDetails(ExportDataOption... options) { if (fileAndDataDetails == null) { JsonArrayBuilder jab = Json.createArrayBuilder(); for (FileMetadata fileMetadata : dv.getFileMetadatas()) { @@ -125,7 +123,7 @@ public JsonArray getTabularDataDetails(Integer offset, Integer length, ExportDat try { fileSearchCriteria = new FileSearchCriteria( MIME_TYPE_INGESTED_FILE, - isOnlyPublicMetadataRequested(options) ? FileSearchCriteria.FileAccessStatus.Public : null, // this is optional + isOnlyPublicMetadataRequested(options) ? FileSearchCriteria.FileAccessStatus.Public : null, null, null, null @@ -141,7 +139,7 @@ public JsonArray getTabularDataDetails(Integer offset, Integer length, ExportDat throw new ExportException("EJB DatasetVersionFilesService is not available"); } - for (FileMetadata fileMetadata : dv.getFileMetadatas()) { + for (FileMetadata fileMetadata : fileMetadatas) { DataFile dataFile = fileMetadata.getDataFile(); jab.add(JsonPrinter.json(dataFile, fileMetadata, true)); } @@ -149,7 +147,7 @@ public JsonArray getTabularDataDetails(Integer offset, Integer length, ExportDat } @Override - public Optional getPrerequisiteInputStream() { + public Optional getPrerequisiteInputStream(ExportDataOption... options) { return Optional.ofNullable(is); } @@ -168,7 +166,7 @@ public void setDatasetVersionFilesService(DatasetVersionFilesServiceBean dataset private boolean isOnlyDatasetLevelMetadataRequested(ExportDataOption... options) { for (ExportDataOption option : options) { - if (option == ExportDataOption.DatasetOnly) { + if (option.isDatasetMetadataOnly()) { return true; } } @@ -181,7 +179,7 @@ private boolean isOnlyPublicMetadataRequested(ExportDataOption... options) throw for (ExportDataOption option : options) { - if (option == ExportDataOption.PublicFilesOnly) { + if (option.isPublicFilesOnly()) { return true; } else { throw new ExportException("Unsupported data export option"); @@ -191,4 +189,5 @@ private boolean isOnlyPublicMetadataRequested(ExportDataOption... options) throw // By default, we return the metadata for all files - embargoed, restricted, etc.: return false; } + } From 3cdb98f669482b77a3c61112dce40b5ee7383c5d Mon Sep 17 00:00:00 2001 From: Leonid Andreev Date: Tue, 2 Sep 2025 09:59:56 -0400 Subject: [PATCH 07/25] work in progress (this is working for me, since I have dataverse-spi 2.1.0 built locally, but isn't going to work in Jenkins/for anyone else just yet) #11405 --- .../edu/harvard/iq/dataverse/api/Files.java | 2 +- .../iq/dataverse/api/dto/DataFileDTO.java | 7 + .../iq/dataverse/export/DCTermsExporter.java | 3 +- .../iq/dataverse/export/DDIExporter.java | 2 +- .../dataverse/export/DublinCoreExporter.java | 3 +- .../iq/dataverse/export/ExportService.java | 4 +- .../export/InternalExportDataProvider.java | 163 +++++++++++------- .../dataverse/export/ddi/DdiExportUtil.java | 146 +++++++++++----- .../iq/dataverse/util/json/JsonPrinter.java | 109 ++++++------ 9 files changed, 268 insertions(+), 171 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Files.java b/src/main/java/edu/harvard/iq/dataverse/api/Files.java index 5834e7e0008..811884ad7dd 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Files.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Files.java @@ -923,7 +923,7 @@ public Response getFileDataTables(@Context ContainerRequestContext crc, @PathPar if (!dataFile.isTabularData()) { return badRequest(BundleUtil.getStringFromBundle("files.api.only.tabular.supported")); } - return ok(jsonDT(dataFile.getDataTables())); + return ok(jsonDT(dataFile.getDataTables(), true)); } @POST diff --git a/src/main/java/edu/harvard/iq/dataverse/api/dto/DataFileDTO.java b/src/main/java/edu/harvard/iq/dataverse/api/dto/DataFileDTO.java index 318bad4f3a3..f4537b0b7ad 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/dto/DataFileDTO.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/dto/DataFileDTO.java @@ -20,6 +20,7 @@ public class DataFileDTO { private String md5; private String description; private String pidURL; + private List tabularTags; public String getPidURL() { return pidURL; @@ -119,5 +120,11 @@ public void setDescription(String description) { this.description = description; } + public List getTabularTags() { + return tabularTags; + } + public void setTabularTags(List tabularTags) { + this.tabularTags = tabularTags; + } } diff --git a/src/main/java/edu/harvard/iq/dataverse/export/DCTermsExporter.java b/src/main/java/edu/harvard/iq/dataverse/export/DCTermsExporter.java index f82c0d9ad3d..ad5010fdf50 100644 --- a/src/main/java/edu/harvard/iq/dataverse/export/DCTermsExporter.java +++ b/src/main/java/edu/harvard/iq/dataverse/export/DCTermsExporter.java @@ -8,6 +8,7 @@ import io.gdcc.spi.export.Exporter; import io.gdcc.spi.export.XMLExporter; import edu.harvard.iq.dataverse.util.BundleUtil; +import io.gdcc.spi.export.ExportDataContext; import java.io.OutputStream; import java.util.Locale; import java.util.Optional; @@ -38,7 +39,7 @@ public String getDisplayName(Locale locale) { @Override public void exportDataset(ExportDataProvider dataProvider, OutputStream outputStream) throws ExportException { try { - DublinCoreExportUtil.datasetJson2dublincore(dataProvider.getDatasetJson(), outputStream, DublinCoreExportUtil.DC_FLAVOR_DCTERMS); + DublinCoreExportUtil.datasetJson2dublincore(dataProvider.getDatasetJson(ExportDataContext.context().withDatasetMetadataOnly()), outputStream, DublinCoreExportUtil.DC_FLAVOR_DCTERMS); } catch (XMLStreamException xse) { throw new ExportException("Caught XMLStreamException performing DCTERMS export", xse); } diff --git a/src/main/java/edu/harvard/iq/dataverse/export/DDIExporter.java b/src/main/java/edu/harvard/iq/dataverse/export/DDIExporter.java index 0130c18b22b..e03363a155e 100644 --- a/src/main/java/edu/harvard/iq/dataverse/export/DDIExporter.java +++ b/src/main/java/edu/harvard/iq/dataverse/export/DDIExporter.java @@ -48,7 +48,7 @@ public String getDisplayName(Locale locale) { @Override public void exportDataset(ExportDataProvider dataProvider, OutputStream outputStream) throws ExportException { try { - DdiExportUtil.datasetJson2ddi(dataProvider.getDatasetJson(), dataProvider.getDatasetFileDetails(), + DdiExportUtil.datasetJson2ddi(dataProvider.getDatasetJson(), dataProvider, outputStream); } catch (XMLStreamException xse) { throw new ExportException("Caught XMLStreamException performing DDI export", xse); diff --git a/src/main/java/edu/harvard/iq/dataverse/export/DublinCoreExporter.java b/src/main/java/edu/harvard/iq/dataverse/export/DublinCoreExporter.java index 0fa32dd4bfa..db3c28deb78 100644 --- a/src/main/java/edu/harvard/iq/dataverse/export/DublinCoreExporter.java +++ b/src/main/java/edu/harvard/iq/dataverse/export/DublinCoreExporter.java @@ -5,6 +5,7 @@ import edu.harvard.iq.dataverse.export.dublincore.DublinCoreExportUtil; import io.gdcc.spi.export.ExportDataProvider; import io.gdcc.spi.export.ExportException; +import io.gdcc.spi.export.ExportDataContext; import io.gdcc.spi.export.Exporter; import io.gdcc.spi.export.XMLExporter; import edu.harvard.iq.dataverse.util.BundleUtil; @@ -38,7 +39,7 @@ public String getDisplayName(Locale locale) { @Override public void exportDataset(ExportDataProvider dataProvider, OutputStream outputStream) throws ExportException { try { - DublinCoreExportUtil.datasetJson2dublincore(dataProvider.getDatasetJson(), outputStream, + DublinCoreExportUtil.datasetJson2dublincore(dataProvider.getDatasetJson(ExportDataContext.context().withDatasetMetadataOnly()), outputStream, DublinCoreExportUtil.DC_FLAVOR_OAI); } catch (XMLStreamException xse) { throw new ExportException("Caught XMLStreamException performing DC export", xse); diff --git a/src/main/java/edu/harvard/iq/dataverse/export/ExportService.java b/src/main/java/edu/harvard/iq/dataverse/export/ExportService.java index b98f88e386f..b33305c0f80 100644 --- a/src/main/java/edu/harvard/iq/dataverse/export/ExportService.java +++ b/src/main/java/edu/harvard/iq/dataverse/export/ExportService.java @@ -64,7 +64,7 @@ public class ExportService { private Map exporterMap = new HashMap<>(); private static final Logger logger = Logger.getLogger(ExportService.class.getCanonicalName()); - + private ExportService() { /* * Step 1 - find the EXPORTERS dir and add all jar files there to a class loader @@ -378,7 +378,7 @@ public void exportFormat(Dataset dataset, String formatName) throws ExportExcept } } - + public Exporter getExporter(String formatName) throws ExportException { Exporter e = exporterMap.get(formatName); if (e != null) { diff --git a/src/main/java/edu/harvard/iq/dataverse/export/InternalExportDataProvider.java b/src/main/java/edu/harvard/iq/dataverse/export/InternalExportDataProvider.java index 9c4e84f28e3..f4962cefc0b 100644 --- a/src/main/java/edu/harvard/iq/dataverse/export/InternalExportDataProvider.java +++ b/src/main/java/edu/harvard/iq/dataverse/export/InternalExportDataProvider.java @@ -3,6 +3,7 @@ import java.io.InputStream; import java.util.Optional; +import jakarta.enterprise.inject.spi.CDI; import jakarta.json.Json; import jakarta.json.JsonArray; import jakarta.json.JsonArrayBuilder; @@ -21,7 +22,7 @@ import edu.harvard.iq.dataverse.util.json.JsonUtil; import static edu.harvard.iq.dataverse.util.FileUtil.MIME_TYPE_INGESTED_FILE; import io.gdcc.spi.export.ExportException; -import io.gdcc.spi.export.ExportDataOption; +import io.gdcc.spi.export.ExportDataContext; import java.util.List; /** @@ -37,7 +38,6 @@ public class InternalExportDataProvider implements ExportDataProvider { private JsonObject oreRepresentation = null; private JsonArray fileAndDataDetails = null; private InputStream is = null; - private DatasetVersionFilesServiceBean datasetVersionFilesService; InternalExportDataProvider(DatasetVersion dv) { this.dv = dv; @@ -47,35 +47,30 @@ public class InternalExportDataProvider implements ExportDataProvider { this.dv = dv; this.is=is; } - - InternalExportDataProvider(DatasetVersion dv, DatasetVersionFilesServiceBean datasetVersionFilesService) { - this.dv = dv; - this.datasetVersionFilesService = datasetVersionFilesService; - } @Override - public JsonObject getDatasetJson(ExportDataOption... options) { - if (isOnlyDatasetLevelMetadataRequested(options)) { + public JsonObject getDatasetJson(ExportDataContext... context) { + if (isOnlyDatasetLevelMetadataRequested(context)) { // If we already have the "full" Json representation (with files) // generated, should we return it (potentially moving MUCH more json // than the client needs, or spend extra cycles generating the short // form from scratch? - I'm choosing to go with latter. if (jsonRepresentationNoFiles == null) { - final JsonObjectBuilder datasetAsJsonBuilder = JsonPrinter.jsonAsDatasetDto(dv, false); + final JsonObjectBuilder datasetAsJsonBuilder = JsonPrinter.datasetAsJsonForDTO(dv, false); jsonRepresentationNoFiles = datasetAsJsonBuilder.build(); } return jsonRepresentationNoFiles; } if (jsonRepresentation == null) { - final JsonObjectBuilder datasetAsJsonBuilder = JsonPrinter.jsonAsDatasetDto(dv); + final JsonObjectBuilder datasetAsJsonBuilder = JsonPrinter.datasetAsJsonForDTO(dv); jsonRepresentation = datasetAsJsonBuilder.build(); } return jsonRepresentation; } @Override - public JsonObject getDatasetSchemaDotOrg(ExportDataOption... options) { + public JsonObject getDatasetSchemaDotOrg(ExportDataContext... context) { if (schemaDotOrgRepresentation == null) { String jsonLdAsString = dv.getJsonLd(); schemaDotOrgRepresentation = JsonUtil.getJsonObject(jsonLdAsString); @@ -84,7 +79,7 @@ public JsonObject getDatasetSchemaDotOrg(ExportDataOption... options) { } @Override - public JsonObject getDatasetORE(ExportDataOption... options) { + public JsonObject getDatasetORE(ExportDataContext... context) { if (oreRepresentation == null) { oreRepresentation = new OREMap(dv).getOREMap(); } @@ -92,18 +87,18 @@ public JsonObject getDatasetORE(ExportDataOption... options) { } @Override - public String getDataCiteXml(ExportDataOption... options) { + public String getDataCiteXml(ExportDataContext... context) { return DOIDataCiteRegisterService.getMetadataFromDvObject( dv.getDataset().getGlobalId().asString(), new DataCitation(dv).getDataCiteMetadata(), dv.getDataset()); } @Override - public JsonArray getDatasetFileDetails(ExportDataOption... options) { + public JsonArray getDatasetFileDetails(ExportDataContext... context) { if (fileAndDataDetails == null) { JsonArrayBuilder jab = Json.createArrayBuilder(); for (FileMetadata fileMetadata : dv.getFileMetadatas()) { DataFile dataFile = fileMetadata.getDataFile(); - jab.add(JsonPrinter.json(dataFile, fileMetadata, true)); + jab.add(JsonPrinter.json(dataFile, fileMetadata, true, false, true)); } fileAndDataDetails = jab.build(); } @@ -111,34 +106,47 @@ public JsonArray getDatasetFileDetails(ExportDataOption... options) { } @Override - public JsonArray getTabularDataDetails(Integer offset, Integer length, ExportDataOption... options) throws ExportException { + /** + * This new (as of dataverse-spi 2.1.0) method will attempt to retrieve + * the requested tabular metadata more efficiently, by calling the + * DatasetVersionFilesServiceBean method directly. Which, among other things, + * allows to retrieve this information in batches. If for whatever reason + * that fails - if, for example, the EJB is not available in this context, + * we will throw an ExportException, giving the exporter a chance to try and + * retrieve this information using the traditional all-at-once method via + * getDatasetFileDetails(); + * + */ + public JsonArray getTabularDataDetails(ExportDataContext... context) throws ExportException { JsonArrayBuilder jab = Json.createArrayBuilder(); - - List fileMetadatas = null; - Optional datasetVersionFilesServiceOptional = getDatasetVersionFilesService(); - - if (datasetVersionFilesServiceOptional.isPresent()) { - - FileSearchCriteria fileSearchCriteria; - try { - fileSearchCriteria = new FileSearchCriteria( - MIME_TYPE_INGESTED_FILE, - isOnlyPublicMetadataRequested(options) ? FileSearchCriteria.FileAccessStatus.Public : null, - null, - null, - null - ); - } catch (IllegalArgumentException e) { - throw new ExportException("Failed to build a retrieval query for tabular file metadata"); - } - - - fileMetadatas = datasetVersionFilesServiceOptional.get().getFileMetadatas(dv, length, offset, fileSearchCriteria, DatasetVersionFilesServiceBean.FileOrderCriteria.NameAZ); - } else { - throw new ExportException("EJB DatasetVersionFilesService is not available"); + List fileMetadatas; + DatasetVersionFilesServiceBean datasetVersionFilesService = null; + try { + datasetVersionFilesService = CDI.current().select(DatasetVersionFilesServiceBean.class).get(); + } catch (java.lang.IllegalArgumentException | IllegalStateException ie) { + throw new ExportException("EJB DatasetVersionFilesService is not available; " + ie.getMessage()); } - + + if (datasetVersionFilesService == null) { + throw new ExportException("EJB DatasetVersionFilesService is not available"); + } + + FileSearchCriteria fileSearchCriteria; + try { + fileSearchCriteria = new FileSearchCriteria( + MIME_TYPE_INGESTED_FILE, + isOnlyPublicMetadataRequested(context) ? FileSearchCriteria.FileAccessStatus.Public : null, + null, + null, + null + ); + } catch (IllegalArgumentException e) { + throw new ExportException("Failed to build a retrieval query for tabular file metadata"); + } + + fileMetadatas = datasetVersionFilesService.getFileMetadatas(dv, getLength(context), getOffset(context), fileSearchCriteria, DatasetVersionFilesServiceBean.FileOrderCriteria.NameAZ); + for (FileMetadata fileMetadata : fileMetadatas) { DataFile dataFile = fileMetadata.getDataFile(); jab.add(JsonPrinter.json(dataFile, fileMetadata, true)); @@ -147,47 +155,70 @@ public JsonArray getTabularDataDetails(Integer offset, Integer length, ExportDat } @Override - public Optional getPrerequisiteInputStream(ExportDataOption... options) { + public Optional getPrerequisiteInputStream(ExportDataContext... context) { return Optional.ofNullable(is); } - public Optional getDatasetVersionFilesService() { - return Optional.ofNullable(datasetVersionFilesService); - } - public void setPrerequisiteInputStream(InputStream prereqStream) { this.is=prereqStream; } - public void setDatasetVersionFilesService(DatasetVersionFilesServiceBean datasetVersionFilesService) { - this.datasetVersionFilesService = datasetVersionFilesService; - } - - private boolean isOnlyDatasetLevelMetadataRequested(ExportDataOption... options) { - for (ExportDataOption option : options) { - - if (option.isDatasetMetadataOnly()) { - return true; - } + /** + * Only one context object is supported + * @param contexts + * @return + */ + private boolean isOnlyDatasetLevelMetadataRequested(ExportDataContext... contexts) { + for (ExportDataContext context : contexts) { + return context.isDatasetMetadataOnly(); } - // By default, we pack both the Dataset, and the File-level metadata in that Json + // By default, if no context is supplied, we pack both the Dataset, and + // the File-level metadata in that Json return false; } - private boolean isOnlyPublicMetadataRequested(ExportDataOption... options) throws ExportException { + /** + * Only one context object is supported + * + * @param contexts + * @return + */ + private boolean isOnlyPublicMetadataRequested(ExportDataContext... contexts) { + + for (ExportDataContext context : contexts) { + return context.isPublicFilesOnly(); + } - for (ExportDataOption option : options) { + // By default, if no context is supplied, we return the metadata for all + // files - embargoed, restricted, etc.: + return false; + } - if (option.isPublicFilesOnly()) { - return true; - } else { - throw new ExportException("Unsupported data export option"); - } + /** + * Only one context object is supported + * + * @param contexts + * @return + */ + private Integer getOffset(ExportDataContext... contexts) { + for (ExportDataContext context : contexts) { + return context.getOffset(); } + return null; + } - // By default, we return the metadata for all files - embargoed, restricted, etc.: - return false; + /** + * Only one context object is supported + * + * @param contexts + * @return + */ + private Integer getLength(ExportDataContext... contexts) { + for (ExportDataContext context : contexts) { + return context.getLength(); + } + return null; } } diff --git a/src/main/java/edu/harvard/iq/dataverse/export/ddi/DdiExportUtil.java b/src/main/java/edu/harvard/iq/dataverse/export/ddi/DdiExportUtil.java index f5cc86bf8ee..f30b3d16b42 100644 --- a/src/main/java/edu/harvard/iq/dataverse/export/ddi/DdiExportUtil.java +++ b/src/main/java/edu/harvard/iq/dataverse/export/ddi/DdiExportUtil.java @@ -6,6 +6,7 @@ import edu.harvard.iq.dataverse.DatasetFieldConstant; import edu.harvard.iq.dataverse.DvObjectContainer; import edu.harvard.iq.dataverse.GlobalId; +import edu.harvard.iq.dataverse.api.dto.DataTableDTO; import edu.harvard.iq.dataverse.api.dto.MetadataBlockDTO; import edu.harvard.iq.dataverse.api.dto.DatasetDTO; import edu.harvard.iq.dataverse.api.dto.DatasetVersionDTO; @@ -31,6 +32,7 @@ import edu.harvard.iq.dataverse.util.xml.XmlPrinter; import edu.harvard.iq.dataverse.util.xml.XmlUtil; import edu.harvard.iq.dataverse.util.xml.XmlWriterUtil; +import io.gdcc.spi.export.ExportDataProvider; import java.io.ByteArrayOutputStream; import java.io.IOException; @@ -81,6 +83,7 @@ public class DdiExportUtil { public static final String NOTE_TYPE_CONTENTTYPE = "DATAVERSE:CONTENTTYPE"; public static final String NOTE_SUBJECT_CONTENTTYPE = "Content/MIME Type"; public static final String CITATION_BLOCK_NAME = "citation"; + public static final int DATATABLES_BATCH_SIZE = 50; //Some tests don't send real PIDs that can be parsed //Use constant empty PID in these cases @@ -125,7 +128,13 @@ private static void dtoddi(DatasetDTO datasetDto, OutputStream outputStream) thr xmlw.writeAttribute("xml:lang", datasetDto.getMetadataLanguage()); } createStdyDscr(xmlw, datasetDto); - createOtherMats(xmlw, datasetDto.getDatasetVersion().getFiles()); + if (datasetDto.getDatasetVersion().getFiles() != null) { + // We create "otherMat" sections with skipTabularFiles = false, because + // this is the short version of the DDI where all the files, whether + // ingested or not, are encoded as otherMats: + + createOtherMats(xmlw, datasetDto.getDatasetVersion().getFiles(), false); + } xmlw.writeEndElement(); // codeBook xmlw.flush(); } finally { @@ -142,11 +151,11 @@ private static void dtoddi(DatasetDTO datasetDto, OutputStream outputStream) thr // "full" ddi, with the the "" and "/" sections: - public static void datasetJson2ddi(JsonObject datasetDtoAsJson, JsonArray fileDetails, OutputStream outputStream) throws XMLStreamException { + public static void datasetJson2ddi(JsonObject datasetDtoAsJson, ExportDataProvider dataProvider, OutputStream outputStream) throws XMLStreamException { logger.fine(JsonUtil.prettyPrint(datasetDtoAsJson.toString())); Gson gson = new Gson(); DatasetDTO datasetDto = gson.fromJson(datasetDtoAsJson.toString(), DatasetDTO.class); - + XMLStreamWriter xmlw = null; try { xmlw = XMLOutputFactory.newInstance().createXMLStreamWriter(outputStream); @@ -160,9 +169,37 @@ public static void datasetJson2ddi(JsonObject datasetDtoAsJson, JsonArray fileDe xmlw.writeAttribute("xml:lang", datasetDto.getMetadataLanguage()); } createStdyDscr(xmlw, datasetDto); - createFileDscr(xmlw, fileDetails); - createDataDscr(xmlw, fileDetails); - createOtherMatsFromFileMetadatas(xmlw, fileDetails); + + // If there are no files in this dataset, we can stop here + if (datasetDto.getDatasetVersion().getFiles() != null) { + + // The Files and Data section, for the rich metadata describing + // the "ingested" tabular data files. + // Note that as of 6.8, we are generating the fileDscr from the DTOs + // supplied by ExportDataProvider.ExportDataProvider.getDatasetJson() + int tabularFilesTotal = createFileDscrs(xmlw, datasetDto.getDatasetVersion().getFiles()); + + if (tabularFilesTotal > 0) { + // Now that we know that there is 1 or more ingested tabular file + // in the dataset, we can try and produce the dataDscr section. + // A dataset with a large number + // of ingested files may contain more of such metadata than is + // practical or desirable to pass around as a single chunk of json. + // As of the ExportDataProvider v2.1.0 a more efficient method is + // provided for retrieving this information in chunks of length-offset + // datatables-worth at a time. + if (tabularFilesTotal <= DATATABLES_BATCH_SIZE) { + createDataDscr(xmlw, dataProvider.getDatasetFileDetails()); + } else { + createDataDscrInBatches(xmlw, dataProvider); + } + } + // otherMats section: + // Note that we are asking createOtherMats() to skip tabular files, + // since we have already created the fileDscr and dataDscr sections + // for those. + createOtherMats(xmlw, datasetDto.getDatasetVersion().getFiles(), true); + } xmlw.writeEndElement(); // codeBook xmlw.flush(); } finally { @@ -1427,7 +1464,7 @@ private static void writeNotesElement(XMLStreamWriter xmlw, DatasetVersionDTO da // see if there's more information that we could encode in this otherMat. // contentType? Unfs and such? (in the "short" DDI that is being used for // harvesting *all* files are encoded as otherMats; even tabular ones. - private static void createOtherMats(XMLStreamWriter xmlw, List fileDtos) throws XMLStreamException { + private static void createOtherMats(XMLStreamWriter xmlw, List fileDtos, boolean skipTabularFiles) throws XMLStreamException { // The preferred URL for this dataverse, for cooking up the file access API links: String dataverseUrl = SystemConfig.getDataverseSiteUrlStatic(); @@ -1435,7 +1472,7 @@ private static void createOtherMats(XMLStreamWriter xmlw, List fileDtos // We'll continue using the scheme we've used before, in DVN2-3: non-tabular files are put into otherMat, // tabular ones - in fileDscr sections. (fileDscr sections have special fields for numbers of variables // and observations, etc.) - if (fileDTo.getDataFile().getDataTables() == null || fileDTo.getDataFile().getDataTables().isEmpty()) { + if (!(skipTabularFiles && isTabularData(fileDTo))) { xmlw.writeStartElement("otherMat"); XmlWriterUtil.writeAttribute(xmlw, "ID", "f" + fileDTo.getDataFile().getId()); String pidURL = fileDTo.getDataFile().getPidURL(); @@ -1473,6 +1510,13 @@ private static void createOtherMats(XMLStreamWriter xmlw, List fileDtos // tell if this file is in fact tabular data - so that we know if it needs an // otherMat, or a fileDscr section. // -- L.A. 4.5 + // [update:] Since the comment above was written, the method below was changed + // to operate on a JsonArray, as provided by the FileDetails method in the + // ExportDataProvider. However, As of 6.8, this method is no longer used at + // all. This is because the DTOs supplied by ExportDataProvider.getDatasetJson() + // DO in fact contain enough information to generate the otherMat sections + // properly, whether this is a short or a full version of the DDI. I am however leaving + // this method here for reference. private static void createOtherMatsFromFileMetadatas(XMLStreamWriter xmlw, JsonArray fileDetails) throws XMLStreamException { // The preferred URL for this dataverse, for cooking up the file access API links: @@ -1483,7 +1527,7 @@ private static void createOtherMatsFromFileMetadatas(XMLStreamWriter xmlw, JsonA // We'll continue using the scheme we've used before, in DVN2-3: non-tabular files are put into otherMat, // tabular ones - in fileDscr sections. (fileDscr sections have special fields for numbers of variables // and observations, etc.) - if (!fileJson.containsKey("dataTables")) { + if (!fileJson.getBoolean("tabularData", false)) { xmlw.writeStartElement("otherMat"); xmlw.writeAttribute("ID", "f" + fileJson.getJsonNumber(("id").toString())); if (fileJson.containsKey("pidUrl")){ @@ -1519,12 +1563,14 @@ private static void createOtherMatsFromFileMetadatas(XMLStreamWriter xmlw, JsonA } private static void writeFileDescription(XMLStreamWriter xmlw, FileDTO fileDTo) throws XMLStreamException { - xmlw.writeStartElement("txt"); String description = fileDTo.getDataFile().getDescription(); if (description != null) { + xmlw.writeStartElement("txt"); + xmlw.writeCharacters(description); + xmlw.writeEndElement(); // txt + } - xmlw.writeEndElement(); // txt } @@ -1646,6 +1692,13 @@ public static void createDataDscr(XMLStreamWriter xmlw, JsonArray fileDetails) t xmlw.writeEndElement(); // dataDscr } } + + public static void createDataDscrInBatches(XMLStreamWriter xmlw, ExportDataProvider exportDataProvider) throws XMLStreamException { + int offset = 0; + boolean inprogress = true; + + } + private static void createVarGroupDDI(XMLStreamWriter xmlw, JsonObject varGrp) throws XMLStreamException { xmlw.writeStartElement("varGrp"); xmlw.writeAttribute("ID", "VG" + varGrp.getJsonNumber("id").toString()); @@ -1909,45 +1962,42 @@ private static void createVarDDI(XMLStreamWriter xmlw, JsonObject dvar, String f } - private static void createFileDscr(XMLStreamWriter xmlw, JsonArray fileDetails) throws XMLStreamException { + private static int createFileDscrs(XMLStreamWriter xmlw, List fileDtos) throws XMLStreamException { String dataverseUrl = SystemConfig.getDataverseSiteUrlStatic(); - for (int i =0;i field) - if (fileJson.containsKey("description")) { + if (fileDTo.getDataFile().getDescription() != null) { xmlw.writeStartElement("notes"); xmlw.writeAttribute("level", LEVEL_FILE); xmlw.writeAttribute("type", NOTE_TYPE_FILEDESCRIPTION); xmlw.writeAttribute("subject", NOTE_SUBJECT_FILEDESCRIPTION); - xmlw.writeCharacters(fileJson.getString("description")); + xmlw.writeCharacters(fileDTo.getDataFile().getDescription()); xmlw.writeEndElement(); // notes } // TODO: add the remaining fileDscr elements! xmlw.writeEndElement(); // fileDscr + counter++; } } + return counter; } - - - - - + public static void datasetHtmlDDI(InputStream datafile, OutputStream outputStream) throws XMLStreamException { try { @@ -2051,5 +2099,9 @@ public static void datasetHtmlDDI(InputStream datafile, OutputStream outputStrea public static void injectSettingsService(SettingsServiceBean settingsSvc) { settingsService=settingsSvc; } + + private static boolean isTabularData(FileDTO fileDTO) { + return !(fileDTO.getDataFile().getDataTables() == null || fileDTO.getDataFile().getDataTables().isEmpty()); + } } diff --git a/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java b/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java index 9fd50d1a868..ca4cc0011a1 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java @@ -58,6 +58,7 @@ import jakarta.ejb.Singleton; import jakarta.json.JsonArray; import jakarta.json.JsonObject; +import java.math.BigDecimal; /** * Convert objects to Json. @@ -491,6 +492,11 @@ public static JsonObjectBuilder json(DatasetVersion dsv, List anonymized } public static JsonObjectBuilder json(DatasetVersion dsv, List anonymizedFieldTypeNamesList, boolean includeFiles, boolean returnOwners, boolean includeMetadataBlocks) { + return json(dsv, anonymizedFieldTypeNamesList, includeFiles, returnOwners, includeMetadataBlocks, false); + } + + public static JsonObjectBuilder json(DatasetVersion dsv, List anonymizedFieldTypeNamesList, + boolean includeFiles, boolean returnOwners, boolean includeMetadataBlocks, boolean forExportDataProvider) { Dataset dataset = dsv.getDataset(); JsonObjectBuilder bld = jsonObjectBuilder() .add("id", dsv.getId()).add("datasetId", dataset.getId()) @@ -548,7 +554,7 @@ public static JsonObjectBuilder json(DatasetVersion dsv, List anonymized bld.add("isPartOf", getOwnersFromDvObject(dataset)); } if (includeFiles) { - bld.add("files", jsonFileMetadatas(dsv.getFileMetadatas())); + bld.add("files", jsonFileMetadatas(dsv.getFileMetadatas(), forExportDataProvider)); } return bld; @@ -573,6 +579,22 @@ public static JsonObjectBuilder jsonDataFileList(List dataFiles){ return bld; } + public static JsonObjectBuilder datasetAsJsonForDTO(DatasetVersion dsv) { + return datasetAsJsonForDTO(dsv, true); + } + + /** + * Same as above, but gives an option to skip the file-level info + * @param dsv + * @param includeFiles + * @return + */ + public static JsonObjectBuilder datasetAsJsonForDTO(DatasetVersion dsv, boolean includeFiles) { + JsonObjectBuilder jsonForDTO = JsonPrinter.json(dsv.getDataset()); + jsonForDTO.add("datasetVersion", versionAsJsonForDTO(dsv, includeFiles)); + return jsonForDTO; + } + /** * Export formats such as DDI require the citation to be included. See * https://github.com/IQSS/dataverse/issues/2579 for more on DDI export. @@ -581,44 +603,20 @@ public static JsonObjectBuilder jsonDataFileList(List dataFiles){ * to the regular `json` method for DatasetVersion? Will anything break? * Unit tests for that method could not be found. */ - public static JsonObjectBuilder jsonWithCitation(DatasetVersion dsv, boolean includeFiles) { - JsonObjectBuilder dsvWithCitation = JsonPrinter.json(dsv, includeFiles); + private static JsonObjectBuilder versionAsJsonForDTO(DatasetVersion dsv, boolean includeFiles) { + JsonObjectBuilder dsvWithCitation = JsonPrinter.json(dsv, null, includeFiles, false,true, true); dsvWithCitation.add("citation", dsv.getCitation()); return dsvWithCitation; } - /** - * Export formats such as DDI require the persistent identifier components - * such as "protocol", "authority" and "identifier" to be included so we - * create a JSON object we can convert to a DatasetDTO which can include a - * DatasetVersionDTO, which has all the metadata fields we need to export. - * See https://github.com/IQSS/dataverse/issues/2579 for more on DDI export. - * - * @todo Instead of having this separate method, should "datasetVersion" be - * added to the regular `json` method for Dataset? Will anything break? Unit - * tests for that method could not be found. If we keep this method as-is - * should the method be renamed? - */ - public static JsonObjectBuilder jsonAsDatasetDto(DatasetVersion dsv) { - return jsonAsDatasetDto(dsv, true); + public static JsonArrayBuilder jsonFileMetadatas(Collection fmds) { + return jsonFileMetadatas(fmds, false); } - /** - * Same as above, but gives an option to skip the file-level info - * @param dsv - * @param includeFiles - * @return - */ - public static JsonObjectBuilder jsonAsDatasetDto(DatasetVersion dsv, boolean includeFiles) { - JsonObjectBuilder datasetDtoAsJson = JsonPrinter.json(dsv.getDataset()); - datasetDtoAsJson.add("datasetVersion", jsonWithCitation(dsv, includeFiles)); - return datasetDtoAsJson; - } - - public static JsonArrayBuilder jsonFileMetadatas(Collection fmds) { + public static JsonArrayBuilder jsonFileMetadatas(Collection fmds, boolean forExportDataProvider) { JsonArrayBuilder filesArr = Json.createArrayBuilder(); for (FileMetadata fmd : fmds) { - filesArr.add(JsonPrinter.json(fmd)); + filesArr.add(JsonPrinter.json(fmd, false, false, forExportDataProvider)); } return filesArr; @@ -832,6 +830,10 @@ public static JsonObjectBuilder json(FileMetadata fmd){ } public static JsonObjectBuilder json(FileMetadata fmd, boolean returnOwners, boolean printDatasetVersion) { + return json(fmd, returnOwners, printDatasetVersion, false); + } + + public static JsonObjectBuilder json(FileMetadata fmd, boolean returnOwners, boolean printDatasetVersion, boolean forExportDataProvider) { NullSafeJsonBuilder builder = jsonObjectBuilder(); // deprecated: .add("category", fmd.getCategory()) @@ -847,7 +849,7 @@ public static JsonObjectBuilder json(FileMetadata fmd, boolean returnOwners, boo .add("version", fmd.getVersion()) .add("datasetVersionId", fmd.getDatasetVersion().getId()) .add("categories", getFileCategories(fmd)) - .add("dataFile", JsonPrinter.json(fmd.getDataFile(), fmd, false, returnOwners)); + .add("dataFile", JsonPrinter.json(fmd.getDataFile(), fmd, forExportDataProvider, returnOwners)); if (printDatasetVersion) { builder.add("datasetVersion", json(fmd.getDatasetVersion(), false)); @@ -877,15 +879,12 @@ public static JsonObjectBuilder json(DataFile df, FileMetadata fileMetadata, boo return json(df, fileMetadata, forExportDataProvider, false); } + public static JsonObjectBuilder json(DataFile df, FileMetadata fileMetadata, boolean forExportDataProvider, boolean returnOwners) { - // File names are no longer stored in the DataFile entity; - // (they are instead in the FileMetadata (as "labels") - this way - // the filename can change between versions... - // It does appear that for some historical purpose we still need the - // filename in the file DTO (?)... We rely on it to be there for the - // DDI export, for example. So we need to make sure this is is the - // *correct* file name - i.e., that it comes from the right version. - // (TODO...? L.A. 4.5, Aug 7 2016) + return json(df, fileMetadata, forExportDataProvider, returnOwners, false); + } + + public static JsonObjectBuilder json(DataFile df, FileMetadata fileMetadata, boolean forExportDataProvider, boolean returnOwners, boolean includeVariables) { String fileName = null; if (fileMetadata == null){ @@ -949,14 +948,18 @@ public static JsonObjectBuilder json(DataFile df, FileMetadata fileMetadata, boo * The restricted state was not included prior to #9175 so to avoid backward * incompatability, it is now only added when generating json for the * InternalExportDataProvider fileDetails. + * [update]: more fields have been added below that are only there + * when the json is requested by the InternalExportDataProvider. */ if (forExportDataProvider) { builder.add("restricted", df.isRestricted()) - .add("fileMetadataId", fileMetadata.getId()) - .add("dataTables", df.getDataTables().isEmpty() ? null : JsonPrinter.jsonDT(df.getDataTables())) - .add("varGroups", fileMetadata.getVarGroups().isEmpty() - ? JsonPrinter.jsonVarGroup(fileMetadata.getVarGroups()) - : null); + .add("fileMetadataId", fileMetadata.getId()) + .add("dataTables", df.getDataTables().isEmpty() ? null : jsonDT(df.getDataTables(), includeVariables)); + if (includeVariables) { + builder.add("varGroups", fileMetadata.getVarGroups().isEmpty() + ? JsonPrinter.jsonVarGroup(fileMetadata.getVarGroups()) + : null); + } } if (returnOwners){ builder.add("isPartOf", getOwnersFromDvObject(df, fileMetadata.getDatasetVersion())); @@ -965,22 +968,24 @@ public static JsonObjectBuilder json(DataFile df, FileMetadata fileMetadata, boo } //Started from https://github.com/RENCI-NRIG/dataverse/, i.e. https://github.com/RENCI-NRIG/dataverse/commit/2b5a1225b42cf1caba85e18abfeb952171c6754a - public static JsonArrayBuilder jsonDT(List ldt) { + public static JsonArrayBuilder jsonDT(List ldt, boolean includeVariables) { JsonArrayBuilder ldtArr = Json.createArrayBuilder(); for(DataTable dt: ldt){ - ldtArr.add(JsonPrinter.json(dt)); + ldtArr.add(JsonPrinter.json(dt, includeVariables)); } return ldtArr; } - public static JsonObjectBuilder json(DataTable dt) { - return jsonObjectBuilder() + public static JsonObjectBuilder json(DataTable dt, boolean includeVariables) { + JsonObjectBuilder builder = jsonObjectBuilder() .add("varQuantity", dt.getVarQuantity()) .add("caseQuantity", dt.getCaseQuantity()) .add("recordsPerCase", dt.getRecordsPerCase()) - .add("UNF", dt.getUnf()) - .add("dataVariables", JsonPrinter.jsonDV(dt.getDataVariables())) - ; + .add("UNF", dt.getUnf()); + if (includeVariables) { + builder.add("dataVariables", JsonPrinter.jsonDV(dt.getDataVariables())); + } + return builder; } public static JsonArrayBuilder jsonDV(List dvl) { From de6228715c5c8f75d85de30dec12844e74d62ec0 Mon Sep 17 00:00:00 2001 From: Leonid Andreev Date: Tue, 2 Sep 2025 16:26:44 -0400 Subject: [PATCH 08/25] alternative, batch implementation #11405 --- .../dataverse/export/ddi/DdiExportUtil.java | 123 +++++++++++++----- 1 file changed, 93 insertions(+), 30 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/export/ddi/DdiExportUtil.java b/src/main/java/edu/harvard/iq/dataverse/export/ddi/DdiExportUtil.java index f30b3d16b42..f4f65550287 100644 --- a/src/main/java/edu/harvard/iq/dataverse/export/ddi/DdiExportUtil.java +++ b/src/main/java/edu/harvard/iq/dataverse/export/ddi/DdiExportUtil.java @@ -32,7 +32,9 @@ import edu.harvard.iq.dataverse.util.xml.XmlPrinter; import edu.harvard.iq.dataverse.util.xml.XmlUtil; import edu.harvard.iq.dataverse.util.xml.XmlWriterUtil; +import io.gdcc.spi.export.ExportDataContext; import io.gdcc.spi.export.ExportDataProvider; +import io.gdcc.spi.export.ExportException; import java.io.ByteArrayOutputStream; import java.io.IOException; @@ -83,7 +85,7 @@ public class DdiExportUtil { public static final String NOTE_TYPE_CONTENTTYPE = "DATAVERSE:CONTENTTYPE"; public static final String NOTE_SUBJECT_CONTENTTYPE = "Content/MIME Type"; public static final String CITATION_BLOCK_NAME = "citation"; - public static final int DATATABLES_BATCH_SIZE = 50; + public static final int DATATABLES_BATCH_SIZE = 1; //50; //Some tests don't send real PIDs that can be parsed //Use constant empty PID in these cases @@ -1640,7 +1642,7 @@ public static void createDataDscr(XMLStreamWriter xmlw, JsonArray fileDetails) t return; } - boolean tabularData = false; + boolean dataDscrWritten = false; // we're not writing the opening tag until we find an actual // tabular datafile. @@ -1654,51 +1656,86 @@ public static void createDataDscr(XMLStreamWriter xmlw, JsonArray fileDetails) t * should instead use the "Data Variable Metadata Access" endpoint.) * These days we skip restricted files to avoid this exposure. */ - if (fileJson.containsKey("restricted") && fileJson.getBoolean("restricted")) { + if (isFileRestricted(fileJson)) { continue; } - if(fileJson.containsKey("embargo")) { - String dateString = fileJson.getJsonObject("embargo").getString("dateAvailable"); - LocalDate endDate = LocalDate.parse(dateString); - if (endDate != null && endDate.isAfter(LocalDate.now())) { - //Embargo is active so skip - continue; - } - } if (fileJson.containsKey("dataTables")) { - if (!tabularData) { + if (!dataDscrWritten) { xmlw.writeStartElement("dataDscr"); - tabularData = true; - } - if(fileJson.containsKey("varGroups")) { - JsonArray varGroups = fileJson.getJsonArray("varGroups"); - for (int j=0;j Date: Thu, 4 Sep 2025 10:38:55 -0400 Subject: [PATCH 09/25] logging statements for debugging #11405 --- .../edu/harvard/iq/dataverse/export/ddi/DdiExportUtil.java | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/main/java/edu/harvard/iq/dataverse/export/ddi/DdiExportUtil.java b/src/main/java/edu/harvard/iq/dataverse/export/ddi/DdiExportUtil.java index f4f65550287..2d078d3302e 100644 --- a/src/main/java/edu/harvard/iq/dataverse/export/ddi/DdiExportUtil.java +++ b/src/main/java/edu/harvard/iq/dataverse/export/ddi/DdiExportUtil.java @@ -1683,6 +1683,7 @@ public static void createDataDscrInBatches(XMLStreamWriter xmlw, ExportDataProvi try { while (inProgress) { JsonArray tabularFileDetails = exportDataProvider.getTabularDataDetails(ExportDataContext.context().withOffset(offset).withLength(DATATABLES_BATCH_SIZE)); + logger.info("retrieved "+tabularFileDetails.size()+" tabular file data entries"); for (int i = 0; i < tabularFileDetails.size(); i++) { JsonObject fileJson = tabularFileDetails.getJsonObject(i); @@ -1701,6 +1702,7 @@ public static void createDataDscrInBatches(XMLStreamWriter xmlw, ExportDataProvi } } + offset+=DATATABLES_BATCH_SIZE; inProgress = tabularFileDetails.size() == DATATABLES_BATCH_SIZE; } } catch (ExportException ee) { @@ -1728,6 +1730,7 @@ private static void createVariablesForDataFile(XMLStreamWriter xmlw, JsonObject } JsonObject dataTable = fileJson.getJsonArray("dataTables").getJsonObject(0); JsonArray vars = dataTable.getJsonArray("dataVariables"); + logger.info(vars.size() + " variables retrieved for file " + fileJson.getJsonNumber("id")); if (vars != null) { for (int j = 0; j < vars.size(); j++) { createVarDDI(xmlw, vars.getJsonObject(j), fileJson.getJsonNumber("id").toString(), @@ -2000,9 +2003,11 @@ private static void createVarDDI(XMLStreamWriter xmlw, JsonObject dvar, String f } private static int createFileDscrs(XMLStreamWriter xmlw, List fileDtos) throws XMLStreamException { + logger.info(fileDtos.size() + "files passed"); String dataverseUrl = SystemConfig.getDataverseSiteUrlStatic(); int counter = 0; for (FileDTO fileDTo : fileDtos) { + logger.info("processing file "+ fileDTo.getDataFile().getId()); if (isTabularData(fileDTo)) { xmlw.writeStartElement("fileDscr"); @@ -2089,6 +2094,7 @@ private static int createFileDscrs(XMLStreamWriter xmlw, List fileDtos) counter++; } } + logger.info("produced "+counter+" fileDscr entries; returning"); return counter; } From a8a7ac96d5883aee87af38d1bb222d7969b4fb2b Mon Sep 17 00:00:00 2001 From: Leonid Andreev Date: Fri, 5 Sep 2025 17:12:54 -0400 Subject: [PATCH 10/25] A quick fix for the batch-based datatable metadata processing #11405 --- .../iq/dataverse/export/InternalExportDataProvider.java | 2 +- .../edu/harvard/iq/dataverse/export/ddi/DdiExportUtil.java | 2 +- .../java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java | 5 ++++- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/export/InternalExportDataProvider.java b/src/main/java/edu/harvard/iq/dataverse/export/InternalExportDataProvider.java index f4962cefc0b..c8c1d8efeb0 100644 --- a/src/main/java/edu/harvard/iq/dataverse/export/InternalExportDataProvider.java +++ b/src/main/java/edu/harvard/iq/dataverse/export/InternalExportDataProvider.java @@ -149,7 +149,7 @@ public JsonArray getTabularDataDetails(ExportDataContext... context) throws Expo for (FileMetadata fileMetadata : fileMetadatas) { DataFile dataFile = fileMetadata.getDataFile(); - jab.add(JsonPrinter.json(dataFile, fileMetadata, true)); + jab.add(JsonPrinter.jsonDatafileWithDatatableForExport(dataFile, fileMetadata)); } return jab.build(); } diff --git a/src/main/java/edu/harvard/iq/dataverse/export/ddi/DdiExportUtil.java b/src/main/java/edu/harvard/iq/dataverse/export/ddi/DdiExportUtil.java index 2d078d3302e..b9695d6f838 100644 --- a/src/main/java/edu/harvard/iq/dataverse/export/ddi/DdiExportUtil.java +++ b/src/main/java/edu/harvard/iq/dataverse/export/ddi/DdiExportUtil.java @@ -85,7 +85,7 @@ public class DdiExportUtil { public static final String NOTE_TYPE_CONTENTTYPE = "DATAVERSE:CONTENTTYPE"; public static final String NOTE_SUBJECT_CONTENTTYPE = "Content/MIME Type"; public static final String CITATION_BLOCK_NAME = "citation"; - public static final int DATATABLES_BATCH_SIZE = 1; //50; + public static final int DATATABLES_BATCH_SIZE = 50; //Some tests don't send real PIDs that can be parsed //Use constant empty PID in these cases diff --git a/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java b/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java index ca4cc0011a1..82d55d0e3e0 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java @@ -879,11 +879,14 @@ public static JsonObjectBuilder json(DataFile df, FileMetadata fileMetadata, boo return json(df, fileMetadata, forExportDataProvider, false); } - public static JsonObjectBuilder json(DataFile df, FileMetadata fileMetadata, boolean forExportDataProvider, boolean returnOwners) { return json(df, fileMetadata, forExportDataProvider, returnOwners, false); } + public static JsonObjectBuilder jsonDatafileWithDatatableForExport(DataFile df, FileMetadata fileMetadata) { + return json(df, fileMetadata, true, false, true); + } + public static JsonObjectBuilder json(DataFile df, FileMetadata fileMetadata, boolean forExportDataProvider, boolean returnOwners, boolean includeVariables) { String fileName = null; From b1ed132cbabcbe5019faed168e169c5a6b55554c Mon Sep 17 00:00:00 2001 From: Leonid Andreev Date: Mon, 8 Sep 2025 14:47:17 -0400 Subject: [PATCH 11/25] implemented a dedicated method in the VersionFileService for looking up filemetadatas of ingested tabular datafiles only (no longer relying on the mime type, which was suboptimal). #11405 --- .../DatasetVersionFilesServiceBean.java | 49 +++++++++++++++++++ .../export/InternalExportDataProvider.java | 20 ++------ .../dataverse/export/ddi/DdiExportUtil.java | 10 ++-- 3 files changed, 59 insertions(+), 20 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetVersionFilesServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/DatasetVersionFilesServiceBean.java index 27c91e8b312..4687d72bdbc 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DatasetVersionFilesServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/DatasetVersionFilesServiceBean.java @@ -180,6 +180,55 @@ public List getFileMetadatas(DatasetVersion datasetVersion, Intege } return typedQuery.getResultList(); } + + /** + * Similar to the above, but dedicated for retrieving FileMetadatas of only + * tabular datafiles in the specified DatasetVersion. Used in the metadata + * export subsystem. + * + * @param datasetVersion the DatasetVersion to access + * @param limit for pagination, can be null + * @param offset for pagination, can be null + * @param publicFilesOnly skip restricted, embargoed etc. files + * @return a FileMetadata list from the specified DatasetVersion + */ + public List getTabularDataFileMetadatas(DatasetVersion datasetVersion, Integer limit, Integer offset, boolean publicFilesOnly) { + CriteriaBuilder criteriaBuilder = em.getCriteriaBuilder(); + CriteriaQuery criteriaQuery = criteriaBuilder.createQuery(FileMetadata.class); + + Root fileMetadataRoot = criteriaQuery.from(FileMetadata.class); + Predicate basePredicate = criteriaBuilder.equal(fileMetadataRoot.get("datasetVersion").get("id"), datasetVersion.getId()); + + Root dataTableRoot = criteriaQuery.from(DataTable.class); + Predicate tabularPredicate = criteriaBuilder.equal(dataTableRoot.get("dataFile"), fileMetadataRoot.get("dataFile")); + + Predicate combinedPredicate; + + if (publicFilesOnly) { + combinedPredicate = criteriaBuilder.and(basePredicate, tabularPredicate); + } else { + combinedPredicate = criteriaBuilder.and(basePredicate, + tabularPredicate, + createSearchCriteriaAccessStatusPredicate(FileSearchCriteria.FileAccessStatus.Public, + criteriaBuilder, + fileMetadataRoot)); + } + + criteriaQuery + .select(fileMetadataRoot) + .where(combinedPredicate) + .orderBy(criteriaBuilder.asc(fileMetadataRoot.get("label"))); + + TypedQuery typedQuery = em.createQuery(criteriaQuery); + if (limit != null) { + typedQuery.setMaxResults(limit); + } + if (offset != null) { + typedQuery.setFirstResult(offset); + } + + return typedQuery.getResultList(); + } /** * Returns the total download size of all files for a particular DatasetVersion diff --git a/src/main/java/edu/harvard/iq/dataverse/export/InternalExportDataProvider.java b/src/main/java/edu/harvard/iq/dataverse/export/InternalExportDataProvider.java index c8c1d8efeb0..c0dc92b0f7e 100644 --- a/src/main/java/edu/harvard/iq/dataverse/export/InternalExportDataProvider.java +++ b/src/main/java/edu/harvard/iq/dataverse/export/InternalExportDataProvider.java @@ -132,21 +132,11 @@ public JsonArray getTabularDataDetails(ExportDataContext... context) throws Expo throw new ExportException("EJB DatasetVersionFilesService is not available"); } - FileSearchCriteria fileSearchCriteria; - try { - fileSearchCriteria = new FileSearchCriteria( - MIME_TYPE_INGESTED_FILE, - isOnlyPublicMetadataRequested(context) ? FileSearchCriteria.FileAccessStatus.Public : null, - null, - null, - null - ); - } catch (IllegalArgumentException e) { - throw new ExportException("Failed to build a retrieval query for tabular file metadata"); - } - - fileMetadatas = datasetVersionFilesService.getFileMetadatas(dv, getLength(context), getOffset(context), fileSearchCriteria, DatasetVersionFilesServiceBean.FileOrderCriteria.NameAZ); - + fileMetadatas = datasetVersionFilesService.getTabularDataFileMetadatas(dv, + getLength(context), + getOffset(context), + isOnlyPublicMetadataRequested(context)); + for (FileMetadata fileMetadata : fileMetadatas) { DataFile dataFile = fileMetadata.getDataFile(); jab.add(JsonPrinter.jsonDatafileWithDatatableForExport(dataFile, fileMetadata)); diff --git a/src/main/java/edu/harvard/iq/dataverse/export/ddi/DdiExportUtil.java b/src/main/java/edu/harvard/iq/dataverse/export/ddi/DdiExportUtil.java index b9695d6f838..6cbc467ac96 100644 --- a/src/main/java/edu/harvard/iq/dataverse/export/ddi/DdiExportUtil.java +++ b/src/main/java/edu/harvard/iq/dataverse/export/ddi/DdiExportUtil.java @@ -1683,7 +1683,7 @@ public static void createDataDscrInBatches(XMLStreamWriter xmlw, ExportDataProvi try { while (inProgress) { JsonArray tabularFileDetails = exportDataProvider.getTabularDataDetails(ExportDataContext.context().withOffset(offset).withLength(DATATABLES_BATCH_SIZE)); - logger.info("retrieved "+tabularFileDetails.size()+" tabular file data entries"); + logger.fine("retrieved " + tabularFileDetails.size() + " tabular file data entries"); for (int i = 0; i < tabularFileDetails.size(); i++) { JsonObject fileJson = tabularFileDetails.getJsonObject(i); @@ -1730,7 +1730,7 @@ private static void createVariablesForDataFile(XMLStreamWriter xmlw, JsonObject } JsonObject dataTable = fileJson.getJsonArray("dataTables").getJsonObject(0); JsonArray vars = dataTable.getJsonArray("dataVariables"); - logger.info(vars.size() + " variables retrieved for file " + fileJson.getJsonNumber("id")); + logger.fine(vars.size() + " variables retrieved for file " + fileJson.getJsonNumber("id")); if (vars != null) { for (int j = 0; j < vars.size(); j++) { createVarDDI(xmlw, vars.getJsonObject(j), fileJson.getJsonNumber("id").toString(), @@ -2003,11 +2003,11 @@ private static void createVarDDI(XMLStreamWriter xmlw, JsonObject dvar, String f } private static int createFileDscrs(XMLStreamWriter xmlw, List fileDtos) throws XMLStreamException { - logger.info(fileDtos.size() + "files passed"); + logger.fine("total " + fileDtos.size() + " file DTOs to process for fileDscr"); String dataverseUrl = SystemConfig.getDataverseSiteUrlStatic(); int counter = 0; for (FileDTO fileDTo : fileDtos) { - logger.info("processing file "+ fileDTo.getDataFile().getId()); + logger.fine("processing file " + fileDTo.getDataFile().getId()); if (isTabularData(fileDTo)) { xmlw.writeStartElement("fileDscr"); @@ -2094,7 +2094,7 @@ private static int createFileDscrs(XMLStreamWriter xmlw, List fileDtos) counter++; } } - logger.info("produced "+counter+" fileDscr entries; returning"); + logger.fine("produced " + counter + " fileDscr entries."); return counter; } From 933f6a7228abd686a2171884ae215526456db3df Mon Sep 17 00:00:00 2001 From: Leonid Andreev Date: Tue, 9 Sep 2025 15:29:03 -0400 Subject: [PATCH 12/25] experimental attempts to build with snapshot releases of dataverse-spi #11405 --- modules/dataverse-parent/pom.xml | 18 ++++++++++++------ pom.xml | 2 +- 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/modules/dataverse-parent/pom.xml b/modules/dataverse-parent/pom.xml index e6acf8d48be..5b6c39f53ab 100644 --- a/modules/dataverse-parent/pom.xml +++ b/modules/dataverse-parent/pom.xml @@ -419,19 +419,25 @@ unidata-all Unidata All https://artifacts.unidata.ucar.edu/repository/unidata-all/ + + false + - + + --> diff --git a/pom.xml b/pom.xml index 38f38a431a8..c89ada43836 100644 --- a/pom.xml +++ b/pom.xml @@ -671,7 +671,7 @@ io.gdcc dataverse-spi - 2.1.0 + 2.1.0-SNAPSHOT javax.cache From a5ca53d26904f2184b28e71f1034c6a3aabb4264 Mon Sep 17 00:00:00 2001 From: Leonid Andreev Date: Tue, 9 Sep 2025 17:18:49 -0400 Subject: [PATCH 13/25] fixing the typo in the repo url :( #11405 --- modules/dataverse-parent/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/dataverse-parent/pom.xml b/modules/dataverse-parent/pom.xml index 5b6c39f53ab..87c11a4e2c4 100644 --- a/modules/dataverse-parent/pom.xml +++ b/modules/dataverse-parent/pom.xml @@ -428,7 +428,7 @@ central-portal-snapshots Central Portal Snapshots - https://central.sonatype.org/repository/maven-snapshots/ + https://central.sonatype.com/repository/maven-snapshots/ false From 11cd77c05dd4efb5c6ee1d893589a3ceee817fdb Mon Sep 17 00:00:00 2001 From: Leonid Andreev Date: Tue, 16 Sep 2025 10:43:06 -0400 Subject: [PATCH 14/25] Adding the option of re-exporting select formats only to the API #11405 --- .../iq/dataverse/DatasetServiceBean.java | 15 +++-- .../harvard/iq/dataverse/api/Metadata.java | 11 +++- .../iq/dataverse/export/ExportService.java | 62 ++++++++++++++++++- .../harvest/server/OAIRecordServiceBean.java | 7 ++- 4 files changed, 84 insertions(+), 11 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/DatasetServiceBean.java index f52163192f7..1d58d309a54 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DatasetServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/DatasetServiceBean.java @@ -739,7 +739,7 @@ public void exportAllDatasets(boolean forceReExport) { || dataset.getLastExportTime().before(publicationDate)))) { countAll++; try { - recordService.exportAllFormatsInNewTransaction(dataset); + recordService.exportFormatsInNewTransaction(dataset, null); exportLogger.info("Success exporting dataset: " + dataset.getDisplayName() + " " + dataset.getGlobalId().asString()); countSuccess++; } catch (Exception ex) { @@ -763,13 +763,18 @@ public void exportAllDatasets(boolean forceReExport) { @Asynchronous public void reExportDatasetAsync(Dataset dataset) { - exportDataset(dataset, true); + exportDataset(dataset, true, null); + } + + @Asynchronous + public void reExportDatasetAsync(Dataset dataset, List formatNames) { + exportDataset(dataset, true, formatNames); } - public void exportDataset(Dataset dataset, boolean forceReExport) { + private void exportDataset(Dataset dataset, boolean forceReExport, List formatNames) { if (dataset != null) { // Note that the logic for handling a dataset is similar to what is implemented in exportAllDatasets, - // but when only one dataset is exported we do not log in a separate export logging file + // but when only one dataset is exported we do not use a dedicated log file if (dataset.isReleased() && dataset.getReleasedVersion() != null && !dataset.isDeaccessioned()) { // can't trust dataset.getPublicationDate(), no. @@ -778,7 +783,7 @@ public void exportDataset(Dataset dataset, boolean forceReExport) { && (dataset.getLastExportTime() == null || dataset.getLastExportTime().before(publicationDate)))) { try { - recordService.exportAllFormatsInNewTransaction(dataset); + recordService.exportFormatsInNewTransaction(dataset, formatNames); logger.info("Success exporting dataset: " + dataset.getDisplayName() + " " + dataset.getGlobalId().asString()); } catch (Exception ex) { logger.log(Level.INFO, "Error exporting dataset: " + dataset.getDisplayName() + " " + dataset.getGlobalId().asString() + "; " + ex.getMessage(), ex); diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Metadata.java b/src/main/java/edu/harvard/iq/dataverse/api/Metadata.java index bd937878286..351409b39ba 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Metadata.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Metadata.java @@ -19,6 +19,9 @@ import edu.harvard.iq.dataverse.harvest.server.OAISetServiceBean; import edu.harvard.iq.dataverse.harvest.server.OAISet; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; /** * @@ -64,10 +67,14 @@ public Response reExportAll() { @GET @Path("{id}/reExportDataset") - public Response indexDatasetByPersistentId(@PathParam("id") String id) { + public Response indexDatasetByPersistentId(@PathParam("id") String id, @QueryParam("formats") String formats) { try { Dataset dataset = findDatasetOrDie(id); - datasetService.reExportDatasetAsync(dataset); + List formatNames = null; + if (formats != null) { + formatNames = new ArrayList<>(Arrays.asList(formats.split(","))); + } + datasetService.reExportDatasetAsync(dataset, formatNames); return ok("export started"); } catch (WrappedResponse wr) { return wr.getResponse(); diff --git a/src/main/java/edu/harvard/iq/dataverse/export/ExportService.java b/src/main/java/edu/harvard/iq/dataverse/export/ExportService.java index b33305c0f80..0d51f84db7d 100644 --- a/src/main/java/edu/harvard/iq/dataverse/export/ExportService.java +++ b/src/main/java/edu/harvard/iq/dataverse/export/ExportService.java @@ -318,16 +318,72 @@ public void exportAllFormats(Dataset dataset) throws ExportException { } } + + public void exportFormats(Dataset dataset, List formatNames) throws ExportException { + try { + if (formatNames == null) { + clearAllCachedFormats(dataset); + } else { + clearCachedFormats(dataset, formatNames); + } + } catch (IOException ex) { + Logger.getLogger(ExportService.class.getName()).log(Level.SEVERE, null, ex); + } - public void clearAllCachedFormats(Dataset dataset) throws IOException { try { + DatasetVersion releasedVersion = dataset.getReleasedVersion(); + if (releasedVersion == null) { + throw new ExportException("No released version for dataset " + dataset.getGlobalId().toString()); + } + InternalExportDataProvider dataProvider = new InternalExportDataProvider(releasedVersion); for (Exporter e : exporterMap.values()) { String formatName = e.getFormatName(); - clearCachedExport(dataset, formatName); + if (formatNames == null || formatNames.contains(formatName)) { + if (e.getPrerequisiteFormatName().isPresent()) { + String prereqFormatName = e.getPrerequisiteFormatName().get(); + try (InputStream preReqStream = getExport(dataset.getReleasedVersion(), prereqFormatName)) { + dataProvider.setPrerequisiteInputStream(preReqStream); + cacheExport(dataset, dataProvider, formatName, e); + dataProvider.setPrerequisiteInputStream(null); + } catch (IOException ioe) { + throw new ExportException("Could not get prerequisite " + e.getPrerequisiteFormatName() + " to create " + formatName + "export for dataset " + dataset.getId(), ioe); + } + } else { + cacheExport(dataset, dataProvider, formatName, e); + } + } } + // Finally, if we have been able to successfully export in all available + // formats, we'll increment the "last exported" time stamp: + dataset.setLastExportTime(new Timestamp(new Date().getTime())); + + } catch (ServiceConfigurationError serviceError) { + throw new ExportException("Service configuration error during export. " + serviceError.getMessage()); + } catch (RuntimeException e) { + logger.log(Level.FINE, e.getMessage(), e); + throw new ExportException( + "Unknown runtime exception exporting metadata. " + (e.getMessage() == null ? "" : e.getMessage())); + } + } - dataset.setLastExportTime(null); + public void clearAllCachedFormats(Dataset dataset) throws IOException { + List formatNames = new ArrayList<>(); + + for (Exporter e : exporterMap.values()) { + String formatName = e.getFormatName(); + formatNames.add(formatName); + clearCachedExport(dataset, formatName); + } + clearCachedFormats(dataset, formatNames); + dataset.setLastExportTime(null); + } + + public void clearCachedFormats(Dataset dataset, List formatNames) throws IOException { + try { + for (String formatName : formatNames) { + clearCachedExport(dataset, formatName); + } } catch (IOException ex) { // not fatal } diff --git a/src/main/java/edu/harvard/iq/dataverse/harvest/server/OAIRecordServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/harvest/server/OAIRecordServiceBean.java index cc15d4c978b..6719d46adf0 100644 --- a/src/main/java/edu/harvard/iq/dataverse/harvest/server/OAIRecordServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/harvest/server/OAIRecordServiceBean.java @@ -259,9 +259,14 @@ public void exportAllFormats(Dataset dataset) { @TransactionAttribute(REQUIRES_NEW) public void exportAllFormatsInNewTransaction(Dataset dataset) throws ExportException { + exportFormatsInNewTransaction(dataset, null); + } + + @TransactionAttribute(REQUIRES_NEW) + public void exportFormatsInNewTransaction(Dataset dataset, List formatNames) throws ExportException { try { ExportService exportServiceInstance = ExportService.getInstance(); - exportServiceInstance.exportAllFormats(dataset); + exportServiceInstance.exportFormats(dataset, formatNames); dataset = datasetService.merge(dataset); } catch (Exception e) { logger.log(Level.FINE, "Caught unknown exception while trying to export", e); From 7dac667392f494fa0198e72cceeb12c3143ffa07 Mon Sep 17 00:00:00 2001 From: Leonid Andreev Date: Mon, 10 Nov 2025 16:45:32 -0500 Subject: [PATCH 15/25] Made the batch-sizing math smarter (processing datavariable metadata in batches based on the number of variables, instead of the fixed number of datatables-worth at a time). #11405 --- .../dataverse/export/ddi/DdiExportUtil.java | 112 ++++++++++++------ 1 file changed, 73 insertions(+), 39 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/export/ddi/DdiExportUtil.java b/src/main/java/edu/harvard/iq/dataverse/export/ddi/DdiExportUtil.java index 6cbc467ac96..96a5fac2790 100644 --- a/src/main/java/edu/harvard/iq/dataverse/export/ddi/DdiExportUtil.java +++ b/src/main/java/edu/harvard/iq/dataverse/export/ddi/DdiExportUtil.java @@ -85,7 +85,8 @@ public class DdiExportUtil { public static final String NOTE_TYPE_CONTENTTYPE = "DATAVERSE:CONTENTTYPE"; public static final String NOTE_SUBJECT_CONTENTTYPE = "Content/MIME Type"; public static final String CITATION_BLOCK_NAME = "citation"; - public static final int DATATABLES_BATCH_SIZE = 50; + public static final int DATATABLES_BATCH_SIZE = 50; + public static final int DATAVARIABLES_BATCH_SIZE = 10000; // todo: review //Some tests don't send real PIDs that can be parsed //Use constant empty PID in these cases @@ -179,9 +180,9 @@ public static void datasetJson2ddi(JsonObject datasetDtoAsJson, ExportDataProvid // the "ingested" tabular data files. // Note that as of 6.8, we are generating the fileDscr from the DTOs // supplied by ExportDataProvider.ExportDataProvider.getDatasetJson() - int tabularFilesTotal = createFileDscrs(xmlw, datasetDto.getDatasetVersion().getFiles()); + List varQuantityMap = createFileDscrs(xmlw, datasetDto.getDatasetVersion().getFiles()); - if (tabularFilesTotal > 0) { + if (varQuantityMap != null && !varQuantityMap.isEmpty()) { // Now that we know that there is 1 or more ingested tabular file // in the dataset, we can try and produce the dataDscr section. // A dataset with a large number @@ -190,10 +191,11 @@ public static void datasetJson2ddi(JsonObject datasetDtoAsJson, ExportDataProvid // As of the ExportDataProvider v2.1.0 a more efficient method is // provided for retrieving this information in chunks of length-offset // datatables-worth at a time. - if (tabularFilesTotal <= DATATABLES_BATCH_SIZE) { - createDataDscr(xmlw, dataProvider.getDatasetFileDetails()); + //if (tabularFilesTotal <= DATATABLES_BATCH_SIZE) { + if (isVarQuantityLimitExceeded(varQuantityMap)) { + createDataDscrInBatches(xmlw, varQuantityMap, dataProvider); } else { - createDataDscrInBatches(xmlw, dataProvider); + createDataDscr(xmlw, dataProvider.getDatasetFileDetails()); } } // otherMats section: @@ -216,6 +218,18 @@ public static void datasetJson2ddi(JsonObject datasetDtoAsJson, ExportDataProvid } } + private static boolean isVarQuantityLimitExceeded(List varQuantityMap) { + if (varQuantityMap != null) { + long varQuantityCount = 0; + for (long varQuantity : varQuantityMap) { + varQuantityCount += varQuantity; + if (varQuantityCount > DATAVARIABLES_BATCH_SIZE) { + return true; + } + } + } + return false; + } /** * @todo This is just a stub, copied from DDIExportServiceBean. It should * produce valid DDI based on @@ -1674,36 +1688,45 @@ public static void createDataDscr(XMLStreamWriter xmlw, JsonArray fileDetails) t xmlw.writeEndElement(); // dataDscr } } - - public static void createDataDscrInBatches(XMLStreamWriter xmlw, ExportDataProvider exportDataProvider) throws XMLStreamException { - int offset = 0; - boolean inProgress = true; + + private static void createDataDscrInBatches(XMLStreamWriter xmlw, List varQuantityMap, ExportDataProvider exportDataProvider) throws XMLStreamException { boolean dataDscrWritten = false; try { - while (inProgress) { - JsonArray tabularFileDetails = exportDataProvider.getTabularDataDetails(ExportDataContext.context().withOffset(offset).withLength(DATATABLES_BATCH_SIZE)); - logger.fine("retrieved " + tabularFileDetails.size() + " tabular file data entries"); + int dataTableStart = 0; + int dataTablesThisBatch = 0; + int varQuantityThisBatch = 0; + + for (int dataTableCurrent = 0; dataTableCurrent < varQuantityMap.size(); dataTableCurrent++) { + varQuantityThisBatch += varQuantityMap.get(dataTableCurrent); + dataTablesThisBatch++; - for (int i = 0; i < tabularFileDetails.size(); i++) { - JsonObject fileJson = tabularFileDetails.getJsonObject(i); + if (varQuantityThisBatch >= DATAVARIABLES_BATCH_SIZE || dataTableCurrent == varQuantityMap.size() - 1) { + JsonArray tabularFileDetails = exportDataProvider.getTabularDataDetails(ExportDataContext.context().withOffset(dataTableStart).withLength(dataTablesThisBatch)); + logger.fine("requested: " + dataTablesThisBatch + " tabular file data entries; retrieved: " + tabularFileDetails.size()); + logger.fine("total number of variables in this batch: " + varQuantityThisBatch); - if (isFileRestricted(fileJson)) { - continue; - } - - if (fileJson.containsKey("dataTables")) { - if (!dataDscrWritten) { - xmlw.writeStartElement("dataDscr"); - dataDscrWritten = true; + for (int i = 0; i < tabularFileDetails.size(); i++) { + JsonObject fileJson = tabularFileDetails.getJsonObject(i); + + if (isFileRestricted(fileJson)) { + continue; + } + + if (fileJson.containsKey("dataTables")) { + if (!dataDscrWritten) { + xmlw.writeStartElement("dataDscr"); + dataDscrWritten = true; + } + + createVariablesForDataFile(xmlw, fileJson); } - - createVariablesForDataFile(xmlw, fileJson); } + + dataTableStart += dataTablesThisBatch; + dataTablesThisBatch = 0; + varQuantityThisBatch = 0; } - - offset+=DATATABLES_BATCH_SIZE; - inProgress = tabularFileDetails.size() == DATATABLES_BATCH_SIZE; } } catch (ExportException ee) { if (dataDscrWritten) { @@ -1718,7 +1741,6 @@ public static void createDataDscrInBatches(XMLStreamWriter xmlw, ExportDataProvi createDataDscr(xmlw, exportDataProvider.getDatasetFileDetails()); } } - } private static void createVariablesForDataFile(XMLStreamWriter xmlw, JsonObject fileJson) throws XMLStreamException { @@ -2001,16 +2023,20 @@ private static void createVarDDI(XMLStreamWriter xmlw, JsonObject dvar, String f xmlw.writeEndElement(); //var } - - private static int createFileDscrs(XMLStreamWriter xmlw, List fileDtos) throws XMLStreamException { - logger.fine("total " + fileDtos.size() + " file DTOs to process for fileDscr"); + + private static List createFileDscrs(XMLStreamWriter xmlw, List fileDtos) throws XMLStreamException { + List ret = new ArrayList<>(); + + logger.fine("total " + fileDtos.size() + " file DTOs to process for fileDscr"); String dataverseUrl = SystemConfig.getDataverseSiteUrlStatic(); int counter = 0; + long totalVarQuantity = 0; + for (FileDTO fileDTo : fileDtos) { logger.fine("processing file " + fileDTo.getDataFile().getId()); if (isTabularData(fileDTo)) { xmlw.writeStartElement("fileDscr"); - + xmlw.writeAttribute("ID", "f" + fileDTo.getDataFile().getId()); xmlw.writeAttribute("URI", dataverseUrl + "/api/access/datafile/" + fileDTo.getDataFile().getId()); @@ -2018,7 +2044,7 @@ private static int createFileDscrs(XMLStreamWriter xmlw, List fileDtos) xmlw.writeStartElement("fileName"); xmlw.writeCharacters(fileDTo.getDataFile().getFilename()); xmlw.writeEndElement(); // fileName - + DataTableDTO dataTableDTO = fileDTo.getDataFile().getDataTables().get(0); if (dataTableDTO.getCaseQuantity() != null || dataTableDTO.getVarQuantity() != null @@ -2031,10 +2057,18 @@ private static int createFileDscrs(XMLStreamWriter xmlw, List fileDtos) xmlw.writeEndElement(); // caseQnty } - if (dataTableDTO.getVarQuantity() != null) { + Long varQuantity = dataTableDTO.getVarQuantity(); + + if (varQuantity != null) { xmlw.writeStartElement("varQnty"); xmlw.writeCharacters(dataTableDTO.getVarQuantity().toString()); xmlw.writeEndElement(); // varQnty + totalVarQuantity += varQuantity; + ret.add(varQuantity.intValue()); + } else { + // Strictly speaking, this should never happen - Dataverse is + // supposed to know the number of variables in every ingested tabular file. + ret.add(0); } if (dataTableDTO.getRecordsPerCase() != null) { @@ -2077,7 +2111,7 @@ private static int createFileDscrs(XMLStreamWriter xmlw, List fileDtos) xmlw.writeEndElement(); // notes } } - + // Adding a dedicated node for the description entry (for // non-tabular files we format it under the field) if (fileDTo.getDataFile().getDescription() != null) { @@ -2091,11 +2125,11 @@ private static int createFileDscrs(XMLStreamWriter xmlw, List fileDtos) // TODO: add the remaining fileDscr elements! xmlw.writeEndElement(); // fileDscr - counter++; + counter++; } } - logger.fine("produced " + counter + " fileDscr entries."); - return counter; + logger.fine("produced " + counter + " fileDscr entries; total number of variables found: " + totalVarQuantity); + return ret; } public static void datasetHtmlDDI(InputStream datafile, OutputStream outputStream) throws XMLStreamException { From b2dc80e357b5744120b82066b517d8a585470da0 Mon Sep 17 00:00:00 2001 From: Leonid Andreev Date: Wed, 6 May 2026 15:29:42 -0400 Subject: [PATCH 16/25] first stab at making the internal exportdataprovider use the latest iteration of dataverse-spi 2.1.0 #11405 --- .../export/InternalExportDataProvider.java | 166 ++++++++++++------ 1 file changed, 113 insertions(+), 53 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/export/InternalExportDataProvider.java b/src/main/java/edu/harvard/iq/dataverse/export/InternalExportDataProvider.java index c0dc92b0f7e..70a20a87306 100644 --- a/src/main/java/edu/harvard/iq/dataverse/export/InternalExportDataProvider.java +++ b/src/main/java/edu/harvard/iq/dataverse/export/InternalExportDataProvider.java @@ -14,16 +14,29 @@ import edu.harvard.iq.dataverse.DatasetVersion; import edu.harvard.iq.dataverse.DatasetVersionFilesServiceBean; import edu.harvard.iq.dataverse.FileMetadata; -import edu.harvard.iq.dataverse.FileSearchCriteria; import edu.harvard.iq.dataverse.pidproviders.doi.datacite.DOIDataCiteRegisterService; import io.gdcc.spi.export.ExportDataProvider; import edu.harvard.iq.dataverse.util.bagit.OREMap; import edu.harvard.iq.dataverse.util.json.JsonPrinter; import edu.harvard.iq.dataverse.util.json.JsonUtil; -import static edu.harvard.iq.dataverse.util.FileUtil.MIME_TYPE_INGESTED_FILE; import io.gdcc.spi.export.ExportException; -import io.gdcc.spi.export.ExportDataContext; +//import io.gdcc.spi.export.ExportDataContext; +import io.gdcc.spi.export.DatasetExportQuery; +import io.gdcc.spi.export.DatasetMetadataPredicates; +import io.gdcc.spi.export.FileExportQuery; +import io.gdcc.spi.export.FileMetadataPredicates; +import io.gdcc.spi.export.PageRequest; +import java.io.IOException; +import java.io.StringReader; import java.util.List; +import java.util.Set; +import java.util.stream.Stream; +import javax.xml.parsers.DocumentBuilder; +import javax.xml.parsers.DocumentBuilderFactory; +import javax.xml.parsers.ParserConfigurationException; +import org.w3c.dom.Document; +import org.xml.sax.InputSource; +import org.xml.sax.SAXException; /** * Provides all data necessary to create an export @@ -49,8 +62,8 @@ public class InternalExportDataProvider implements ExportDataProvider { } @Override - public JsonObject getDatasetJson(ExportDataContext... context) { - if (isOnlyDatasetLevelMetadataRequested(context)) { + public JsonObject getDatasetJson(DatasetExportQuery query) { + if (isOnlyDatasetLevelMetadataRequested(query)) { // If we already have the "full" Json representation (with files) // generated, should we return it (potentially moving MUCH more json // than the client needs, or spend extra cycles generating the short @@ -70,7 +83,7 @@ public JsonObject getDatasetJson(ExportDataContext... context) { } @Override - public JsonObject getDatasetSchemaDotOrg(ExportDataContext... context) { + public JsonObject getDatasetSchemaDotOrg(DatasetExportQuery query) { if (schemaDotOrgRepresentation == null) { String jsonLdAsString = dv.getJsonLd(); schemaDotOrgRepresentation = JsonUtil.getJsonObject(jsonLdAsString); @@ -79,7 +92,7 @@ public JsonObject getDatasetSchemaDotOrg(ExportDataContext... context) { } @Override - public JsonObject getDatasetORE(ExportDataContext... context) { + public JsonObject getDatasetORE(DatasetExportQuery query) { if (oreRepresentation == null) { oreRepresentation = new OREMap(dv).getOREMap(); } @@ -87,13 +100,33 @@ public JsonObject getDatasetORE(ExportDataContext... context) { } @Override - public String getDataCiteXml(ExportDataContext... context) { + public String getDataCiteXml() { + // @todo Is this the best way to obtain the metadata? - as opposed to + // going through the normal Export framework? (it may be, if it needs + // to be version-specific - ?) return DOIDataCiteRegisterService.getMetadataFromDvObject( dv.getDataset().getGlobalId().asString(), new DataCitation(dv).getDataCiteMetadata(), dv.getDataset()); } @Override - public JsonArray getDatasetFileDetails(ExportDataContext... context) { + public Document getDataCiteXml(DatasetExportQuery query) { + // Note that the query parameter is ignored, for now + String dataciteXmlString = getDataCiteXml(); + + DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance(); + + try { + DocumentBuilder builder = factory.newDocumentBuilder(); + + return builder.parse(new InputSource(new StringReader(dataciteXmlString))); + } catch (ParserConfigurationException | SAXException | IOException px) { + return null; + } + + } + + @Override + public JsonArray getDatasetFileDetails() { if (fileAndDataDetails == null) { JsonArrayBuilder jab = Json.createArrayBuilder(); for (FileMetadata fileMetadata : dv.getFileMetadatas()) { @@ -106,6 +139,55 @@ public JsonArray getDatasetFileDetails(ExportDataContext... context) { } @Override + public Stream getDatasetFileDetails(FileExportQuery query) { + if (fileAndDataDetails == null) { + JsonArrayBuilder jab = Json.createArrayBuilder(); + for (FileMetadata fileMetadata : dv.getFileMetadatas()) { + DataFile dataFile = fileMetadata.getDataFile(); + jab.add(JsonPrinter.json(dataFile, fileMetadata, true, false, true)); + } + fileAndDataDetails = jab.build(); + } + return fileAndDataDetails.stream().map(jsonValue -> jsonValue.asJsonObject()); + } + + @Override + public Stream getDatasetFileDetails(FileExportQuery query, PageRequest pageRequest) { + JsonArrayBuilder jab = Json.createArrayBuilder(); + + List fileMetadatas; + DatasetVersionFilesServiceBean datasetVersionFilesService = null; + try { + datasetVersionFilesService = CDI.current().select(DatasetVersionFilesServiceBean.class).get(); + } catch (java.lang.IllegalArgumentException | IllegalStateException ie) { + throw new ExportException("EJB DatasetVersionFilesService is not available; " + ie.getMessage()); + } + + if (datasetVersionFilesService == null) { + throw new ExportException("EJB DatasetVersionFilesService is not available"); + } + + /* + * Defaulting to retrieving tabular/DataVariable-level metadata, for now; + * we will want to honor the related predicates in the long run. + */ + fileMetadatas = datasetVersionFilesService.getTabularDataFileMetadatas(dv, + pageRequest.getOffset(), + pageRequest.getOffset(), + isOnlyPublicMetadataRequested(query)); + + for (FileMetadata fileMetadata : fileMetadatas) { + DataFile dataFile = fileMetadata.getDataFile(); + jab.add(JsonPrinter.jsonDatafileWithDatatableForExport(dataFile, fileMetadata)); + } + + return jab.build().stream().map(jsonValue -> jsonValue.asJsonObject()); + } + + //@Override + // This method, specifically for tabular files only, was in my initial + // implementation of 2.1.0, but later dropped in favor of a more flexible + // getDatasetFileDetails(...) method /** * This new (as of dataverse-spi 2.1.0) method will attempt to retrieve * the requested tabular metadata more efficiently, by calling the @@ -117,7 +199,7 @@ public JsonArray getDatasetFileDetails(ExportDataContext... context) { * getDatasetFileDetails(); * */ - public JsonArray getTabularDataDetails(ExportDataContext... context) throws ExportException { + /*public JsonArray getTabularDataDetails(ExportDataContext... context) throws ExportException { JsonArrayBuilder jab = Json.createArrayBuilder(); List fileMetadatas; @@ -142,10 +224,10 @@ public JsonArray getTabularDataDetails(ExportDataContext... context) throws Expo jab.add(JsonPrinter.jsonDatafileWithDatatableForExport(dataFile, fileMetadata)); } return jab.build(); - } + }*/ @Override - public Optional getPrerequisiteInputStream(ExportDataContext... context) { + public Optional getPrerequisiteInputStream(DatasetExportQuery query) { return Optional.ofNullable(is); } @@ -155,60 +237,38 @@ public void setPrerequisiteInputStream(InputStream prereqStream) { /** * Only one context object is supported - * @param contexts + * @param DatasetExportQuery * @return */ - private boolean isOnlyDatasetLevelMetadataRequested(ExportDataContext... contexts) { - for (ExportDataContext context : contexts) { - return context.isDatasetMetadataOnly(); + private boolean isOnlyDatasetLevelMetadataRequested(DatasetExportQuery query) { + + Set predicates = query.getDatasetPredicates(); + + for (DatasetMetadataPredicates p : predicates) { + // @todo This is pending on adding a dedicated DATASET_LEVEL_ONLY predicate + // to the enum + //if (p.equals(DatasetMetadataPredicates.DATASET_LEVEL_ONLY)) return true; } - // By default, if no context is supplied, we pack both the Dataset, and - // the File-level metadata in that Json + // The default assumption is we pack both the Dataset, and the File-level + // metadata in the Json return false; } /** - * Only one context object is supported + * Are we skipping non-public, restricted and/or embargoed files? * - * @param contexts - * @return + * @param FileExportQuery + * @return yes or no */ - private boolean isOnlyPublicMetadataRequested(ExportDataContext... contexts) { + private boolean isOnlyPublicMetadataRequested(FileExportQuery query) { - for (ExportDataContext context : contexts) { - return context.isPublicFilesOnly(); + Set predicates = query.getFilePredicates(); + + for (FileMetadataPredicates p : predicates) { + if (p.equals(FileMetadataPredicates.ONLY_PUBLIC_FILES)) return true; } - // By default, if no context is supplied, we return the metadata for all - // files - embargoed, restricted, etc.: return false; } - - /** - * Only one context object is supported - * - * @param contexts - * @return - */ - private Integer getOffset(ExportDataContext... contexts) { - for (ExportDataContext context : contexts) { - return context.getOffset(); - } - return null; - } - - /** - * Only one context object is supported - * - * @param contexts - * @return - */ - private Integer getLength(ExportDataContext... contexts) { - for (ExportDataContext context : contexts) { - return context.getLength(); - } - return null; - } - } From da772b41e6a96a77d1728ff32c2f5b2fab8d6d8f Mon Sep 17 00:00:00 2001 From: Leonid Andreev Date: Fri, 8 May 2026 13:51:55 -0400 Subject: [PATCH 17/25] some cleanup/debugging needed to get the exports working again, when built with the new iteration of dataverse-spi 2.1.0 #11405 --- .../iq/dataverse/export/DCTermsExporter.java | 5 +- .../dataverse/export/DublinCoreExporter.java | 4 +- .../export/InternalExportDataProvider.java | 108 +++++++++--------- .../dataverse/export/ddi/DdiExportUtil.java | 39 +++++-- 4 files changed, 93 insertions(+), 63 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/export/DCTermsExporter.java b/src/main/java/edu/harvard/iq/dataverse/export/DCTermsExporter.java index ad5010fdf50..75d9360efd3 100644 --- a/src/main/java/edu/harvard/iq/dataverse/export/DCTermsExporter.java +++ b/src/main/java/edu/harvard/iq/dataverse/export/DCTermsExporter.java @@ -8,7 +8,7 @@ import io.gdcc.spi.export.Exporter; import io.gdcc.spi.export.XMLExporter; import edu.harvard.iq.dataverse.util.BundleUtil; -import io.gdcc.spi.export.ExportDataContext; +//import io.gdcc.spi.export.ExportDataContext; import java.io.OutputStream; import java.util.Locale; import java.util.Optional; @@ -39,7 +39,8 @@ public String getDisplayName(Locale locale) { @Override public void exportDataset(ExportDataProvider dataProvider, OutputStream outputStream) throws ExportException { try { - DublinCoreExportUtil.datasetJson2dublincore(dataProvider.getDatasetJson(ExportDataContext.context().withDatasetMetadataOnly()), outputStream, DublinCoreExportUtil.DC_FLAVOR_DCTERMS); + // @todo add + DublinCoreExportUtil.datasetJson2dublincore(dataProvider.getDatasetJson(/*ExportDataContext.context().withDatasetMetadataOnly()*/), outputStream, DublinCoreExportUtil.DC_FLAVOR_DCTERMS); } catch (XMLStreamException xse) { throw new ExportException("Caught XMLStreamException performing DCTERMS export", xse); } diff --git a/src/main/java/edu/harvard/iq/dataverse/export/DublinCoreExporter.java b/src/main/java/edu/harvard/iq/dataverse/export/DublinCoreExporter.java index db3c28deb78..d368fbe214b 100644 --- a/src/main/java/edu/harvard/iq/dataverse/export/DublinCoreExporter.java +++ b/src/main/java/edu/harvard/iq/dataverse/export/DublinCoreExporter.java @@ -5,7 +5,7 @@ import edu.harvard.iq.dataverse.export.dublincore.DublinCoreExportUtil; import io.gdcc.spi.export.ExportDataProvider; import io.gdcc.spi.export.ExportException; -import io.gdcc.spi.export.ExportDataContext; +//import io.gdcc.spi.export.ExportDataContext; import io.gdcc.spi.export.Exporter; import io.gdcc.spi.export.XMLExporter; import edu.harvard.iq.dataverse.util.BundleUtil; @@ -39,7 +39,7 @@ public String getDisplayName(Locale locale) { @Override public void exportDataset(ExportDataProvider dataProvider, OutputStream outputStream) throws ExportException { try { - DublinCoreExportUtil.datasetJson2dublincore(dataProvider.getDatasetJson(ExportDataContext.context().withDatasetMetadataOnly()), outputStream, + DublinCoreExportUtil.datasetJson2dublincore(dataProvider.getDatasetJson(/*ExportDataContext.context().withDatasetMetadataOnly()*/), outputStream, DublinCoreExportUtil.DC_FLAVOR_OAI); } catch (XMLStreamException xse) { throw new ExportException("Caught XMLStreamException performing DC export", xse); diff --git a/src/main/java/edu/harvard/iq/dataverse/export/InternalExportDataProvider.java b/src/main/java/edu/harvard/iq/dataverse/export/InternalExportDataProvider.java index 70a20a87306..40d5eed09fa 100644 --- a/src/main/java/edu/harvard/iq/dataverse/export/InternalExportDataProvider.java +++ b/src/main/java/edu/harvard/iq/dataverse/export/InternalExportDataProvider.java @@ -152,42 +152,6 @@ public Stream getDatasetFileDetails(FileExportQuery query) { } @Override - public Stream getDatasetFileDetails(FileExportQuery query, PageRequest pageRequest) { - JsonArrayBuilder jab = Json.createArrayBuilder(); - - List fileMetadatas; - DatasetVersionFilesServiceBean datasetVersionFilesService = null; - try { - datasetVersionFilesService = CDI.current().select(DatasetVersionFilesServiceBean.class).get(); - } catch (java.lang.IllegalArgumentException | IllegalStateException ie) { - throw new ExportException("EJB DatasetVersionFilesService is not available; " + ie.getMessage()); - } - - if (datasetVersionFilesService == null) { - throw new ExportException("EJB DatasetVersionFilesService is not available"); - } - - /* - * Defaulting to retrieving tabular/DataVariable-level metadata, for now; - * we will want to honor the related predicates in the long run. - */ - fileMetadatas = datasetVersionFilesService.getTabularDataFileMetadatas(dv, - pageRequest.getOffset(), - pageRequest.getOffset(), - isOnlyPublicMetadataRequested(query)); - - for (FileMetadata fileMetadata : fileMetadatas) { - DataFile dataFile = fileMetadata.getDataFile(); - jab.add(JsonPrinter.jsonDatafileWithDatatableForExport(dataFile, fileMetadata)); - } - - return jab.build().stream().map(jsonValue -> jsonValue.asJsonObject()); - } - - //@Override - // This method, specifically for tabular files only, was in my initial - // implementation of 2.1.0, but later dropped in favor of a more flexible - // getDatasetFileDetails(...) method /** * This new (as of dataverse-spi 2.1.0) method will attempt to retrieve * the requested tabular metadata more efficiently, by calling the @@ -199,7 +163,7 @@ public Stream getDatasetFileDetails(FileExportQuery query, PageReque * getDatasetFileDetails(); * */ - /*public JsonArray getTabularDataDetails(ExportDataContext... context) throws ExportException { + public Stream getDatasetFileDetails(FileExportQuery query, PageRequest pageRequest) { JsonArrayBuilder jab = Json.createArrayBuilder(); List fileMetadatas; @@ -214,27 +178,34 @@ public Stream getDatasetFileDetails(FileExportQuery query, PageReque throw new ExportException("EJB DatasetVersionFilesService is not available"); } - fileMetadatas = datasetVersionFilesService.getTabularDataFileMetadatas(dv, - getLength(context), - getOffset(context), - isOnlyPublicMetadataRequested(context)); - - for (FileMetadata fileMetadata : fileMetadatas) { - DataFile dataFile = fileMetadata.getDataFile(); - jab.add(JsonPrinter.jsonDatafileWithDatatableForExport(dataFile, fileMetadata)); + if (isOnlyTabularMetadataRequested(query) && isDataVariableMetadataRequested(query)) { + + fileMetadatas = datasetVersionFilesService.getTabularDataFileMetadatas(dv, + pageRequest.getLimit(), + pageRequest.getOffset(), + isOnlyPublicMetadataRequested(query)); + + for (FileMetadata fileMetadata : fileMetadatas) { + DataFile dataFile = fileMetadata.getDataFile(); + jab.add(JsonPrinter.jsonDatafileWithDatatableForExport(dataFile, fileMetadata)); + } + + return jab.build().stream().map(jsonValue -> jsonValue.asJsonObject()); + } else { + throw new ExportException("This implementation of getDatasetFileDetails() (paginated version) " + + "only supports request for detailed DataVariable metadata, for tabular DataFiles only"); } - return jab.build(); - }*/ - + } + @Override public Optional getPrerequisiteInputStream(DatasetExportQuery query) { return Optional.ofNullable(is); } - + public void setPrerequisiteInputStream(InputStream prereqStream) { this.is=prereqStream; } - + /** * Only one context object is supported * @param DatasetExportQuery @@ -256,17 +227,50 @@ private boolean isOnlyDatasetLevelMetadataRequested(DatasetExportQuery query) { } /** - * Are we skipping non-public, restricted and/or embargoed files? + * Are we skipping non-public, restricted and embargoed files? * * @param FileExportQuery * @return yes or no */ private boolean isOnlyPublicMetadataRequested(FileExportQuery query) { + return checkForPredicate(query, FileMetadataPredicates.ONLY_PUBLIC_FILES); + } + /** + * Is this metadata request only for ingested tabular files (i.e., files + * with linked DataTable objects) + * + * @param FileExportQuery + * @return yes or no + */ + private boolean isOnlyTabularMetadataRequested(FileExportQuery query) { + return checkForPredicate(query, FileMetadataPredicates.ONLY_TABULAR_FILES); + } + + /** + * Is detailed information about DataVariable objects associated with the + * tabular DataTable requested? + * + * @param FileExportQuery + * @return yes or no + */ + private boolean isDataVariableMetadataRequested(FileExportQuery query) { + return checkForPredicate(query, FileMetadataPredicates.INCLUDE_TABULAR_DATA_VARIABLES); + } + + /** + * Service method for checking a FileExportQuery for a specific predicate specified. + * + * @param query + * @param predicate + * @return + */ + private boolean checkForPredicate(FileExportQuery query, FileMetadataPredicates predicate) { + Set predicates = query.getFilePredicates(); for (FileMetadataPredicates p : predicates) { - if (p.equals(FileMetadataPredicates.ONLY_PUBLIC_FILES)) return true; + if (p.equals(predicate)) return true; } return false; diff --git a/src/main/java/edu/harvard/iq/dataverse/export/ddi/DdiExportUtil.java b/src/main/java/edu/harvard/iq/dataverse/export/ddi/DdiExportUtil.java index 96a5fac2790..b25b0566c75 100644 --- a/src/main/java/edu/harvard/iq/dataverse/export/ddi/DdiExportUtil.java +++ b/src/main/java/edu/harvard/iq/dataverse/export/ddi/DdiExportUtil.java @@ -32,9 +32,12 @@ import edu.harvard.iq.dataverse.util.xml.XmlPrinter; import edu.harvard.iq.dataverse.util.xml.XmlUtil; import edu.harvard.iq.dataverse.util.xml.XmlWriterUtil; -import io.gdcc.spi.export.ExportDataContext; +//import io.gdcc.spi.export.ExportDataContext; +import io.gdcc.spi.export.PageRequest; +import io.gdcc.spi.export.FileExportQuery; import io.gdcc.spi.export.ExportDataProvider; import io.gdcc.spi.export.ExportException; +import io.gdcc.spi.export.FileMetadataPredicates; import java.io.ByteArrayOutputStream; import java.io.IOException; @@ -68,6 +71,7 @@ import javax.xml.transform.stream.StreamSource; import javax.xml.transform.stream.StreamResult; import java.io.InputStream; +import java.util.stream.Stream; public class DdiExportUtil { @@ -85,8 +89,9 @@ public class DdiExportUtil { public static final String NOTE_TYPE_CONTENTTYPE = "DATAVERSE:CONTENTTYPE"; public static final String NOTE_SUBJECT_CONTENTTYPE = "Content/MIME Type"; public static final String CITATION_BLOCK_NAME = "citation"; - public static final int DATATABLES_BATCH_SIZE = 50; - public static final int DATAVARIABLES_BATCH_SIZE = 10000; // todo: review + //public static final int DATATABLES_BATCH_SIZE = 50; + //public static final int DATAVARIABLES_BATCH_SIZE = 10000; // todo: review + public static final int DATAVARIABLES_BATCH_SIZE = 50; //Some tests don't send real PIDs that can be parsed //Use constant empty PID in these cases @@ -1702,14 +1707,25 @@ private static void createDataDscrInBatches(XMLStreamWriter xmlw, List dataTablesThisBatch++; if (varQuantityThisBatch >= DATAVARIABLES_BATCH_SIZE || dataTableCurrent == varQuantityMap.size() - 1) { - JsonArray tabularFileDetails = exportDataProvider.getTabularDataDetails(ExportDataContext.context().withOffset(dataTableStart).withLength(dataTablesThisBatch)); - logger.fine("requested: " + dataTablesThisBatch + " tabular file data entries; retrieved: " + tabularFileDetails.size()); + //JsonArray tabularFileDetails = exportDataProvider.getTabularDataDetails(ExportDataContext.context().withOffset(dataTableStart).withLength(dataTablesThisBatch)); + FileExportQuery exportQuery = FileExportQuery.builder() + .addFilePredicate(FileMetadataPredicates.ONLY_PUBLIC_FILES) + .addFilePredicate(FileMetadataPredicates.ONLY_TABULAR_FILES) + .addFilePredicate(FileMetadataPredicates.INCLUDE_TABULAR_DATA_VARIABLES) + .build(); + PageRequest paginationRequest = PageRequest.of(dataTableStart, dataTablesThisBatch); + Stream tabularFileDetails = exportDataProvider.getDatasetFileDetails(exportQuery, paginationRequest); logger.fine("total number of variables in this batch: " + varQuantityThisBatch); - for (int i = 0; i < tabularFileDetails.size(); i++) { - JsonObject fileJson = tabularFileDetails.getJsonObject(i); + int i = 0; + //for (int i = 0; i < tabularFileDetails.size(); i++) { + Iterator it = tabularFileDetails.iterator(); + while (it.hasNext()) { + JsonObject fileJson = it.next(); if (isFileRestricted(fileJson)) { + // This should not really happen - since we are explicitly + // requesting public files only; but, better safe ... continue; } @@ -1720,8 +1736,17 @@ private static void createDataDscrInBatches(XMLStreamWriter xmlw, List } createVariablesForDataFile(xmlw, fileJson); + // @todo let's confirm here that the number of variables + // we got is what we expected; a mismatch here would + // indicate that the dataset and/or files in it have + // somehow changed since the initial lookup, and therefore + // the export should be aborted. } + i++; } + + logger.fine("requested: " + dataTablesThisBatch + " tabular file data entries; retrieved: " + i); + dataTableStart += dataTablesThisBatch; dataTablesThisBatch = 0; From db203c67e7cc0c003f39aa89b8c186608cd6a700 Mon Sep 17 00:00:00 2001 From: Leonid Andreev Date: Tue, 26 May 2026 18:20:39 -0400 Subject: [PATCH 18/25] extra data consistency logic #11405 --- .../iq/dataverse/export/ddi/DdiExportUtil.java | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/export/ddi/DdiExportUtil.java b/src/main/java/edu/harvard/iq/dataverse/export/ddi/DdiExportUtil.java index b25b0566c75..9e26eb34ae2 100644 --- a/src/main/java/edu/harvard/iq/dataverse/export/ddi/DdiExportUtil.java +++ b/src/main/java/edu/harvard/iq/dataverse/export/ddi/DdiExportUtil.java @@ -89,9 +89,8 @@ public class DdiExportUtil { public static final String NOTE_TYPE_CONTENTTYPE = "DATAVERSE:CONTENTTYPE"; public static final String NOTE_SUBJECT_CONTENTTYPE = "Content/MIME Type"; public static final String CITATION_BLOCK_NAME = "citation"; - //public static final int DATATABLES_BATCH_SIZE = 50; //public static final int DATAVARIABLES_BATCH_SIZE = 10000; // todo: review - public static final int DATAVARIABLES_BATCH_SIZE = 50; + public static final int DATAVARIABLES_BATCH_SIZE = 100; //Some tests don't send real PIDs that can be parsed //Use constant empty PID in these cases @@ -1735,12 +1734,19 @@ private static void createDataDscrInBatches(XMLStreamWriter xmlw, List dataDscrWritten = true; } - createVariablesForDataFile(xmlw, fileJson); - // @todo let's confirm here that the number of variables + int howmany = createVariablesForDataFile(xmlw, fileJson); + // let's confirm here that the number of variables // we got is what we expected; a mismatch here would // indicate that the dataset and/or files in it have // somehow changed since the initial lookup, and therefore // the export should be aborted. + int howmanyExpected = varQuantityMap.get(dataTableStart + i); + if (howmanyExpected != howmany) { + throw new XMLStreamException("Number of variables mismatch. Expected: " + + howmanyExpected + + "; processed from datatable: " + + howmany); + } } i++; } @@ -1768,7 +1774,7 @@ private static void createDataDscrInBatches(XMLStreamWriter xmlw, List } } - private static void createVariablesForDataFile(XMLStreamWriter xmlw, JsonObject fileJson) throws XMLStreamException { + private static int createVariablesForDataFile(XMLStreamWriter xmlw, JsonObject fileJson) throws XMLStreamException { if (fileJson.containsKey("varGroups")) { JsonArray varGroups = fileJson.getJsonArray("varGroups"); for (int j = 0; j < varGroups.size(); j++) { @@ -1784,6 +1790,7 @@ private static void createVariablesForDataFile(XMLStreamWriter xmlw, JsonObject fileJson.getJsonNumber("fileMetadataId").toString()); } } + return vars.size(); } private static void createVarGroupDDI(XMLStreamWriter xmlw, JsonObject varGrp) throws XMLStreamException { From 515e0ea21fa317ad2f54aac790f3d082fc2695df Mon Sep 17 00:00:00 2001 From: Leonid Andreev Date: Wed, 27 May 2026 13:41:37 -0400 Subject: [PATCH 19/25] Added further data consistency checks to avoid any ambiguity in how the data objects are retrieved. #11405 --- .../java/edu/harvard/iq/dataverse/DatasetVersion.java | 2 +- .../iq/dataverse/DatasetVersionFilesServiceBean.java | 8 +++++++- .../iq/dataverse/export/InternalExportDataProvider.java | 1 - .../harvard/iq/dataverse/export/ddi/DdiExportUtil.java | 8 ++++---- 4 files changed, 12 insertions(+), 7 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetVersion.java b/src/main/java/edu/harvard/iq/dataverse/DatasetVersion.java index cfd4c886bf3..4265c381f78 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DatasetVersion.java +++ b/src/main/java/edu/harvard/iq/dataverse/DatasetVersion.java @@ -167,7 +167,7 @@ public enum VersionState { private Dataset dataset; @OneToMany(mappedBy = "datasetVersion", cascade = {CascadeType.REMOVE, CascadeType.MERGE, CascadeType.PERSIST}) - @OrderBy("label") // this is not our preferred ordering, which is with the AlphaNumericComparator, but does allow the files to be grouped by category + @OrderBy("label, id") // this is not our preferred ordering, which is with the AlphaNumericComparator, but does allow the files to be grouped by category; adding id, to avoid ambiguity when there are duplicate filenames. (L.A., 05-2026) private List fileMetadatas = new ArrayList(); @OneToOne(cascade = {CascadeType.MERGE, CascadeType.PERSIST, CascadeType.REMOVE}, orphanRemoval=true) diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetVersionFilesServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/DatasetVersionFilesServiceBean.java index 4687d72bdbc..b4b013908a4 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DatasetVersionFilesServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/DatasetVersionFilesServiceBean.java @@ -214,10 +214,16 @@ public List getTabularDataFileMetadatas(DatasetVersion datasetVers fileMetadataRoot)); } + List orderList = new ArrayList<>(); + + // Ordering the resulting fileMetadatas by label AND id, to avoid any ambiguity when there are duplicate filenames in the version + orderList.add(criteriaBuilder.asc(fileMetadataRoot.get("label"))); + orderList.add(criteriaBuilder.asc(fileMetadataRoot.get("id"))); + criteriaQuery .select(fileMetadataRoot) .where(combinedPredicate) - .orderBy(criteriaBuilder.asc(fileMetadataRoot.get("label"))); + .orderBy(orderList); TypedQuery typedQuery = em.createQuery(criteriaQuery); if (limit != null) { diff --git a/src/main/java/edu/harvard/iq/dataverse/export/InternalExportDataProvider.java b/src/main/java/edu/harvard/iq/dataverse/export/InternalExportDataProvider.java index 40d5eed09fa..401e66b06fc 100644 --- a/src/main/java/edu/harvard/iq/dataverse/export/InternalExportDataProvider.java +++ b/src/main/java/edu/harvard/iq/dataverse/export/InternalExportDataProvider.java @@ -20,7 +20,6 @@ import edu.harvard.iq.dataverse.util.json.JsonPrinter; import edu.harvard.iq.dataverse.util.json.JsonUtil; import io.gdcc.spi.export.ExportException; -//import io.gdcc.spi.export.ExportDataContext; import io.gdcc.spi.export.DatasetExportQuery; import io.gdcc.spi.export.DatasetMetadataPredicates; import io.gdcc.spi.export.FileExportQuery; diff --git a/src/main/java/edu/harvard/iq/dataverse/export/ddi/DdiExportUtil.java b/src/main/java/edu/harvard/iq/dataverse/export/ddi/DdiExportUtil.java index 9e26eb34ae2..db4eac8678d 100644 --- a/src/main/java/edu/harvard/iq/dataverse/export/ddi/DdiExportUtil.java +++ b/src/main/java/edu/harvard/iq/dataverse/export/ddi/DdiExportUtil.java @@ -1716,7 +1716,7 @@ private static void createDataDscrInBatches(XMLStreamWriter xmlw, List Stream tabularFileDetails = exportDataProvider.getDatasetFileDetails(exportQuery, paginationRequest); logger.fine("total number of variables in this batch: " + varQuantityThisBatch); - int i = 0; + int count = 0; //for (int i = 0; i < tabularFileDetails.size(); i++) { Iterator it = tabularFileDetails.iterator(); while (it.hasNext()) { @@ -1740,7 +1740,7 @@ private static void createDataDscrInBatches(XMLStreamWriter xmlw, List // indicate that the dataset and/or files in it have // somehow changed since the initial lookup, and therefore // the export should be aborted. - int howmanyExpected = varQuantityMap.get(dataTableStart + i); + int howmanyExpected = varQuantityMap.get(dataTableStart + count); if (howmanyExpected != howmany) { throw new XMLStreamException("Number of variables mismatch. Expected: " + howmanyExpected @@ -1748,10 +1748,10 @@ private static void createDataDscrInBatches(XMLStreamWriter xmlw, List + howmany); } } - i++; + count++; } - logger.fine("requested: " + dataTablesThisBatch + " tabular file data entries; retrieved: " + i); + logger.fine("requested: " + dataTablesThisBatch + " tabular file data entries; retrieved: " + count); dataTableStart += dataTablesThisBatch; From cd1a9ed3888ba4dc50846bbc56f5cf0b1b292858 Mon Sep 17 00:00:00 2001 From: Leonid Andreev Date: Thu, 18 Jun 2026 13:58:07 -0400 Subject: [PATCH 20/25] code changes that were needed in develop in order to sync up the export branch. #11405 --- .../harvard/iq/dataverse/api/Datasets.java | 2 +- .../export/CroissantExporterSlimTest.java | 199 +++++++++++++++++- .../export/CroissantExporterTest.java | 199 +++++++++++++++++- .../dataverse/util/json/JsonPrinterTest.java | 2 +- 4 files changed, 398 insertions(+), 4 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java index a00130de31a..e321eddc693 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java @@ -539,7 +539,7 @@ public Response getVersion(@Context ContainerRequestContext crc, } JsonObjectBuilder jsonBuilder = json(requestedDatasetVersion, null, includeFiles, - returnOwners, includeMetadataBlocks, ignoreSettingExcludeEmailFromExport); + returnOwners, includeMetadataBlocks, false, ignoreSettingExcludeEmailFromExport); return ok(jsonBuilder); }, getRequestUser(crc)); diff --git a/src/test/java/edu/harvard/iq/dataverse/export/CroissantExporterSlimTest.java b/src/test/java/edu/harvard/iq/dataverse/export/CroissantExporterSlimTest.java index fcbc9611818..f06354831d5 100644 --- a/src/test/java/edu/harvard/iq/dataverse/export/CroissantExporterSlimTest.java +++ b/src/test/java/edu/harvard/iq/dataverse/export/CroissantExporterSlimTest.java @@ -1,8 +1,11 @@ package edu.harvard.iq.dataverse.export; +import io.gdcc.spi.export.DatasetExportQuery; import static org.junit.jupiter.api.Assertions.*; import io.gdcc.spi.export.ExportDataProvider; +import io.gdcc.spi.export.FileExportQuery; +import io.gdcc.spi.export.PageRequest; import jakarta.json.Json; import jakarta.json.JsonArray; import jakarta.json.JsonObject; @@ -23,9 +26,11 @@ import java.nio.file.Paths; import java.util.HashMap; import java.util.Map; +import java.util.stream.Stream; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; import org.skyscreamer.jsonassert.JSONAssert; +import org.w3c.dom.Document; public class CroissantExporterSlimTest { @@ -109,6 +114,38 @@ public String getDataCiteXml() { return null; } } + + // extra methods in the new-and-improved ExportDataProvider interface + // (that we do not need for our current purposes) + + public JsonObject getDatasetJson(DatasetExportQuery q) { + return null; + } + + @Override + public JsonObject getDatasetORE(DatasetExportQuery q) { + return null; + } + + @Override + public Stream getDatasetFileDetails(FileExportQuery q) { + return null; + } + + @Override + public Stream getDatasetFileDetails(FileExportQuery q, PageRequest p) { + return null; + } + + @Override + public JsonObject getDatasetSchemaDotOrg(DatasetExportQuery q) { + return null; + } + + @Override + public Document getDataCiteXml(DatasetExportQuery q) { + return null; + } }; outputStreamMax = new ByteArrayOutputStream(); @@ -173,6 +210,38 @@ public String getDataCiteXml() { return null; } } + + // extra methods in the new-and-improved ExportDataProvider interface + // (that we do not need for our current purposes) + + public JsonObject getDatasetJson(DatasetExportQuery q) { + return null; + } + + @Override + public JsonObject getDatasetORE(DatasetExportQuery q) { + return null; + } + + @Override + public Stream getDatasetFileDetails(FileExportQuery q) { + return null; + } + + @Override + public Stream getDatasetFileDetails(FileExportQuery q, PageRequest p) { + return null; + } + + @Override + public JsonObject getDatasetSchemaDotOrg(DatasetExportQuery q) { + return null; + } + + @Override + public Document getDataCiteXml(DatasetExportQuery q) { + return null; + } }; outputStreamCars = new ByteArrayOutputStream(); @@ -237,6 +306,38 @@ public String getDataCiteXml() { return null; } } + + // extra methods in the new-and-improved ExportDataProvider interface + // (that we do not need for our current purposes) + + public JsonObject getDatasetJson(DatasetExportQuery q) { + return null; + } + + @Override + public JsonObject getDatasetORE(DatasetExportQuery q) { + return null; + } + + @Override + public Stream getDatasetFileDetails(FileExportQuery q) { + return null; + } + + @Override + public Stream getDatasetFileDetails(FileExportQuery q, PageRequest p) { + return null; + } + + @Override + public JsonObject getDatasetSchemaDotOrg(DatasetExportQuery q) { + return null; + } + + @Override + public Document getDataCiteXml(DatasetExportQuery q) { + return null; + } }; outputStreamRestricted = new ByteArrayOutputStream(); @@ -301,6 +402,38 @@ public String getDataCiteXml() { return null; } } + + // extra methods in the new-and-improved ExportDataProvider interface + // (that we do not need for our current purposes) + + public JsonObject getDatasetJson(DatasetExportQuery q) { + return null; + } + + @Override + public JsonObject getDatasetORE(DatasetExportQuery q) { + return null; + } + + @Override + public Stream getDatasetFileDetails(FileExportQuery q) { + return null; + } + + @Override + public Stream getDatasetFileDetails(FileExportQuery q, PageRequest p) { + return null; + } + + @Override + public JsonObject getDatasetSchemaDotOrg(DatasetExportQuery q) { + return null; + } + + @Override + public Document getDataCiteXml(DatasetExportQuery q) { + return null; + } }; outputStreamJunk = new ByteArrayOutputStream(); @@ -365,6 +498,38 @@ public String getDataCiteXml() { return null; } } + + // extra methods in the new-and-improved ExportDataProvider interface + // (that we do not need for our current purposes) + + public JsonObject getDatasetJson(DatasetExportQuery q) { + return null; + } + + @Override + public JsonObject getDatasetORE(DatasetExportQuery q) { + return null; + } + + @Override + public Stream getDatasetFileDetails(FileExportQuery q) { + return null; + } + + @Override + public Stream getDatasetFileDetails(FileExportQuery q, PageRequest p) { + return null; + } + + @Override + public JsonObject getDatasetSchemaDotOrg(DatasetExportQuery q) { + return null; + } + + @Override + public Document getDataCiteXml(DatasetExportQuery q) { + return null; + } }; outputStreamDraft = new ByteArrayOutputStream(); @@ -429,7 +594,39 @@ public String getDataCiteXml() { return null; } } - }; + + // extra methods in the new-and-improved ExportDataProvider interface + // (that we do not need for our current purposes) + + public JsonObject getDatasetJson(DatasetExportQuery q) { + return null; + } + + @Override + public JsonObject getDatasetORE(DatasetExportQuery q) { + return null; + } + + @Override + public Stream getDatasetFileDetails(FileExportQuery q) { + return null; + } + + @Override + public Stream getDatasetFileDetails(FileExportQuery q, PageRequest p) { + return null; + } + + @Override + public JsonObject getDatasetSchemaDotOrg(DatasetExportQuery q) { + return null; + } + + @Override + public Document getDataCiteXml(DatasetExportQuery q) { + return null; + } + }; } @Test diff --git a/src/test/java/edu/harvard/iq/dataverse/export/CroissantExporterTest.java b/src/test/java/edu/harvard/iq/dataverse/export/CroissantExporterTest.java index 6c6da792d4e..826441dfb70 100644 --- a/src/test/java/edu/harvard/iq/dataverse/export/CroissantExporterTest.java +++ b/src/test/java/edu/harvard/iq/dataverse/export/CroissantExporterTest.java @@ -3,6 +3,9 @@ import static org.junit.jupiter.api.Assertions.*; import io.gdcc.spi.export.ExportDataProvider; +import io.gdcc.spi.export.DatasetExportQuery; +import io.gdcc.spi.export.FileExportQuery; +import io.gdcc.spi.export.PageRequest; import jakarta.json.Json; import jakarta.json.JsonArray; import jakarta.json.JsonObject; @@ -23,9 +26,11 @@ import java.nio.file.Paths; import java.util.HashMap; import java.util.Map; +import java.util.stream.Stream; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; import org.skyscreamer.jsonassert.JSONAssert; +import org.w3c.dom.Document; public class CroissantExporterTest { @@ -109,6 +114,38 @@ public String getDataCiteXml() { return null; } } + + // extra methods in the new-and-improved ExportDataProvider interface + // (that we do not need for our current purposes) + + public JsonObject getDatasetJson(DatasetExportQuery q) { + return null; + } + + @Override + public JsonObject getDatasetORE(DatasetExportQuery q) { + return null; + } + + @Override + public Stream getDatasetFileDetails(FileExportQuery q) { + return null; + } + + @Override + public Stream getDatasetFileDetails(FileExportQuery q, PageRequest p) { + return null; + } + + @Override + public JsonObject getDatasetSchemaDotOrg(DatasetExportQuery q) { + return null; + } + + @Override + public Document getDataCiteXml(DatasetExportQuery q) { + return null; + } }; outputStreamMax = new ByteArrayOutputStream(); @@ -173,6 +210,38 @@ public String getDataCiteXml() { return null; } } + + // extra methods in the new-and-improved ExportDataProvider interface + // (that we do not need for our current purposes) + + public JsonObject getDatasetJson(DatasetExportQuery q) { + return null; + } + + @Override + public JsonObject getDatasetORE(DatasetExportQuery q) { + return null; + } + + @Override + public Stream getDatasetFileDetails(FileExportQuery q) { + return null; + } + + @Override + public Stream getDatasetFileDetails(FileExportQuery q, PageRequest p) { + return null; + } + + @Override + public JsonObject getDatasetSchemaDotOrg(DatasetExportQuery q) { + return null; + } + + @Override + public Document getDataCiteXml(DatasetExportQuery q) { + return null; + } }; outputStreamCars = new ByteArrayOutputStream(); @@ -237,6 +306,38 @@ public String getDataCiteXml() { return null; } } + + // extra methods in the new-and-improved ExportDataProvider interface + // (that we do not need for our current purposes) + + public JsonObject getDatasetJson(DatasetExportQuery q) { + return null; + } + + @Override + public JsonObject getDatasetORE(DatasetExportQuery q) { + return null; + } + + @Override + public Stream getDatasetFileDetails(FileExportQuery q) { + return null; + } + + @Override + public Stream getDatasetFileDetails(FileExportQuery q, PageRequest p) { + return null; + } + + @Override + public JsonObject getDatasetSchemaDotOrg(DatasetExportQuery q) { + return null; + } + + @Override + public Document getDataCiteXml(DatasetExportQuery q) { + return null; + } }; outputStreamRestricted = new ByteArrayOutputStream(); @@ -301,6 +402,38 @@ public String getDataCiteXml() { return null; } } + + // extra methods in the new-and-improved ExportDataProvider interface + // (that we do not need for our current purposes) + + public JsonObject getDatasetJson(DatasetExportQuery q) { + return null; + } + + @Override + public JsonObject getDatasetORE(DatasetExportQuery q) { + return null; + } + + @Override + public Stream getDatasetFileDetails(FileExportQuery q) { + return null; + } + + @Override + public Stream getDatasetFileDetails(FileExportQuery q, PageRequest p) { + return null; + } + + @Override + public JsonObject getDatasetSchemaDotOrg(DatasetExportQuery q) { + return null; + } + + @Override + public Document getDataCiteXml(DatasetExportQuery q) { + return null; + } }; outputStreamJunk = new ByteArrayOutputStream(); @@ -365,6 +498,38 @@ public String getDataCiteXml() { return null; } } + + // extra methods in the new-and-improved ExportDataProvider interface + // (that we do not need for our current purposes) + + public JsonObject getDatasetJson(DatasetExportQuery q) { + return null; + } + + @Override + public JsonObject getDatasetORE(DatasetExportQuery q) { + return null; + } + + @Override + public Stream getDatasetFileDetails(FileExportQuery q) { + return null; + } + + @Override + public Stream getDatasetFileDetails(FileExportQuery q, PageRequest p) { + return null; + } + + @Override + public JsonObject getDatasetSchemaDotOrg(DatasetExportQuery q) { + return null; + } + + @Override + public Document getDataCiteXml(DatasetExportQuery q) { + return null; + } }; outputStreamDraft = new ByteArrayOutputStream(); @@ -429,7 +594,39 @@ public String getDataCiteXml() { return null; } } - }; + + // extra methods in the new-and-improved ExportDataProvider interface + // (that we do not need for our current purposes) + + public JsonObject getDatasetJson(DatasetExportQuery q) { + return null; + } + + @Override + public JsonObject getDatasetORE(DatasetExportQuery q) { + return null; + } + + @Override + public Stream getDatasetFileDetails(FileExportQuery q) { + return null; + } + + @Override + public Stream getDatasetFileDetails(FileExportQuery q, PageRequest p) { + return null; + } + + @Override + public JsonObject getDatasetSchemaDotOrg(DatasetExportQuery q) { + return null; + } + + @Override + public Document getDataCiteXml(DatasetExportQuery q) { + return null; + } + }; } @Test diff --git a/src/test/java/edu/harvard/iq/dataverse/util/json/JsonPrinterTest.java b/src/test/java/edu/harvard/iq/dataverse/util/json/JsonPrinterTest.java index 40d6e02f9c2..95a3fbef20e 100644 --- a/src/test/java/edu/harvard/iq/dataverse/util/json/JsonPrinterTest.java +++ b/src/test/java/edu/harvard/iq/dataverse/util/json/JsonPrinterTest.java @@ -555,7 +555,7 @@ public void testDatasetWithGuestbook() { dataset.setGuestbook(guestbook); // verify that the guestbook id is in the dataset response - var jsob = JsonPrinter.json(dataset.getLatestVersion(), null, false, false, false, false).build(); + var jsob = JsonPrinter.json(dataset.getLatestVersion(), null, false, false, false, false, false).build(); System.out.println(jsob); var gbID = jsob.getInt("guestbookId"); assertEquals(1, gbID); From 22b2d39dbd9a58424d0643059f416dca7ae8a2d5 Mon Sep 17 00:00:00 2001 From: Leonid Andreev Date: Sun, 19 Jul 2026 22:39:38 -0400 Subject: [PATCH 21/25] New tests added, some cleanup (this is still in the "experiment" sub-branch). #11405 --- .../DatasetVersionFilesServiceBean.java | 44 ++- .../export/InternalExportDataProvider.java | 88 +++-- .../dataverse/export/ddi/DdiExportUtil.java | 180 +++++---- .../dataverse/export/TabularDataExportIT.java | 359 ++++++++++++++++++ 4 files changed, 533 insertions(+), 138 deletions(-) create mode 100644 src/test/java/edu/harvard/iq/dataverse/export/TabularDataExportIT.java diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetVersionFilesServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/DatasetVersionFilesServiceBean.java index b4b013908a4..74c3e833753 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DatasetVersionFilesServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/DatasetVersionFilesServiceBean.java @@ -180,51 +180,51 @@ public List getFileMetadatas(DatasetVersion datasetVersion, Intege } return typedQuery.getResultList(); } - + /** * Similar to the above, but dedicated for retrieving FileMetadatas of only * tabular datafiles in the specified DatasetVersion. Used in the metadata - * export subsystem. + * export subsystem. * * @param datasetVersion the DatasetVersion to access * @param limit for pagination, can be null * @param offset for pagination, can be null - * @param publicFilesOnly skip restricted, embargoed etc. files + * @param publicFilesOnly skip restricted, embargoed etc. files * @return a FileMetadata list from the specified DatasetVersion */ public List getTabularDataFileMetadatas(DatasetVersion datasetVersion, Integer limit, Integer offset, boolean publicFilesOnly) { CriteriaBuilder criteriaBuilder = em.getCriteriaBuilder(); CriteriaQuery criteriaQuery = criteriaBuilder.createQuery(FileMetadata.class); - + Root fileMetadataRoot = criteriaQuery.from(FileMetadata.class); Predicate basePredicate = criteriaBuilder.equal(fileMetadataRoot.get("datasetVersion").get("id"), datasetVersion.getId()); - - Root dataTableRoot = criteriaQuery.from(DataTable.class); + + Root dataTableRoot = criteriaQuery.from(DataTable.class); Predicate tabularPredicate = criteriaBuilder.equal(dataTableRoot.get("dataFile"), fileMetadataRoot.get("dataFile")); - + Predicate combinedPredicate; - + if (publicFilesOnly) { combinedPredicate = criteriaBuilder.and(basePredicate, tabularPredicate); } else { - combinedPredicate = criteriaBuilder.and(basePredicate, - tabularPredicate, - createSearchCriteriaAccessStatusPredicate(FileSearchCriteria.FileAccessStatus.Public, - criteriaBuilder, + combinedPredicate = criteriaBuilder.and(basePredicate, + tabularPredicate, + createSearchCriteriaAccessStatusPredicate(FileSearchCriteria.FileAccessStatus.Public, + criteriaBuilder, fileMetadataRoot)); } - + List orderList = new ArrayList<>(); - + // Ordering the resulting fileMetadatas by label AND id, to avoid any ambiguity when there are duplicate filenames in the version orderList.add(criteriaBuilder.asc(fileMetadataRoot.get("label"))); orderList.add(criteriaBuilder.asc(fileMetadataRoot.get("id"))); - + criteriaQuery .select(fileMetadataRoot) .where(combinedPredicate) .orderBy(orderList); - + TypedQuery typedQuery = em.createQuery(criteriaQuery); if (limit != null) { typedQuery.setMaxResults(limit); @@ -232,7 +232,7 @@ public List getTabularDataFileMetadatas(DatasetVersion datasetVers if (offset != null) { typedQuery.setFirstResult(offset); } - + return typedQuery.getResultList(); } @@ -280,6 +280,16 @@ public boolean isDataFilePresentInDatasetVersion(DatasetVersion datasetVersion, return count != null && count > 0; } + /** + * This is strictly for use in IT tests! + * + * @param em EntityManager, such as dataverse.util.testing.performance.JpaEntityManagerService.createEntityManager() + * + */ + public void injectEntityManager(EntityManager em) { + this.em = em; + } + private void addAccessStatusCountToTotal(DatasetVersion datasetVersion, Map totalCounts, FileAccessStatus dataFileAccessStatus, FileSearchCriteria searchCriteria) { long fileMetadataCount = getFileMetadataCountByAccessStatus(datasetVersion, dataFileAccessStatus, searchCriteria); if (fileMetadataCount > 0) { diff --git a/src/main/java/edu/harvard/iq/dataverse/export/InternalExportDataProvider.java b/src/main/java/edu/harvard/iq/dataverse/export/InternalExportDataProvider.java index 401e66b06fc..d64af0a79d9 100644 --- a/src/main/java/edu/harvard/iq/dataverse/export/InternalExportDataProvider.java +++ b/src/main/java/edu/harvard/iq/dataverse/export/InternalExportDataProvider.java @@ -39,22 +39,23 @@ /** * Provides all data necessary to create an export - * + * */ public class InternalExportDataProvider implements ExportDataProvider { private final DatasetVersion dv; private JsonObject jsonRepresentation = null; - private JsonObject jsonRepresentationNoFiles = null; + private JsonObject jsonRepresentationNoFiles = null; private JsonObject schemaDotOrgRepresentation = null; private JsonObject oreRepresentation = null; private JsonArray fileAndDataDetails = null; private InputStream is = null; + private DatasetVersionFilesServiceBean datasetVersionFilesService = null; InternalExportDataProvider(DatasetVersion dv) { this.dv = dv; } - + InternalExportDataProvider(DatasetVersion dv, InputStream is) { this.dv = dv; this.is=is; @@ -63,24 +64,24 @@ public class InternalExportDataProvider implements ExportDataProvider { @Override public JsonObject getDatasetJson(DatasetExportQuery query) { if (isOnlyDatasetLevelMetadataRequested(query)) { - // If we already have the "full" Json representation (with files) - // generated, should we return it (potentially moving MUCH more json - // than the client needs, or spend extra cycles generating the short - // form from scratch? - I'm choosing to go with latter. + // If we already have the "full" Json representation (with files) + // generated, should we return it (potentially moving MUCH more json + // than the client needs, or spend extra cycles generating the short + // form from scratch? - I'm choosing to go with latter. if (jsonRepresentationNoFiles == null) { final JsonObjectBuilder datasetAsJsonBuilder = JsonPrinter.datasetAsJsonForDTO(dv, false); jsonRepresentationNoFiles = datasetAsJsonBuilder.build(); } return jsonRepresentationNoFiles; } - + if (jsonRepresentation == null) { final JsonObjectBuilder datasetAsJsonBuilder = JsonPrinter.datasetAsJsonForDTO(dv); jsonRepresentation = datasetAsJsonBuilder.build(); } return jsonRepresentation; } - + @Override public JsonObject getDatasetSchemaDotOrg(DatasetExportQuery query) { if (schemaDotOrgRepresentation == null) { @@ -100,30 +101,30 @@ public JsonObject getDatasetORE(DatasetExportQuery query) { @Override public String getDataCiteXml() { - // @todo Is this the best way to obtain the metadata? - as opposed to - // going through the normal Export framework? (it may be, if it needs - // to be version-specific - ?) + // @todo Is this the best way to obtain the metadata? - as opposed to + // going through the normal Export framework? (it may be, if it needs + // to be version-specific - ?) return DOIDataCiteRegisterService.getMetadataFromDvObject( dv.getDataset().getGlobalId().asString(), new DataCitation(dv).getDataCiteMetadata(), dv.getDataset()); } - + @Override public Document getDataCiteXml(DatasetExportQuery query) { // Note that the query parameter is ignored, for now String dataciteXmlString = getDataCiteXml(); DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance(); - + try { DocumentBuilder builder = factory.newDocumentBuilder(); - + return builder.parse(new InputSource(new StringReader(dataciteXmlString))); } catch (ParserConfigurationException | SAXException | IOException px) { return null; - } + } } - + @Override public JsonArray getDatasetFileDetails() { if (fileAndDataDetails == null) { @@ -136,7 +137,7 @@ public JsonArray getDatasetFileDetails() { } return fileAndDataDetails; } - + @Override public Stream getDatasetFileDetails(FileExportQuery query) { if (fileAndDataDetails == null) { @@ -149,28 +150,29 @@ public Stream getDatasetFileDetails(FileExportQuery query) { } return fileAndDataDetails.stream().map(jsonValue -> jsonValue.asJsonObject()); } - + @Override /** * This new (as of dataverse-spi 2.1.0) method will attempt to retrieve * the requested tabular metadata more efficiently, by calling the - * DatasetVersionFilesServiceBean method directly. Which, among other things, + * DatasetVersionFilesServiceBean method directly. Which, among other things, * allows to retrieve this information in batches. If for whatever reason * that fails - if, for example, the EJB is not available in this context, * we will throw an ExportException, giving the exporter a chance to try and * retrieve this information using the traditional all-at-once method via * getDatasetFileDetails(); - * + * */ public Stream getDatasetFileDetails(FileExportQuery query, PageRequest pageRequest) { JsonArrayBuilder jab = Json.createArrayBuilder(); List fileMetadatas; - DatasetVersionFilesServiceBean datasetVersionFilesService = null; - try { - datasetVersionFilesService = CDI.current().select(DatasetVersionFilesServiceBean.class).get(); - } catch (java.lang.IllegalArgumentException | IllegalStateException ie) { - throw new ExportException("EJB DatasetVersionFilesService is not available; " + ie.getMessage()); + if (datasetVersionFilesService == null) { + try { + datasetVersionFilesService = CDI.current().select(DatasetVersionFilesServiceBean.class).get(); + } catch (java.lang.IllegalArgumentException | IllegalStateException ie) { + throw new ExportException("EJB DatasetVersionFilesService is not available; " + ie.getMessage()); + } } if (datasetVersionFilesService == null) { @@ -206,25 +208,25 @@ public void setPrerequisiteInputStream(InputStream prereqStream) { } /** - * Only one context object is supported + * Only one context object is supported * @param DatasetExportQuery - * @return + * @return */ private boolean isOnlyDatasetLevelMetadataRequested(DatasetExportQuery query) { Set predicates = query.getDatasetPredicates(); - + for (DatasetMetadataPredicates p : predicates) { // @todo This is pending on adding a dedicated DATASET_LEVEL_ONLY predicate // to the enum //if (p.equals(DatasetMetadataPredicates.DATASET_LEVEL_ONLY)) return true; } - // The default assumption is we pack both the Dataset, and the File-level + // The default assumption is we pack both the Dataset, and the File-level // metadata in the Json return false; } - + /** * Are we skipping non-public, restricted and embargoed files? * @@ -236,7 +238,7 @@ private boolean isOnlyPublicMetadataRequested(FileExportQuery query) { } /** - * Is this metadata request only for ingested tabular files (i.e., files + * Is this metadata request only for ingested tabular files (i.e., files * with linked DataTable objects) * * @param FileExportQuery @@ -247,8 +249,8 @@ private boolean isOnlyTabularMetadataRequested(FileExportQuery query) { } /** - * Is detailed information about DataVariable objects associated with the - * tabular DataTable requested? + * Is detailed information about DataVariable objects associated with the + * tabular DataTable requested? * * @param FileExportQuery * @return yes or no @@ -256,22 +258,30 @@ private boolean isOnlyTabularMetadataRequested(FileExportQuery query) { private boolean isDataVariableMetadataRequested(FileExportQuery query) { return checkForPredicate(query, FileMetadataPredicates.INCLUDE_TABULAR_DATA_VARIABLES); } - + /** * Service method for checking a FileExportQuery for a specific predicate specified. - * + * * @param query * @param predicate - * @return + * @return */ private boolean checkForPredicate(FileExportQuery query, FileMetadataPredicates predicate) { - + Set predicates = query.getFilePredicates(); - + for (FileMetadataPredicates p : predicates) { if (p.equals(predicate)) return true; } return false; } + + /** + * This method is exclusively for use in IT tests + * @param versionFilesService + */ + public void injectVersionFilesService(DatasetVersionFilesServiceBean versionFilesService) { + datasetVersionFilesService = versionFilesService; + } } diff --git a/src/main/java/edu/harvard/iq/dataverse/export/ddi/DdiExportUtil.java b/src/main/java/edu/harvard/iq/dataverse/export/ddi/DdiExportUtil.java index db4eac8678d..c409c45417a 100644 --- a/src/main/java/edu/harvard/iq/dataverse/export/ddi/DdiExportUtil.java +++ b/src/main/java/edu/harvard/iq/dataverse/export/ddi/DdiExportUtil.java @@ -83,14 +83,13 @@ public class DdiExportUtil { public static final String LEVEL_DV = "dv"; - + static SettingsServiceBean settingsService; - + public static final String NOTE_TYPE_CONTENTTYPE = "DATAVERSE:CONTENTTYPE"; public static final String NOTE_SUBJECT_CONTENTTYPE = "Content/MIME Type"; public static final String CITATION_BLOCK_NAME = "citation"; - //public static final int DATAVARIABLES_BATCH_SIZE = 10000; // todo: review - public static final int DATAVARIABLES_BATCH_SIZE = 100; + public static final int DATAVARIABLES_BATCH_SIZE = 10000; //Some tests don't send real PIDs that can be parsed //Use constant empty PID in these cases @@ -106,7 +105,7 @@ public static String datasetDtoAsJson2ddi(String datasetDtoAsJson) { return null; } } - + // "short" ddi, without the "" and "/" sections: public static void datasetJson2ddi(JsonObject datasetDtoAsJson, OutputStream outputStream) throws XMLStreamException { logger.fine(JsonUtil.prettyPrint(datasetDtoAsJson.toString())); @@ -114,14 +113,14 @@ public static void datasetJson2ddi(JsonObject datasetDtoAsJson, OutputStream out DatasetDTO datasetDto = gson.fromJson(datasetDtoAsJson.toString(), DatasetDTO.class); dtoddi(datasetDto, outputStream); } - + private static String dto2ddi(DatasetDTO datasetDto) throws XMLStreamException { OutputStream outputStream = new ByteArrayOutputStream(); dtoddi(datasetDto, outputStream); String xml = outputStream.toString(); return XmlPrinter.prettyPrintXml(xml); } - + private static void dtoddi(DatasetDTO datasetDto, OutputStream outputStream) throws XMLStreamException { XMLStreamWriter xmlw = null; try { @@ -156,7 +155,7 @@ private static void dtoddi(DatasetDTO datasetDto, OutputStream outputStream) thr } } - + // "full" ddi, with the the "" and "/" sections: public static void datasetJson2ddi(JsonObject datasetDtoAsJson, ExportDataProvider dataProvider, OutputStream outputStream) throws XMLStreamException { logger.fine(JsonUtil.prettyPrint(datasetDtoAsJson.toString())); @@ -270,25 +269,25 @@ private static void createStdyDscr(XMLStreamWriter xmlw, DatasetDTO datasetDto) } else if ("doi".equals(persistentAgency)) { persistentAgency = "DOI"; } - + //docDesc Block writeDocDescElement (xmlw, datasetDto); //stdyDesc Block xmlw.writeStartElement("stdyDscr"); xmlw.writeStartElement("citation"); xmlw.writeStartElement("titlStmt"); - + XmlWriterUtil.writeFullElement(xmlw, "titl", XmlWriterUtil.dto2Primitive(version, DatasetFieldConstant.title), datasetDto.getMetadataLanguage()); XmlWriterUtil.writeFullElement(xmlw, "subTitl", XmlWriterUtil.dto2Primitive(version, DatasetFieldConstant.subTitle)); FieldDTO altField = dto2FieldDTO( version, DatasetFieldConstant.alternativeTitle, "citation" ); if (altField != null) { writeMultipleElement(xmlw, "altTitl", altField, datasetDto.getMetadataLanguage()); } - + xmlw.writeStartElement("IDNo"); XmlWriterUtil.writeAttribute(xmlw, "agency", persistentAgency); - - + + xmlw.writeCharacters(pidString); xmlw.writeEndElement(); // IDNo writeOtherIdElement(xmlw, version); @@ -296,7 +295,7 @@ private static void createStdyDscr(XMLStreamWriter xmlw, DatasetDTO datasetDto) writeAuthorsElement(xmlw, version); writeProducersElement(xmlw, version); - + xmlw.writeStartElement("distStmt"); //The default is to add Dataverse Repository as a distributor. The excludeinstallationifset setting turns that off if there is a distributor defined in the metadata boolean distributorSet=false; @@ -306,7 +305,7 @@ private static void createStdyDscr(XMLStreamWriter xmlw, DatasetDTO datasetDto) distributorSet=true; } } - + boolean excludeRepository = settingsService.isTrueForKey(SettingsServiceBean.Key.ExportInstallationAsDistributorOnlyWhenNotSet, false); if (!StringUtils.isEmpty(datasetDto.getPublisher()) && !(excludeRepository && distributorSet)) { xmlw.writeStartElement("distrbtr"); @@ -328,14 +327,14 @@ private static void createStdyDscr(XMLStreamWriter xmlw, DatasetDTO datasetDto) xmlw.writeStartElement("holdings"); XmlWriterUtil.writeAttribute(xmlw, "URI", pidUri); xmlw.writeEndElement(); //holdings - + xmlw.writeEndElement(); // citation //End Citation Block - + //Start Study Info Block // Study Info xmlw.writeStartElement("stdyInfo"); - + writeSubjectElement(xmlw, version, datasetDto.getMetadataLanguage()); //Subject and Keywords writeAbstractElement(xmlw, version, datasetDto.getMetadataLanguage()); // Description writeSummaryDescriptionElement(xmlw, version, datasetDto.getMetadataLanguage()); @@ -348,7 +347,7 @@ private static void createStdyDscr(XMLStreamWriter xmlw, DatasetDTO datasetDto) writeOtherStudyMaterial(xmlw , version); XmlWriterUtil.writeFullElement(xmlw, "notes", XmlWriterUtil.dto2Primitive(version, DatasetFieldConstant.datasetLevelErrorNotes)); - + xmlw.writeEndElement(); // stdyDscr } @@ -382,7 +381,7 @@ private static void writeOtherStudyMaterial(XMLStreamWriter xmlw , DatasetVersio */ private static void writeDataAccess(XMLStreamWriter xmlw , DatasetVersionDTO version) throws XMLStreamException { xmlw.writeStartElement("dataAccs"); - + xmlw.writeStartElement("setAvail"); XmlWriterUtil.writeFullElement(xmlw, "accsPlac", version.getDataAccessPlace()); XmlWriterUtil.writeFullElement(xmlw, "origArch", version.getOriginalArchive()); @@ -390,7 +389,7 @@ private static void writeDataAccess(XMLStreamWriter xmlw , DatasetVersionDTO ver XmlWriterUtil.writeFullElement(xmlw, "collSize", version.getSizeOfCollection()); XmlWriterUtil.writeFullElement(xmlw, "complete", version.getStudyCompletion()); xmlw.writeEndElement(); //setAvail - + xmlw.writeStartElement("useStmt"); XmlWriterUtil.writeFullElement(xmlw, "confDec", version.getConfidentialityDeclaration()); XmlWriterUtil.writeFullElement(xmlw, "specPerm", version.getSpecialPermissions()); @@ -433,7 +432,7 @@ private static void writeDataAccess(XMLStreamWriter xmlw , DatasetVersionDTO ver } xmlw.writeEndElement(); //dataAccs } - + private static void writeDocDescElement (XMLStreamWriter xmlw, DatasetDTO datasetDto) throws XMLStreamException { DatasetVersionDTO version = datasetDto.getDatasetVersion(); String persistentProtocol = datasetDto.getProtocol(); @@ -447,7 +446,7 @@ private static void writeDocDescElement (XMLStreamWriter xmlw, DatasetDTO datase } else if ("doi".equals(persistentAgency)) { persistentAgency = "DOI"; } - + String persistentAuthority = datasetDto.getAuthority(); String persistentId = datasetDto.getIdentifier(); GlobalId pid = PidUtil.parseAsGlobalID(persistentProtocol, persistentAuthority, persistentId); @@ -476,7 +475,7 @@ private static void writeDocDescElement (XMLStreamWriter xmlw, DatasetDTO datase xmlw.writeEndElement(); // distrbtr } XmlWriterUtil.writeFullElement(xmlw, "distDate", datasetDto.getPublicationDate()); - + xmlw.writeEndElement(); // diststmt writeVersionStatement(xmlw, version); xmlw.writeStartElement("biblCit"); @@ -484,9 +483,9 @@ private static void writeDocDescElement (XMLStreamWriter xmlw, DatasetDTO datase xmlw.writeEndElement(); // biblCit xmlw.writeEndElement(); // citation xmlw.writeEndElement(); // docDscr - + } - + private static void writeVersionStatement(XMLStreamWriter xmlw, DatasetVersionDTO datasetVersionDTO) throws XMLStreamException{ xmlw.writeStartElement("verStmt"); xmlw.writeAttribute("source","archive"); @@ -504,10 +503,10 @@ private static void writeVersionStatement(XMLStreamWriter xmlw, DatasetVersionDT xmlw.writeCharacters(datasetVersionDTO.getVersionNote()); xmlw.writeEndElement(); // notes } - + xmlw.writeEndElement(); // verStmt } - + /* From the DDI 2.5 schema: @@ -726,14 +725,14 @@ private static void writeSummaryDescriptionElement(XMLStreamWriter xmlw, Dataset xmlw.writeEndElement(); //sumDscr } - + private static void writeMultipleElement(XMLStreamWriter xmlw, String element, FieldDTO fieldDTO, String lang) throws XMLStreamException { for (String value : fieldDTO.getMultiplePrimitive()) { //Write multiple lang vals for controlled vocab, otherwise don't include any lang tag XmlWriterUtil.writeFullElement(xmlw, element, value, fieldDTO.isControlledVocabularyField() ? lang : null); } } - + private static void writeDateElement(XMLStreamWriter xmlw, String element, String cycle, String event, String dateIn) throws XMLStreamException { xmlw.writeStartElement(element); @@ -744,7 +743,7 @@ private static void writeDateElement(XMLStreamWriter xmlw, String element, Strin xmlw.writeEndElement(); } - + /** * Again, is an xs:sequence - order is important and must follow * the schema. -L.A. @@ -802,7 +801,7 @@ private static void writeMethodElement(XMLStreamWriter xmlw , DatasetVersionDTO XmlWriterUtil.writeI18NElement(xmlw, "srcDocu", version, DatasetFieldConstant.accessToSources, lang); xmlw.writeEndElement(); //sources - + XmlWriterUtil.writeI18NElement(xmlw, "collSitu", version, DatasetFieldConstant.dataCollectionSituation, lang); XmlWriterUtil.writeI18NElement(xmlw, "actMin", version, DatasetFieldConstant.actionsToMinimizeLoss, lang); /* "" has the uppercase C: */ @@ -820,12 +819,12 @@ private static void writeMethodElement(XMLStreamWriter xmlw , DatasetVersionDTO XmlWriterUtil.writeI18NElement(xmlw, "EstSmpErr", version, DatasetFieldConstant.samplingErrorEstimates, lang); XmlWriterUtil.writeI18NElement(xmlw, "dataAppr", version, DatasetFieldConstant.otherDataAppraisal, lang); xmlw.writeEndElement(); //anlyInfo - + xmlw.writeEndElement();//method } - + private static void writeSubjectElement(XMLStreamWriter xmlw, DatasetVersionDTO datasetVersionDTO, String lang) throws XMLStreamException{ - + //Key Words and Topic Classification Locale defaultLocale = Locale.getDefault(); xmlw.writeStartElement("subject"); @@ -1003,7 +1002,7 @@ private static void writeAuthorsElement(XMLStreamWriter xmlw, DatasetVersionDTO } } - + private static void writeContactsElement(XMLStreamWriter xmlw, DatasetVersionDTO datasetVersionDTO) throws XMLStreamException { for (Map.Entry entry : datasetVersionDTO.getMetadataBlocks().entrySet()) { @@ -1042,7 +1041,7 @@ private static void writeContactsElement(XMLStreamWriter xmlw, DatasetVersionDTO } } } - + private static void writeProducersElement(XMLStreamWriter xmlw, DatasetVersionDTO version) throws XMLStreamException { xmlw.writeStartElement("prodStmt"); for (Map.Entry entry : version.getMetadataBlocks().entrySet()) { @@ -1082,7 +1081,7 @@ private static void writeProducersElement(XMLStreamWriter xmlw, DatasetVersionDT xmlw.writeEndElement(); //AuthEnty } } - + } } } @@ -1097,11 +1096,11 @@ private static void writeProducersElement(XMLStreamWriter xmlw, DatasetVersionDT writeMultipleElement(xmlw, "prodPlac", prodPlac, null); } writeSoftwareElement(xmlw, version); - + writeGrantElement(xmlw, version); xmlw.writeEndElement(); //prodStmt } - + private static void writeDistributorsElement(XMLStreamWriter xmlw, DatasetVersionDTO datasetVersionDTO, String lang) throws XMLStreamException { for (Map.Entry entry : datasetVersionDTO.getMetadataBlocks().entrySet()) { String key = entry.getKey(); @@ -1149,7 +1148,7 @@ private static void writeDistributorsElement(XMLStreamWriter xmlw, DatasetVersio } } } - + private static void writeRelPublElement(XMLStreamWriter xmlw, DatasetVersionDTO datasetVersionDTO) throws XMLStreamException { for (Map.Entry entry : datasetVersionDTO.getMetadataBlocks().entrySet()) { String key = entry.getKey(); @@ -1224,7 +1223,7 @@ private static void writeRelPublElement(XMLStreamWriter xmlw, DatasetVersionDTO } } } - + private static String appendCommaSeparatedValue(String inVal, String next) { if (!next.isEmpty()) { if (!inVal.isEmpty()) { @@ -1235,7 +1234,7 @@ private static String appendCommaSeparatedValue(String inVal, String next) { } return inVal; } - + private static void writeAbstractElement(XMLStreamWriter xmlw, DatasetVersionDTO datasetVersionDTO, String lang) throws XMLStreamException { for (Map.Entry entry : datasetVersionDTO.getMetadataBlocks().entrySet()) { String key = entry.getKey(); @@ -1302,7 +1301,7 @@ private static void writeGrantElement(XMLStreamWriter xmlw, DatasetVersionDTO da } } } - + private static void writeOtherIdElement(XMLStreamWriter xmlw, DatasetVersionDTO datasetVersionDTO) throws XMLStreamException { for (Map.Entry entry : datasetVersionDTO.getMetadataBlocks().entrySet()) { String key = entry.getKey(); @@ -1334,7 +1333,7 @@ private static void writeOtherIdElement(XMLStreamWriter xmlw, DatasetVersionDTO } } } - + private static void writeSoftwareElement(XMLStreamWriter xmlw, DatasetVersionDTO datasetVersionDTO) throws XMLStreamException { for (Map.Entry entry : datasetVersionDTO.getMetadataBlocks().entrySet()) { String key = entry.getKey(); @@ -1366,7 +1365,7 @@ private static void writeSoftwareElement(XMLStreamWriter xmlw, DatasetVersionDTO } } } - + private static void writeSeriesElement(XMLStreamWriter xmlw, DatasetVersionDTO datasetVersionDTO) throws XMLStreamException { for (Map.Entry entry : datasetVersionDTO.getMetadataBlocks().entrySet()) { String key = entry.getKey(); @@ -1404,7 +1403,7 @@ private static void writeSeriesElement(XMLStreamWriter xmlw, DatasetVersionDTO d } } } - + private static void writeTargetSampleElement(XMLStreamWriter xmlw, DatasetVersionDTO datasetVersionDTO) throws XMLStreamException { for (Map.Entry entry : datasetVersionDTO.getMetadataBlocks().entrySet()) { String key = entry.getKey(); @@ -1436,14 +1435,14 @@ private static void writeTargetSampleElement(XMLStreamWriter xmlw, DatasetVersio xmlw.writeCharacters(sizeFormula); xmlw.writeEndElement(); //sampleSizeFormula } - + xmlw.writeEndElement(); // targetSampleSize } } } } } - + private static void writeNotesElement(XMLStreamWriter xmlw, DatasetVersionDTO datasetVersionDTO) throws XMLStreamException { for (Map.Entry entry : datasetVersionDTO.getMetadataBlocks().entrySet()) { String key = entry.getKey(); @@ -1479,7 +1478,7 @@ private static void writeNotesElement(XMLStreamWriter xmlw, DatasetVersionDTO da } } } - + // TODO: // see if there's more information that we could encode in this otherMat. // contentType? Unfs and such? (in the "short" DDI that is being used for @@ -1487,7 +1486,7 @@ private static void writeNotesElement(XMLStreamWriter xmlw, DatasetVersionDTO da private static void createOtherMats(XMLStreamWriter xmlw, List fileDtos, boolean skipTabularFiles) throws XMLStreamException { // The preferred URL for this dataverse, for cooking up the file access API links: String dataverseUrl = SystemConfig.getDataverseSiteUrlStatic(); - + for (FileDTO fileDTo : fileDtos) { // We'll continue using the scheme we've used before, in DVN2-3: non-tabular files are put into otherMat, // tabular ones - in fileDscr sections. (fileDscr sections have special fields for numbers of variables @@ -1522,7 +1521,7 @@ private static void createOtherMats(XMLStreamWriter xmlw, List fileDtos } } } - + // An alternative version of the createOtherMats method - this one is used // when a "full" DDI is being cooked; just like the fileDscr and data/var sections methods, // it operates on the list of FileMetadata entities, not on File DTOs. This is because @@ -1537,11 +1536,11 @@ private static void createOtherMats(XMLStreamWriter xmlw, List fileDtos // DO in fact contain enough information to generate the otherMat sections // properly, whether this is a short or a full version of the DDI. I am however leaving // this method here for reference. - + private static void createOtherMatsFromFileMetadatas(XMLStreamWriter xmlw, JsonArray fileDetails) throws XMLStreamException { // The preferred URL for this dataverse, for cooking up the file access API links: String dataverseUrl = SystemConfig.getDataverseSiteUrlStatic(); - + for (int i=0;i dto2PrimitiveList(DatasetVersionDTO datasetVersionDTO, String datasetFieldTypeName) { for (Map.Entry entry : datasetVersionDTO.getMetadataBlocks().entrySet()) { MetadataBlockDTO value = entry.getValue(); @@ -1614,7 +1613,7 @@ private static List dto2PrimitiveList(DatasetVersionDTO datasetVersionDT } return null; } - + private static FieldDTO dto2FieldDTO(DatasetVersionDTO datasetVersionDTO, String datasetFieldTypeName, String metadataBlockName) { MetadataBlockDTO block = datasetVersionDTO.getMetadataBlocks().get(metadataBlockName); if (block != null) { @@ -1638,10 +1637,10 @@ private static boolean StringUtilisEmpty(String str) { private static void saveJsonToDisk(String datasetVersionAsJson) throws IOException { Files.write(Paths.get("/tmp/out.json"), datasetVersionAsJson.getBytes()); } - - - - + + + + // Methods specific to the tabular data ("") section. // Note that these do NOT operate on DTO objects, but instead directly // on Dataverse DataVariable, DataTable, etc. objects. @@ -1653,7 +1652,7 @@ private static void saveJsonToDisk(String datasetVersionAsJson) throws IOExcepti // can go through the same DTO state... But we don't have time for it now; // plus, the structure of file-level metadata is currently being re-designed, // so we probably should not invest any time into it right now). -- L.A. 4.5 - + public static void createDataDscr(XMLStreamWriter xmlw, JsonArray fileDetails) throws XMLStreamException { if (fileDetails.isEmpty()) { @@ -1677,13 +1676,13 @@ public static void createDataDscr(XMLStreamWriter xmlw, JsonArray fileDetails) t if (isFileRestricted(fileJson)) { continue; } - + if (fileJson.containsKey("dataTables")) { if (!dataDscrWritten) { xmlw.writeStartElement("dataDscr"); dataDscrWritten = true; } - + createVariablesForDataFile(xmlw, fileJson); } } @@ -1694,18 +1693,31 @@ public static void createDataDscr(XMLStreamWriter xmlw, JsonArray fileDetails) t } private static void createDataDscrInBatches(XMLStreamWriter xmlw, List varQuantityMap, ExportDataProvider exportDataProvider) throws XMLStreamException { + createDataDscrInBatches(xmlw, varQuantityMap, exportDataProvider, DATAVARIABLES_BATCH_SIZE); + } + + /** + * + * This public version of the method exists solely so that it can be called + * with a custom batch size; primarily for tests. + * @param xmlw XML stream writer + * @varQuantityMap mapping of ordered datatables -> number of variables in each + * @exportDataProvider data provider instance + * @variablesBatchSize number of variables to use in forming processing batches + */ + public static void createDataDscrInBatches(XMLStreamWriter xmlw, List varQuantityMap, ExportDataProvider exportDataProvider, int variablesBatchSize) throws XMLStreamException { boolean dataDscrWritten = false; - + try { int dataTableStart = 0; int dataTablesThisBatch = 0; int varQuantityThisBatch = 0; - + for (int dataTableCurrent = 0; dataTableCurrent < varQuantityMap.size(); dataTableCurrent++) { varQuantityThisBatch += varQuantityMap.get(dataTableCurrent); dataTablesThisBatch++; - - if (varQuantityThisBatch >= DATAVARIABLES_BATCH_SIZE || dataTableCurrent == varQuantityMap.size() - 1) { + + if (varQuantityThisBatch >= variablesBatchSize || dataTableCurrent == varQuantityMap.size() - 1) { //JsonArray tabularFileDetails = exportDataProvider.getTabularDataDetails(ExportDataContext.context().withOffset(dataTableStart).withLength(dataTablesThisBatch)); FileExportQuery exportQuery = FileExportQuery.builder() .addFilePredicate(FileMetadataPredicates.ONLY_PUBLIC_FILES) @@ -1750,7 +1762,7 @@ private static void createDataDscrInBatches(XMLStreamWriter xmlw, List } count++; } - + logger.fine("requested: " + dataTablesThisBatch + " tabular file data entries; retrieved: " + count); @@ -1759,6 +1771,10 @@ private static void createDataDscrInBatches(XMLStreamWriter xmlw, List varQuantityThisBatch = 0; } } + if (dataDscrWritten) { + xmlw.writeEndElement(); // dataDscr + } + } catch (ExportException ee) { if (dataDscrWritten) { // Unfortunately, we've already written some output by the time @@ -1792,7 +1808,7 @@ private static int createVariablesForDataFile(XMLStreamWriter xmlw, JsonObject f } return vars.size(); } - + private static void createVarGroupDDI(XMLStreamWriter xmlw, JsonObject varGrp) throws XMLStreamException { xmlw.writeStartElement("varGrp"); xmlw.writeAttribute("ID", "VG" + varGrp.getJsonNumber("id").toString()); @@ -1813,7 +1829,7 @@ private static void createVarGroupDDI(XMLStreamWriter xmlw, JsonObject varGrp) t xmlw.writeEndElement(); //varGrp } - + private static void createVarDDI(XMLStreamWriter xmlw, JsonObject dvar, String fileId, String fileMetadataId) throws XMLStreamException { xmlw.writeStartElement("var"); xmlw.writeAttribute("ID", "v" + dvar.getJsonNumber("id").toString()); @@ -2055,15 +2071,15 @@ private static void createVarDDI(XMLStreamWriter xmlw, JsonObject dvar, String f xmlw.writeEndElement(); //var } - + private static List createFileDscrs(XMLStreamWriter xmlw, List fileDtos) throws XMLStreamException { List ret = new ArrayList<>(); - + logger.fine("total " + fileDtos.size() + " file DTOs to process for fileDscr"); String dataverseUrl = SystemConfig.getDataverseSiteUrlStatic(); int counter = 0; long totalVarQuantity = 0; - + for (FileDTO fileDTo : fileDtos) { logger.fine("processing file " + fileDTo.getDataFile().getId()); if (isTabularData(fileDTo)) { @@ -2090,7 +2106,7 @@ private static List createFileDscrs(XMLStreamWriter xmlw, List } Long varQuantity = dataTableDTO.getVarQuantity(); - + if (varQuantity != null) { xmlw.writeStartElement("varQnty"); xmlw.writeCharacters(dataTableDTO.getVarQuantity().toString()); @@ -2163,7 +2179,7 @@ private static List createFileDscrs(XMLStreamWriter xmlw, List logger.fine("produced " + counter + " fileDscr entries; total number of variables found: " + totalVarQuantity); return ret; } - + public static void datasetHtmlDDI(InputStream datafile, OutputStream outputStream) throws XMLStreamException { try { @@ -2182,7 +2198,7 @@ public static void datasetHtmlDDI(InputStream datafile, OutputStream outputStrea // Set secure processing feature tFactory.setFeature(XMLConstants.FEATURE_SECURE_PROCESSING, true); tFactory.setAttribute(XMLConstants.ACCESS_EXTERNAL_STYLESHEET, ""); - + StreamSource stylesource = new StreamSource(styleSheetInput); Transformer transformer = tFactory.newTransformer(stylesource); @@ -2208,11 +2224,11 @@ public static void datasetHtmlDDI(InputStream datafile, OutputStream outputStrea public static void injectSettingsService(SettingsServiceBean settingsSvc) { settingsService=settingsSvc; } - + private static boolean isTabularData(FileDTO fileDTO) { return !(fileDTO.getDataFile().getDataTables() == null || fileDTO.getDataFile().getDataTables().isEmpty()); } - + /** * Previously (in Dataverse 5.3 and below) the dataDscr section was included * for restricted files but that meant that summary statistics were exposed. @@ -2236,7 +2252,7 @@ private static boolean isFileRestricted(JsonObject fileJson) { } return false; } - + } diff --git a/src/test/java/edu/harvard/iq/dataverse/export/TabularDataExportIT.java b/src/test/java/edu/harvard/iq/dataverse/export/TabularDataExportIT.java new file mode 100644 index 00000000000..0915b7d9f8b --- /dev/null +++ b/src/test/java/edu/harvard/iq/dataverse/export/TabularDataExportIT.java @@ -0,0 +1,359 @@ +package edu.harvard.iq.dataverse.export; + +import edu.harvard.iq.dataverse.DataFile; +import edu.harvard.iq.dataverse.Dataset; +import edu.harvard.iq.dataverse.DatasetVersion; +import edu.harvard.iq.dataverse.DatasetVersionFilesServiceBean; +import edu.harvard.iq.dataverse.export.ddi.DdiExportUtil; +import edu.harvard.iq.dataverse.util.testing.fixtures.DatasetFixtureBuilder; +import edu.harvard.iq.dataverse.util.testing.performance.JpaEntityManagerService; +import edu.harvard.iq.dataverse.util.testing.performance.JpaPerformanceTest; +import edu.harvard.iq.dataverse.util.testing.recipes.DatasetRecipe; +import edu.harvard.iq.dataverse.util.testing.recipes.DatasetTypeRecipe; +import edu.harvard.iq.dataverse.util.testing.recipes.FileRecipe; +import edu.harvard.iq.dataverse.util.testing.recipes.VersionRecipe; +import edu.harvard.iq.dataverse.util.testing.recipes.VariableSetRecipe; +import edu.harvard.iq.dataverse.util.xml.XmlUtil; +import io.gdcc.spi.export.FileExportQuery; +import io.gdcc.spi.export.FileMetadataPredicates; +import io.gdcc.spi.export.PageRequest; +import jakarta.json.Json; +import jakarta.json.JsonArray; +import jakarta.json.JsonArrayBuilder; +import jakarta.json.JsonObject; +import java.io.ByteArrayOutputStream; +import java.io.StringReader; +import net.ttddyy.dsproxy.QueryCount; +import net.ttddyy.dsproxy.QueryCountHolder; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.nio.charset.StandardCharsets; + +import java.time.Instant; +import java.time.temporal.ChronoUnit; +import java.util.ArrayList; +import java.util.HashSet; + +import java.util.Iterator; +import java.util.List; +import java.util.Set; +import java.util.stream.Stream; +import javax.xml.stream.XMLInputFactory; +import javax.xml.stream.XMLOutputFactory; +import javax.xml.stream.XMLStreamConstants; +import javax.xml.stream.XMLStreamException; +import javax.xml.stream.XMLStreamReader; +import javax.xml.stream.XMLStreamWriter; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assumptions.assumeTrue; + +@JpaPerformanceTest +class TabularDataExportIT { + + static JpaEntityManagerService jpa; + + static Dataset tabularFilesDataset; + static int numberOfFiles = 20; + static int numberOfVariables = 100; + static int numberOfBatches = 4; + static List varQuantityMap = new ArrayList<>(); + + @BeforeAll + static void setUp() { + jpa.start(); + + DatasetTypeRecipe datasetType = DatasetTypeRecipe.dataset(); + + DatasetRecipe tabularFiles = DatasetRecipe.of( + datasetType, + VersionRecipe.of( + FileRecipe.tabular(numberOfFiles, VariableSetRecipe.uniform(numberOfVariables)) + ) + ); + + // Build the fixture + var tabularFixture = DatasetFixtureBuilder.builder().recipe(tabularFiles).build(); + + // Some save the type entity in the database and let the ORM create the mappings + jpa.inTransactionVoid(em -> em.persist(tabularFixture.datasetType())); + + // Persist the actual dataset + tabularFilesDataset = tabularFixture.dataset(); + jpa.inTransactionVoid(em -> { + // DataFile has no cascade path from Dataset, so each file must be persisted explicitly before + // the dataset graph is flushed. + for (DataFile dataFile : tabularFixture.dataFiles()) { + em.persist(dataFile); + } + em.persist(tabularFilesDataset); + }); + + for (int i = 0; i < numberOfFiles; i++) { + varQuantityMap.add(numberOfVariables); + // Something to consider in the future - add a more complex recipe, + // with varying numbers of variables in each file. + } + } + + @Test + void exportTabularMetadata() { + Long datasetVersionId = tabularFilesDataset.getVersions().get(0).getId(); + System.out.println("version id: " + datasetVersionId); + + QueryCountHolder.clear(); + Instant start = Instant.now(); + + // First, obtain the entire mess of the tabular data in the dataset in + // in one go, using the legacy .getDatasetFileDetails() method: + String json = jpa.inTransaction(em -> { + var datasetVersion = em.find(DatasetVersion.class, datasetVersionId); + assumeTrue(datasetVersion != null, "No dataset version available in DB. Check fixtures!"); + + InternalExportDataProvider provider = new InternalExportDataProvider(datasetVersion); + JsonArray details = provider.getDatasetFileDetails(); + // We want to parse this json and make sure that the expected number + // of files, datatables and variables has been returned. + + assertEquals(numberOfFiles, details.size(), "Number of tabular files retrieved does not match the number used in the recipe"); + + int total = 0; + for (int i = 0; i < details.size(); i++) { + JsonObject fileJson = details.getJsonObject(i); + JsonObject dataTable = fileJson.getJsonArray("dataTables").getJsonObject(0); + JsonArray vars = dataTable.getJsonArray("dataVariables"); + System.out.println(vars.size() + " variables retrieved for file " + fileJson.getJsonNumber("id")); + total += vars.size(); + } + assertEquals(numberOfFiles * numberOfVariables, total, "Failed to retrieve and parse the expected total number of variables"); + return details.toString(); + }); + + assertNotNull(json); + + System.out.println("test json produced: " + json); + + // Calculate the md5 of the complete output, for verification further down + String md5 = calculateMD5(json); + assertNotNull(md5); + System.out.println("MD5 Hash: " + md5); + + Instant end = Instant.now(); + long elapsed = start.until(end, ChronoUnit.MILLIS); + + QueryCount count = QueryCountHolder.getGrandTotal(); + + long queriesTotal = count.getTotal(); + long queriesSelect = count.getSelect(); + + System.out.println("Elapsed ms: " + elapsed); + System.out.println("Total queries: " + queriesTotal); + System.out.println("Select queries: " + queriesSelect); + + // And now acquire the same content using the new, paginated methods; + // then compare the checksums, to ensure the 2 methods produce the same + // exact result. + start = Instant.now(); + + json = jpa.inTransaction(em -> { + var datasetVersion = em.find(DatasetVersion.class, datasetVersionId); + assumeTrue(datasetVersion != null, "No dataset version available in DB. Check fixtures!"); + + InternalExportDataProvider provider = new InternalExportDataProvider(datasetVersion); + DatasetVersionFilesServiceBean versionFilesService = new DatasetVersionFilesServiceBean(); + versionFilesService.injectEntityManager(em); + provider.injectVersionFilesService(versionFilesService); + + JsonArrayBuilder jab = Json.createArrayBuilder(); + int filesPerBatch = numberOfFiles / numberOfBatches; + + for (int i = 0; i < numberOfBatches; i++) { + + FileExportQuery exportQuery = FileExportQuery.builder() + .addFilePredicate(FileMetadataPredicates.ONLY_PUBLIC_FILES) + .addFilePredicate(FileMetadataPredicates.ONLY_TABULAR_FILES) + .addFilePredicate(FileMetadataPredicates.INCLUDE_TABULAR_DATA_VARIABLES) + .build(); + PageRequest paginationRequest = PageRequest.of(filesPerBatch * i, filesPerBatch); + + Stream details = provider.getDatasetFileDetails(exportQuery, paginationRequest); + + Iterator it = details.iterator(); + int filesRetrieved = 0; + while (it.hasNext()) { + JsonObject fileJson = it.next(); + jab.add(fileJson); + filesRetrieved++; + } + assertEquals(filesPerBatch, filesRetrieved, "Failed to retrieve the expected number of tabular files in batch number " + i); + } + + // Parse the combined json, count the variables, confirm: + JsonArray jsonCombined = jab.build().asJsonArray(); + + int total = 0; + for (int i = 0; i < jsonCombined.size(); i++) { + JsonObject fileJson = jsonCombined.getJsonObject(i); + JsonObject dataTable = fileJson.getJsonArray("dataTables").getJsonObject(0); + JsonArray vars = dataTable.getJsonArray("dataVariables"); + total += vars.size(); + } + assertEquals(numberOfFiles * numberOfVariables, total, "Failed to retrieve and parse the expected total number of variables using new, paginated methods"); + + return jsonCombined.toString(); + }); + + end = Instant.now(); + elapsed = start.until(end, ChronoUnit.MILLIS); + + System.out.println("test json produced, paginated: " + json); + + String md5paginated = calculateMD5(json); + assertNotNull(md5paginated); + System.out.println("MD5 Hash: " + md5paginated); + + assertEquals(md5, md5paginated, "MD5 Hash mismatch between json fragments produced by the legacy, and paginated ExportDataProvder methods"); + + count = QueryCountHolder.getGrandTotal(); + System.out.println("Elapsed ms using paginated methods: " + elapsed); + System.out.println("Total queries using paginated methods: " + (count.getTotal() - queriesTotal)); + System.out.println("Select queries using paginated methods: " + (count.getSelect() - queriesSelect)); + + // And now try to export the dataVariable-level metadata as the + // fragment of DDI xml. + // (we are not interested in exporting a full DDI for this imaginary + // dataset since there are dedicated tests for the dataset-level DDI + // exports elsewhere). + String xml = jpa.inTransaction(em -> { + var datasetVersion = em.find(DatasetVersion.class, datasetVersionId); + assumeTrue(datasetVersion != null, "No dataset version available in DB. Check fixtures!"); + + InternalExportDataProvider provider = new InternalExportDataProvider(datasetVersion); + DatasetVersionFilesServiceBean versionFilesService = new DatasetVersionFilesServiceBean(); + versionFilesService.injectEntityManager(em); + provider.injectVersionFilesService(versionFilesService); + + ByteArrayOutputStream byteOutputStream = new ByteArrayOutputStream(); + + XMLStreamWriter xmlw = null; + try { + xmlw = XMLOutputFactory.newInstance().createXMLStreamWriter(byteOutputStream); + DdiExportUtil.createDataDscrInBatches(xmlw, varQuantityMap, provider, numberOfFiles * numberOfVariables / numberOfBatches); + xmlw.flush(); + } catch (XMLStreamException xse) { + assertTrue(false, "Failed to export the dataDscr DDI section, XMLStreamException: " + xse.getMessage()); + } finally { + if (xmlw != null) { + try { + xmlw.close(); + } catch (XMLStreamException e) { + // we don't care at this point + } + } + } + + return byteOutputStream.toString(StandardCharsets.UTF_8); + }); + + System.out.println("test DDI xml produced: " + xml); + + // Finally, let's parse the resulting XML and confirm that the expected + // numbers of unique datatables and variables have been exported. + // There are obvious ways in which the test can be made more thorough. + // For example, we can trace each variable by name and confirm that each + // one is properly exported in both the json and ddi xml formats. + StringReader reader = new StringReader(xml); + XMLStreamReader xmlr = null; + XMLInputFactory xmlFactory = XmlUtil.getSecureXMLInputFactory(); + + int dataDscrVariablesTotal = 0; + boolean dataDscrComplete = false; + boolean dataDscrVarElementOpen = false; + Set dataDscrDistinctFiles = new HashSet<>(); + + try { + xmlr = xmlFactory.createXMLStreamReader(reader); + } catch (XMLStreamException xse) { + assertTrue(false, "Failed to parse the produced dataDscr fragment as valid xml, XMLStreamException: " + xse.getMessage()); + } + try { + xmlr.nextTag(); + xmlr.require(XMLStreamConstants.START_ELEMENT, null, "dataDscr"); + } catch (XMLStreamException xse) { + assertTrue(false, "The produced xml fragment does not start with a dataDscr tag"); + } + + try { + for (int event = xmlr.next(); event != XMLStreamConstants.END_DOCUMENT; event = xmlr.next()) { + if (event == XMLStreamConstants.START_ELEMENT) { + if (xmlr.getLocalName().equals("var")) { + assertFalse(dataDscrVarElementOpen, "Out of order, nested tag encountered"); + dataDscrVarElementOpen = true; + } else if (xmlr.getLocalName().equals("location")) { + assertTrue(dataDscrVarElementOpen, "Out of order tag encountered"); + String fileId = xmlr.getAttributeValue(null, "fileid"); + assertNotNull(fileId, " element without a valid fileid attribute encountered"); + dataDscrDistinctFiles.add(fileId); + } + } else if (event == XMLStreamConstants.END_ELEMENT) { + if (xmlr.getLocalName().equals("var")) { + assertTrue(dataDscrVarElementOpen, "Out of order closing tag encountered"); + dataDscrVariablesTotal++; + dataDscrVarElementOpen = false; + } else if (xmlr.getLocalName().equals("dataDscr")) { + dataDscrComplete = true; + } + } + } + } catch (XMLStreamException xse) { + assertTrue(false, "Unexpected XMLStreamException when attempting to parse the dataDscr section: " + xse.getMessage()); + } finally { + if (xmlr != null) { + try {xmlr.close();} catch (XMLStreamException e) {} + } + } + + assertTrue(dataDscrComplete, " section not closed properly"); + assertFalse(dataDscrVarElementOpen, "an extra, unterminated section in the "); + assertEquals(numberOfFiles * numberOfVariables, dataDscrVariablesTotal, "Failed to parse the expected total number of variables in the generated section"); + assertEquals(numberOfFiles, dataDscrDistinctFiles.size(), "Invalid number of distinct tabular datafiles in the exported ddi fragment"); + + // In all of the tests above, the final result - the entire dataset-worth + // of exported tabular data - is passed around as a string; in the last + // method, this combined string then get re-parsed as xml for validation + // purposes. This is working adequately for the numbers of files and + // variables used; but if we want to use this code for true stress-testing + // with gigantic amounts of metadata, it will need to be rewritten to + // to stream the data in real time, for writing and reading, to avoid + // having to keep the whole mess in memory at once. + } + + private String calculateMD5(String source) { + String md5 = null; + + try { + MessageDigest md = MessageDigest.getInstance("MD5"); + md.update(source.getBytes(StandardCharsets.UTF_8)); + + byte[] hashBytes = md.digest(); + + StringBuilder hexString = new StringBuilder(); + for (byte b : hashBytes) { + String hex = Integer.toHexString(0xff & b); + if (hex.length() == 1) { + hexString.append('0'); + } + hexString.append(hex); + } + md5 = hexString.toString(); + } catch (NoSuchAlgorithmException e) { + System.err.println("MD5 algorithm not found!"); + //e.printStackTrace(); + } + return md5; + } +} From 523bf0e51e150bf5fe8f20bbdd11e91118c4b74e Mon Sep 17 00:00:00 2001 From: Leonid Andreev Date: Mon, 20 Jul 2026 09:28:36 -0400 Subject: [PATCH 22/25] Minor fix in Oliver's sample JpaEntityManageService test. It was working as is when committed, since there was only one dataset and one datasetversion in the throwaway database. #11405 --- .../iq/dataverse/export/HugeDatasetExportPerformanceIT.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/java/edu/harvard/iq/dataverse/export/HugeDatasetExportPerformanceIT.java b/src/test/java/edu/harvard/iq/dataverse/export/HugeDatasetExportPerformanceIT.java index 1cf4c17495d..63bf826167d 100644 --- a/src/test/java/edu/harvard/iq/dataverse/export/HugeDatasetExportPerformanceIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/export/HugeDatasetExportPerformanceIT.java @@ -60,7 +60,7 @@ static void setUp() { @Test void shouldExportLargeDataset() { - Long datasetVersionId = regularFilesDataset.getId(); + Long datasetVersionId = regularFilesDataset.getVersions().get(0).getId(); QueryCountHolder.clear(); Instant start = Instant.now(); From 4662b9f2aff665a412dbe943e84daf154752063c Mon Sep 17 00:00:00 2001 From: Leonid Andreev Date: Mon, 20 Jul 2026 09:41:41 -0400 Subject: [PATCH 23/25] Adding the new, JpaEntityManagerService-based test to the list of regularly-scheduled workflow action integration tests. #11405 --- tests/integration-tests.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration-tests.txt b/tests/integration-tests.txt index 77bfe683545..f3bf1f68d32 100644 --- a/tests/integration-tests.txt +++ b/tests/integration-tests.txt @@ -1 +1 @@ -DataversesIT,DatasetsIT,SwordIT,AdminIT,BuiltinUsersIT,UsersIT,UtilIT,ConfirmEmailIT,FileMetadataIT,FilesIT,SearchIT,InReviewWorkflowIT,HarvestingServerIT,HarvestingClientsIT,MoveIT,MakeDataCountApiIT,FileTypeDetectionIT,EditDDIIT,ExternalToolsIT,AccessIT,DuplicateFilesIT,DownloadFilesIT,LinkIT,DeleteUsersIT,DeactivateUsersIT,AuxiliaryFilesIT,InvalidCharactersIT,LicensesIT,NotificationsIT,BagIT,MetadataBlocksIT,NetcdfIT,SignpostingIT,FitsIT,LogoutIT,DataRetrieverApiIT,ProvIT,S3AccessIT,OpenApiIT,InfoIT,DatasetFieldsIT,SavedSearchIT,DatasetTypesIT,DataverseFeaturedItemsIT,SendFeedbackApiIT,CustomizationIT,JsonLDExportIT,WorkflowsIT,LDNInboxIT,LocalContextsIT,CorsIT,GuestbooksIT,LocallyFairIT +DataversesIT,DatasetsIT,SwordIT,AdminIT,BuiltinUsersIT,UsersIT,UtilIT,ConfirmEmailIT,FileMetadataIT,FilesIT,SearchIT,InReviewWorkflowIT,HarvestingServerIT,HarvestingClientsIT,MoveIT,MakeDataCountApiIT,FileTypeDetectionIT,EditDDIIT,ExternalToolsIT,AccessIT,DuplicateFilesIT,DownloadFilesIT,LinkIT,DeleteUsersIT,DeactivateUsersIT,AuxiliaryFilesIT,InvalidCharactersIT,LicensesIT,NotificationsIT,BagIT,MetadataBlocksIT,NetcdfIT,SignpostingIT,FitsIT,LogoutIT,DataRetrieverApiIT,ProvIT,S3AccessIT,OpenApiIT,InfoIT,DatasetFieldsIT,SavedSearchIT,DatasetTypesIT,DataverseFeaturedItemsIT,SendFeedbackApiIT,CustomizationIT,JsonLDExportIT,WorkflowsIT,LDNInboxIT,LocalContextsIT,CorsIT,GuestbooksIT,LocallyFairIT,TabularDataExportIT From f24c726f14caf4e9c0dbee998e6128c575454b76 Mon Sep 17 00:00:00 2001 From: Leonid Andreev Date: Tue, 21 Jul 2026 14:47:32 -0400 Subject: [PATCH 24/25] Removed modules/dataverse-spi and the action that used to build snapshot releases of it from the source tree; since dataverse-spi has been living under https://github.com/gdcc/dataverse-spi for some time now. #11405 --- .github/workflows/spi_release.yml | 94 ------- modules/dataverse-spi/.gitignore | 1 - modules/dataverse-spi/pom.xml | 242 ------------------ .../io/gdcc/spi/export/ExportDataContext.java | 61 ----- .../gdcc/spi/export/ExportDataProvider.java | 126 --------- .../io/gdcc/spi/export/ExportException.java | 13 - .../java/io/gdcc/spi/export/Exporter.java | 109 -------- .../java/io/gdcc/spi/export/XMLExporter.java | 37 --- 8 files changed, 683 deletions(-) delete mode 100644 .github/workflows/spi_release.yml delete mode 100644 modules/dataverse-spi/.gitignore delete mode 100644 modules/dataverse-spi/pom.xml delete mode 100644 modules/dataverse-spi/src/main/java/io/gdcc/spi/export/ExportDataContext.java delete mode 100644 modules/dataverse-spi/src/main/java/io/gdcc/spi/export/ExportDataProvider.java delete mode 100644 modules/dataverse-spi/src/main/java/io/gdcc/spi/export/ExportException.java delete mode 100644 modules/dataverse-spi/src/main/java/io/gdcc/spi/export/Exporter.java delete mode 100644 modules/dataverse-spi/src/main/java/io/gdcc/spi/export/XMLExporter.java diff --git a/.github/workflows/spi_release.yml b/.github/workflows/spi_release.yml deleted file mode 100644 index 8fe80925b84..00000000000 --- a/.github/workflows/spi_release.yml +++ /dev/null @@ -1,94 +0,0 @@ -name: Dataverse SPI - -on: - push: - branches: - - "develop" - paths: - - "modules/dataverse-spi/**" - pull_request: - branches: - - "develop" - paths: - - "modules/dataverse-spi/**" - -jobs: - # Note: Pushing packages to Maven Central requires access to secrets, which pull requests from remote forks - # don't have. Skip in these cases. - check-secrets: - name: Check for Secrets Availability - runs-on: ubuntu-latest - outputs: - available: ${{ steps.secret-check.outputs.available }} - steps: - - id: secret-check - # perform secret check & put boolean result as an output - shell: bash - run: | - if [ "${{ secrets.DATAVERSEBOT_SONATYPE_USERNAME }}" != '' ]; then - echo "available=true" >> $GITHUB_OUTPUT; - else - echo "available=false" >> $GITHUB_OUTPUT; - fi - - snapshot: - name: Release Snapshot - needs: check-secrets - runs-on: ubuntu-latest - if: github.event_name == 'pull_request' && needs.check-secrets.outputs.available == 'true' - steps: - - uses: actions/checkout@v7 - - uses: actions/setup-java@v5 - with: - java-version: '21' - distribution: 'adopt' - server-id: central - server-username: MAVEN_USERNAME - server-password: MAVEN_PASSWORD - - uses: actions/cache@v6 - with: - path: ~/.m2 - key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }} - restore-keys: ${{ runner.os }}-m2 - - - name: Deploy Snapshot - run: mvn -f modules/dataverse-spi -Dproject.version.suffix="-PR${{ github.event.number }}-SNAPSHOT" deploy - env: - MAVEN_USERNAME: ${{ secrets.DATAVERSEBOT_SONATYPE_USERNAME }} - MAVEN_PASSWORD: ${{ secrets.DATAVERSEBOT_SONATYPE_TOKEN }} - - release: - name: Release - needs: check-secrets - runs-on: ubuntu-latest - if: github.event_name == 'push' && needs.check-secrets.outputs.available == 'true' - steps: - - uses: actions/checkout@v7 - - uses: actions/setup-java@v5 - with: - java-version: '21' - distribution: 'adopt' - - uses: actions/cache@v6 - with: - path: ~/.m2 - key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }} - restore-keys: ${{ runner.os }}-m2 - - # Running setup-java again overwrites the settings.xml - IT'S MANDATORY TO DO THIS SECOND SETUP!!! - - name: Set up Maven Central Repository - uses: actions/setup-java@v5 - with: - java-version: '21' - distribution: 'adopt' - server-id: central - server-username: MAVEN_USERNAME - server-password: MAVEN_PASSWORD - gpg-private-key: ${{ secrets.DATAVERSEBOT_GPG_KEY }} - gpg-passphrase: MAVEN_GPG_PASSPHRASE - - - name: Sign + Publish Release - run: mvn -f modules/dataverse-spi -P release deploy - env: - MAVEN_USERNAME: ${{ secrets.DATAVERSEBOT_SONATYPE_USERNAME }} - MAVEN_PASSWORD: ${{ secrets.DATAVERSEBOT_SONATYPE_TOKEN }} - MAVEN_GPG_PASSPHRASE: ${{ secrets.DATAVERSEBOT_GPG_PASSWORD }} diff --git a/modules/dataverse-spi/.gitignore b/modules/dataverse-spi/.gitignore deleted file mode 100644 index d75620abf70..00000000000 --- a/modules/dataverse-spi/.gitignore +++ /dev/null @@ -1 +0,0 @@ -.flattened-pom.xml diff --git a/modules/dataverse-spi/pom.xml b/modules/dataverse-spi/pom.xml deleted file mode 100644 index a603e274234..00000000000 --- a/modules/dataverse-spi/pom.xml +++ /dev/null @@ -1,242 +0,0 @@ - - - 4.0.0 - - - edu.harvard.iq - dataverse-parent - ${revision} - ../dataverse-parent - - - io.gdcc - dataverse-spi - 2.1.0${project.version.suffix} - jar - - Dataverse SPI Plugin API - https://dataverse.org - - A package to create out-of-tree Java code for Dataverse Software. Plugin projects can use this package - as an API dependency just like Jakarta EE APIs if they want to create external plugins. These will be loaded - at runtime of a Dataverse installation using SPI. See also https://guides.dataverse.org/en/latest/developers - for more information. - - - - - Apache-2.0 - https://www.apache.org/licenses/LICENSE-2.0.txt - repo - - - - - - Dataverse Core Team - support@dataverse.org - - - - - https://github.com/IQSS/dataverse/issues - GitHub Issues - - - - scm:git:git@github.com:IQSS/dataverse.git - scm:git:git@github.com:IQSS/dataverse.git - git@github.com:IQSS/dataverse.git - HEAD - - - - https://github.com/IQSS/dataverse/actions - github - - -
dataversebot@gdcc.io
- - - - - - - central - https://central.sonatype.com/repository/maven-snapshots/ - - - - ossrh - - https://s01.oss.sonatype.org/service/local/staging/deploy/maven2/ - - - - - - - none - false - - - - - jakarta.json - jakarta.json-api - provided - - - - jakarta.ws.rs - jakarta.ws.rs-api - provided - - - - - - - - maven-compiler-plugin - - ${target.java.version} - - - - - - org.sonatype.plugins - nexus-staging-maven-plugin - true - - - ossrh - - https://s01.oss.sonatype.org - true - - - - org.apache.maven.plugins - maven-release-plugin - - false - release - true - deploy - - - - org.codehaus.mojo - flatten-maven-plugin - - true - oss - - remove - remove - - - - - - flatten - process-resources - - flatten - - - - - flatten.clean - clean - - clean - - - - - - org.apache.maven.plugins - maven-deploy-plugin - - ${skipDeploy} - - - - - - - - release - - - - org.apache.maven.plugins - maven-gpg-plugin - - - sign-artifacts - verify - - sign - - - - - - org.kordamp.maven - pomchecker-maven-plugin - - - process-resources - - check-maven-central - - - - - - - org.apache.maven.plugins - maven-javadoc-plugin - - - attach-javadocs - - jar - - - - - ${target.java.version} - false - ${javadoc.lint} - - - - org.apache.maven.plugins - maven-source-plugin - - - attach-sources - - jar - - - - - - - - - ct - - true - - - - diff --git a/modules/dataverse-spi/src/main/java/io/gdcc/spi/export/ExportDataContext.java b/modules/dataverse-spi/src/main/java/io/gdcc/spi/export/ExportDataContext.java deleted file mode 100644 index 9478d39c4c2..00000000000 --- a/modules/dataverse-spi/src/main/java/io/gdcc/spi/export/ExportDataContext.java +++ /dev/null @@ -1,61 +0,0 @@ -package io.gdcc.spi.export; - -/** - * - * @author landreev - * Provides an optional mechanism for defining various data retrieval options - * for the export subsystem in a way that should allow us adding support for - * more options going forward with minimal or no changes to the already - * implemented export plugins. - */ -public class ExportDataContext { - private boolean datasetMetadataOnly = false; - private boolean publicFilesOnly = false; - private Integer offset = null; - private Integer length = null; - - private ExportDataContext() { - - } - - public static ExportDataContext context() { - ExportDataContext context = new ExportDataContext(); - return context; - } - - public ExportDataContext withDatasetMetadataOnly() { - this.datasetMetadataOnly = true; - return this; - } - - public ExportDataContext withPublicFilesOnly() { - this.publicFilesOnly = true; - return this; - } - - public ExportDataContext withOffset(Integer offset) { - this.offset = offset; - return this; - } - - public ExportDataContext withLength(Integer length) { - this.length = length; - return this; - } - - public boolean isDatasetMetadataOnly() { - return datasetMetadataOnly; - } - - public boolean isPublicFilesOnly() { - return publicFilesOnly; - } - - public Integer getOffset() { - return offset; - } - - public Integer getLength() { - return length; - } -} diff --git a/modules/dataverse-spi/src/main/java/io/gdcc/spi/export/ExportDataProvider.java b/modules/dataverse-spi/src/main/java/io/gdcc/spi/export/ExportDataProvider.java deleted file mode 100644 index 282add42c58..00000000000 --- a/modules/dataverse-spi/src/main/java/io/gdcc/spi/export/ExportDataProvider.java +++ /dev/null @@ -1,126 +0,0 @@ -package io.gdcc.spi.export; - -import java.io.InputStream; -import java.util.Optional; - -import jakarta.json.JsonArray; -import jakarta.json.JsonObject; - -/** - * Provides all the metadata Dataverse has about a given dataset that can then - * be used by an @see Exporter to create a new metadata export format. - * - */ -public interface ExportDataProvider { - - /** - * @return - dataset metadata in the standard Dataverse JSON format used in the - * API and available as the JSON metadata export via the user interface. - * @apiNote - there is no JSON schema defining this output, but the format is - * well documented in the Dataverse online guides. This, and the - * OAI_ORE export are the only two that provide 'complete' - * dataset-level metadata along with basic file metadata for each file - * in the dataset. - * @param context - supplies optional parameters. Needs to support - * context.isDatasetMetadataOnly(). In a situation where we - * need to generate a format like DC that has no use for the - * file-level metadata, it makes sense to skip retrieving and - * formatting it, since there can be a very large number of - * files in a dataset. - */ - - JsonObject getDatasetJson(ExportDataContext... context); - - - /** - * - * @return - dataset metadata in the JSON-LD based OAI_ORE format used in - * Dataverse's archival bag export mechanism and as available in the - * user interface and by API. - * @apiNote - THis, and the JSON format are the only two that provide complete - * dataset-level metadata along with basic file metadata for each file - * in the dataset. - * @param context - supplies optional parameters. - */ - JsonObject getDatasetORE(ExportDataContext... context); - - /** - * Dataverse is capable of extracting DDI-centric metadata from tabular - * datafiles. This detailed metadata, which is only available for successfully - * "ingested" tabular files, is not included in the output of any other methods - * in this interface. - * - * @return - a JSONArray with one entry per ingested tabular dataset file. - * @apiNote - there is no JSON schema available for this output and the format - * is not well documented. Implementers may wish to expore the @see - * edu.harvard.iq.dataverse.export.DDIExporter and the @see - * edu.harvard.iq.dataverse.util.json.JSONPrinter classes where this - * output is used/generated (respectively). - * @param context - supplies optional parameters. - */ - JsonArray getDatasetFileDetails(ExportDataContext... context); - - /** - * Similar to the above, but - * a) retrieves the information for the ingested/tabular data files _only_ - * b) provides an option for retrieving this stuff in batches - * c) provides an option for skipping restricted/embargoed etc. files. - * Intended for datasets with massive numbers of tabular files and datavariables. - * @param context - supplies optional parameters. - * current (2.1.0) known use cases: - * context.isPublicFilesOnly(); - * context.getOffset(); - * context.getLength(); - * @return json array containing the datafile/filemetadata->datatable->datavariable metadata - * @throws ExportException - */ - JsonArray getTabularDataDetails(ExportDataContext ... context) throws ExportException; - - - /** - * - * @return - the subset of metadata conforming to the schema.org standard as - * available in the user interface and as included as header metadata in - * dataset pages (for use by search engines) - * @apiNote - as this metadata export is not complete, it should only be used as - * a starting point for an Exporter if it simplifies your exporter - * relative to using the JSON or OAI_ORE exports. - * @param context - supplies optional parameters. - */ - JsonObject getDatasetSchemaDotOrg(ExportDataContext... context); - - /** - * - * @return - the subset of metadata conforming to the DataCite standard as - * available in the Dataverse user interface and as sent to DataCite when DataCite DOIs are used. - * @apiNote - as this metadata export is not complete, it should only be used as - * a starting point for an Exporter if it simplifies your exporter - * relative to using the JSON or OAI_ORE exports. - * @param context - supplies optional parameters. - */ - String getDataCiteXml(ExportDataContext... context); - - /** - * If an Exporter has specified a prerequisite format name via the - * getPrerequisiteFormatName() method, it can call this method to retrieve - * metadata in that format. - * - * @return - metadata in the specified prerequisite format (if available from - * another internal or added Exporter) as an Optional - * @apiNote - This functionality is intended as way to easily generate alternate - * formats of the ~same metadata, e.g. to support download as XML, - * HTML, PDF for a specific metadata standard (e.g. DDI). It can be - * particularly useful, reative to starting from the output of one of - * the getDataset* methods above, if there are existing libraries that - * can convert between these formats. Note that, since Exporters can be - * replaced, relying on this method could cause your Exporter to - * malfunction, e.g. if you depend on format "ddi" and a third party - * Exporter is configured to replace the internal ddi Exporter in - * Dataverse. - * @param context - supplies optional parameters. - */ - default Optional getPrerequisiteInputStream(ExportDataContext... context) { - return Optional.empty(); - } - - } diff --git a/modules/dataverse-spi/src/main/java/io/gdcc/spi/export/ExportException.java b/modules/dataverse-spi/src/main/java/io/gdcc/spi/export/ExportException.java deleted file mode 100644 index c816a605860..00000000000 --- a/modules/dataverse-spi/src/main/java/io/gdcc/spi/export/ExportException.java +++ /dev/null @@ -1,13 +0,0 @@ -package io.gdcc.spi.export; - -import java.io.IOException; - -public class ExportException extends IOException { - public ExportException(String message) { - super(message); - } - - public ExportException(String message, Throwable cause) { - super(message, cause); - } -} diff --git a/modules/dataverse-spi/src/main/java/io/gdcc/spi/export/Exporter.java b/modules/dataverse-spi/src/main/java/io/gdcc/spi/export/Exporter.java deleted file mode 100644 index 7132e74641b..00000000000 --- a/modules/dataverse-spi/src/main/java/io/gdcc/spi/export/Exporter.java +++ /dev/null @@ -1,109 +0,0 @@ -package io.gdcc.spi.export; - -import java.io.OutputStream; -import java.util.Locale; -import java.util.Optional; - - -/** - * Dataverse allows new metadata export formats to be dynamically added a running instance. This is done by - * deploying new classes that implement this Exporter interface. - */ - -public interface Exporter { - - - /** - * When this method is called, the Exporter should write the metadata to the given OutputStream. - * - * @apiNote When implementing exportDataset, when done writing content, please make sure - * to flush() the outputStream, but NOT close() it! This way an exporter can be - * used to insert the produced metadata into the body of an HTTP response, etc. - * (for example, to insert it into the body of an OAI response, where more XML - * needs to be written, for the outer OAI-PMH record). -- L.A. 4.5 - * - * @param dataProvider - the @see ExportDataProvider interface includes several methods that can be used to retrieve the dataset metadata in different formats. An Exporter should use one or more of these to obtain the values needed to generate metadata in the format it supports. - * @param outputStream - the OutputStream to write the metadata to - * @throws ExportException - if there is an error writing the metadata - */ - void exportDataset(ExportDataProvider dataProvider, OutputStream outputStream) throws ExportException; - - /** - * This method should return the name of the metadata format this Exporter - * provides. - * - * @apiNote Format names are unique identifiers for the formats supported in - * Dataverse. Reusing the same format name as another Exporter will - * result only one implementation being available. Exporters packaged - * as an external Jar file have precedence over the default - * implementations in Dataverse. Hence re-using one of the existing - * format names will result in the Exporter replacing the internal one - * with the same name. The precedence between two external Exporters - * using the same format name is not defined. - * Current format names used internally by Dataverse are: - * Datacite - * dcterms - * ddi - * oai_dc - * html - * dataverse_json - * oai_ddi - * OAI_ORE - * oai_datacite - * schema.org - * - * @return - the unique name of the metadata format this Exporter - */ - String getFormatName(); - - /** - * This method should return the display name of the metadata format this - * Exporter provides. Display names are used in the UI, specifically in the menu - * of avaiable Metadata Exports on the dataset page/metadata tab to identify the - * format. - */ - String getDisplayName(Locale locale); - - /** - * Exporters can specify that they require, as input, the output of another - * exporter. This is done by providing the name of that format in response to a - * call to this method. - * - * @implNote The one current example where this is done is with the html(display - * name "DDI html codebook") exporter which starts from the XML-based - * ddi format produced by that exporter. - * @apiNote - The Exporter can expect that the metadata produced by its - * prerequisite exporter (as defined with this method) will be - * available via the ExportDataProvider.getPrerequisiteInputStream() - * method. The default implementation of this method returns an empty - * value which means the getPrerequisiteInputStream() method of the - * ExportDataProvider sent in the exportDataset method will return an - * empty Optional. - * - */ - default Optional getPrerequisiteFormatName() { - return Optional.empty(); - } - - /** - * Harvestable Exporters will be available as options in Dataverse's Harvesting mechanism. - * @return true to make this exporter available as a harvesting option. - */ - Boolean isHarvestable(); - - /** - * If an Exporter is available to users, its format will be generated for every - * published dataset and made available via the dataset page/metadata - * tab/Metadata Exports menu item and via the API. - * @return true to make this exporter available to users. - */ - Boolean isAvailableToUsers(); - - /** - * To support effective downloads of metadata in this Exporter's format, the Exporter should specify an appropriate mime type. - * @apiNote - It is recommended to used the @see javax.ws.rs.core.MediaType enum to specify the mime type. - * @return The mime type, e.g. "application/json", "text/plain", etc. - */ - String getMediaType(); - -} diff --git a/modules/dataverse-spi/src/main/java/io/gdcc/spi/export/XMLExporter.java b/modules/dataverse-spi/src/main/java/io/gdcc/spi/export/XMLExporter.java deleted file mode 100644 index 3c3fa35c69d..00000000000 --- a/modules/dataverse-spi/src/main/java/io/gdcc/spi/export/XMLExporter.java +++ /dev/null @@ -1,37 +0,0 @@ -package io.gdcc.spi.export; - -import jakarta.ws.rs.core.MediaType; - -/** - * XML Exporter is an extension of the base Exporter interface that adds the - * additional methods needed for generating XML metadata export formats. - */ -public interface XMLExporter extends Exporter { - - /** - * @implNote for the ddi exporter, this method returns "ddi:codebook:2_5" - * @return - the name space of the XML schema - */ - String getXMLNameSpace(); - - /** - * @apiNote According to the XML specification, the value must be a URI - * @implNote for the ddi exporter, this method returns - * "https://ddialliance.org/Specification/DDI-Codebook/2.5/XMLSchema/codebook.xsd" - * @return - the location of the XML schema as a String (must be a valid URI) - */ - String getXMLSchemaLocation(); - - /** - * @implNote for the ddi exporter, this method returns "2.5" - * @return - the version of the XML schema - */ - String getXMLSchemaVersion(); - - /** - * @return - should always be MediaType.APPLICATION_XML - */ - public default String getMediaType() { - return MediaType.APPLICATION_XML; - }; -} From 3d318be51014cf1d4a339126f63562a79c32ab0a Mon Sep 17 00:00:00 2001 From: Leonid Andreev Date: Tue, 21 Jul 2026 15:02:23 -0400 Subject: [PATCH 25/25] ... failed to add this change to the commit; unit tests have just failed on the pr on account of it. #11405 --- modules/dataverse-parent/pom.xml | 1 - 1 file changed, 1 deletion(-) diff --git a/modules/dataverse-parent/pom.xml b/modules/dataverse-parent/pom.xml index 33b42621c5a..85a92e0df72 100644 --- a/modules/dataverse-parent/pom.xml +++ b/modules/dataverse-parent/pom.xml @@ -14,7 +14,6 @@ ../../pom.xml ../../scripts/zipdownload ../container-base - ../dataverse-spi