diff --git a/.github/workflows/spi_release.yml b/.github/workflows/spi_release.yml deleted file mode 100644 index 8fe80925b84..00000000000 --- a/.github/workflows/spi_release.yml +++ /dev/null @@ -1,94 +0,0 @@ -name: Dataverse SPI - -on: - push: - branches: - - "develop" - paths: - - "modules/dataverse-spi/**" - pull_request: - branches: - - "develop" - paths: - - "modules/dataverse-spi/**" - -jobs: - # Note: Pushing packages to Maven Central requires access to secrets, which pull requests from remote forks - # don't have. Skip in these cases. - check-secrets: - name: Check for Secrets Availability - runs-on: ubuntu-latest - outputs: - available: ${{ steps.secret-check.outputs.available }} - steps: - - id: secret-check - # perform secret check & put boolean result as an output - shell: bash - run: | - if [ "${{ secrets.DATAVERSEBOT_SONATYPE_USERNAME }}" != '' ]; then - echo "available=true" >> $GITHUB_OUTPUT; - else - echo "available=false" >> $GITHUB_OUTPUT; - fi - - snapshot: - name: Release Snapshot - needs: check-secrets - runs-on: ubuntu-latest - if: github.event_name == 'pull_request' && needs.check-secrets.outputs.available == 'true' - steps: - - uses: actions/checkout@v7 - - uses: actions/setup-java@v5 - with: - java-version: '21' - distribution: 'adopt' - server-id: central - server-username: MAVEN_USERNAME - server-password: MAVEN_PASSWORD - - uses: actions/cache@v6 - with: - path: ~/.m2 - key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }} - restore-keys: ${{ runner.os }}-m2 - - - name: Deploy Snapshot - run: mvn -f modules/dataverse-spi -Dproject.version.suffix="-PR${{ github.event.number }}-SNAPSHOT" deploy - env: - MAVEN_USERNAME: ${{ secrets.DATAVERSEBOT_SONATYPE_USERNAME }} - MAVEN_PASSWORD: ${{ secrets.DATAVERSEBOT_SONATYPE_TOKEN }} - - release: - name: Release - needs: check-secrets - runs-on: ubuntu-latest - if: github.event_name == 'push' && needs.check-secrets.outputs.available == 'true' - steps: - - uses: actions/checkout@v7 - - uses: actions/setup-java@v5 - with: - java-version: '21' - distribution: 'adopt' - - uses: actions/cache@v6 - with: - path: ~/.m2 - key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }} - restore-keys: ${{ runner.os }}-m2 - - # Running setup-java again overwrites the settings.xml - IT'S MANDATORY TO DO THIS SECOND SETUP!!! - - name: Set up Maven Central Repository - uses: actions/setup-java@v5 - with: - java-version: '21' - distribution: 'adopt' - server-id: central - server-username: MAVEN_USERNAME - server-password: MAVEN_PASSWORD - gpg-private-key: ${{ secrets.DATAVERSEBOT_GPG_KEY }} - gpg-passphrase: MAVEN_GPG_PASSPHRASE - - - name: Sign + Publish Release - run: mvn -f modules/dataverse-spi -P release deploy - env: - MAVEN_USERNAME: ${{ secrets.DATAVERSEBOT_SONATYPE_USERNAME }} - MAVEN_PASSWORD: ${{ secrets.DATAVERSEBOT_SONATYPE_TOKEN }} - MAVEN_GPG_PASSPHRASE: ${{ secrets.DATAVERSEBOT_GPG_PASSWORD }} diff --git a/modules/dataverse-parent/pom.xml b/modules/dataverse-parent/pom.xml index a31b01328ae..85a92e0df72 100644 --- a/modules/dataverse-parent/pom.xml +++ b/modules/dataverse-parent/pom.xml @@ -14,7 +14,6 @@ ../../pom.xml ../../scripts/zipdownload ../container-base - ../dataverse-spi - - - ossrh - - https://s01.oss.sonatype.org/service/local/staging/deploy/maven2/ - - - - - - - none - false - - - - - jakarta.json - jakarta.json-api - provided - - - - jakarta.ws.rs - jakarta.ws.rs-api - provided - - - - - - - - maven-compiler-plugin - - ${target.java.version} - - - - - - org.sonatype.plugins - nexus-staging-maven-plugin - true - - - ossrh - - https://s01.oss.sonatype.org - true - - - - org.apache.maven.plugins - maven-release-plugin - - false - release - true - deploy - - - - org.codehaus.mojo - flatten-maven-plugin - - true - oss - - remove - remove - - - - - - flatten - process-resources - - flatten - - - - - flatten.clean - clean - - clean - - - - - - org.apache.maven.plugins - maven-deploy-plugin - - ${skipDeploy} - - - - - - - - release - - - - org.apache.maven.plugins - maven-gpg-plugin - - - sign-artifacts - verify - - sign - - - - - - org.kordamp.maven - pomchecker-maven-plugin - - - process-resources - - check-maven-central - - - - - - - org.apache.maven.plugins - maven-javadoc-plugin - - - attach-javadocs - - jar - - - - - ${target.java.version} - false - ${javadoc.lint} - - - - org.apache.maven.plugins - maven-source-plugin - - - attach-sources - - jar - - - - - - - - - ct - - true - - - - diff --git a/modules/dataverse-spi/src/main/java/io/gdcc/spi/export/ExportDataContext.java b/modules/dataverse-spi/src/main/java/io/gdcc/spi/export/ExportDataContext.java deleted file mode 100644 index 9478d39c4c2..00000000000 --- a/modules/dataverse-spi/src/main/java/io/gdcc/spi/export/ExportDataContext.java +++ /dev/null @@ -1,61 +0,0 @@ -package io.gdcc.spi.export; - -/** - * - * @author landreev - * Provides an optional mechanism for defining various data retrieval options - * for the export subsystem in a way that should allow us adding support for - * more options going forward with minimal or no changes to the already - * implemented export plugins. - */ -public class ExportDataContext { - private boolean datasetMetadataOnly = false; - private boolean publicFilesOnly = false; - private Integer offset = null; - private Integer length = null; - - private ExportDataContext() { - - } - - public static ExportDataContext context() { - ExportDataContext context = new ExportDataContext(); - return context; - } - - public ExportDataContext withDatasetMetadataOnly() { - this.datasetMetadataOnly = true; - return this; - } - - public ExportDataContext withPublicFilesOnly() { - this.publicFilesOnly = true; - return this; - } - - public ExportDataContext withOffset(Integer offset) { - this.offset = offset; - return this; - } - - public ExportDataContext withLength(Integer length) { - this.length = length; - return this; - } - - public boolean isDatasetMetadataOnly() { - return datasetMetadataOnly; - } - - public boolean isPublicFilesOnly() { - return publicFilesOnly; - } - - public Integer getOffset() { - return offset; - } - - public Integer getLength() { - return length; - } -} diff --git a/modules/dataverse-spi/src/main/java/io/gdcc/spi/export/ExportDataProvider.java b/modules/dataverse-spi/src/main/java/io/gdcc/spi/export/ExportDataProvider.java deleted file mode 100644 index 4197d978e79..00000000000 --- a/modules/dataverse-spi/src/main/java/io/gdcc/spi/export/ExportDataProvider.java +++ /dev/null @@ -1,123 +0,0 @@ -package io.gdcc.spi.export; - -import java.io.InputStream; -import java.util.Optional; - -import jakarta.json.JsonArray; -import jakarta.json.JsonObject; - -/** - * Provides all the metadata Dataverse has about a given dataset that can then - * be used by an @see Exporter to create a new metadata export format. - * - */ -public interface ExportDataProvider { - - /** - * @return - dataset metadata in the standard Dataverse JSON format used in the - * API and available as the JSON metadata export via the user interface. - * @apiNote - there is no JSON schema defining this output, but the format is - * well documented in the Dataverse online guides. This, and the - * OAI_ORE export are the only two that provide 'complete' - * dataset-level metadata along with basic file metadata for each file - * in the dataset. - * @param context - supplies optional parameters. Needs to support - * context.isDatasetMetadataOnly(). In a situation where we - * need to generate a format like DC that has no use for the - * file-level metadata, it makes sense to skip retrieving and - * formatting it, since there can be a very large number of - * files in a dataset. - */ - JsonObject getDatasetJson(ExportDataContext... context); - - /** - * - * @return - dataset metadata in the JSON-LD based OAI_ORE format used in - * Dataverse's archival bag export mechanism and as available in the - * user interface and by API. - * @apiNote - THis, and the JSON format are the only two that provide complete - * dataset-level metadata along with basic file metadata for each file - * in the dataset. - * @param context - supplies optional parameters. - */ - JsonObject getDatasetORE(ExportDataContext... context); - - /** - * Dataverse is capable of extracting DDI-centric metadata from tabular - * datafiles. This detailed metadata, which is only available for successfully - * "ingested" tabular files, is not included in the output of any other methods - * in this interface. - * - * @return - a JSONArray with one entry per ingested tabular dataset file. - * @apiNote - there is no JSON schema available for this output and the format - * is not well documented. Implementers may wish to expore the @see - * edu.harvard.iq.dataverse.export.DDIExporter and the @see - * edu.harvard.iq.dataverse.util.json.JSONPrinter classes where this - * output is used/generated (respectively). - * @param context - supplies optional parameters. - */ - JsonArray getDatasetFileDetails(ExportDataContext... context); - - /** - * Similar to the above, but - * a) retrieves the information for the ingested/tabular data files _only_ - * b) provides an option for retrieving this stuff in batches - * c) provides an option for skipping restricted/embargoed etc. files. - * Intended for datasets with massive numbers of tabular files and datavariables. - * @param context - supplies optional parameters. - * current (2.1.0) known use cases: - * context.isPublicFilesOnly(); - * context.getOffset(); - * context.getLength(); - * @return json array containing the datafile/filemetadata->datatable->datavariable metadata - * @throws ExportException - */ - JsonArray getTabularDataDetails(ExportDataContext ... context) throws ExportException; - - /** - * - * @return - the subset of metadata conforming to the schema.org standard as - * available in the user interface and as included as header metadata in - * dataset pages (for use by search engines) - * @apiNote - as this metadata export is not complete, it should only be used as - * a starting point for an Exporter if it simplifies your exporter - * relative to using the JSON or OAI_ORE exports. - * @param context - supplies optional parameters. - */ - JsonObject getDatasetSchemaDotOrg(ExportDataContext... context); - - /** - * - * @return - the subset of metadata conforming to the DataCite standard as - * available in the Dataverse user interface and as sent to DataCite when DataCite DOIs are used. - * @apiNote - as this metadata export is not complete, it should only be used as - * a starting point for an Exporter if it simplifies your exporter - * relative to using the JSON or OAI_ORE exports. - * @param context - supplies optional parameters. - */ - String getDataCiteXml(ExportDataContext... context); - - /** - * If an Exporter has specified a prerequisite format name via the - * getPrerequisiteFormatName() method, it can call this method to retrieve - * metadata in that format. - * - * @return - metadata in the specified prerequisite format (if available from - * another internal or added Exporter) as an Optional - * @apiNote - This functionality is intended as way to easily generate alternate - * formats of the ~same metadata, e.g. to support download as XML, - * HTML, PDF for a specific metadata standard (e.g. DDI). It can be - * particularly useful, reative to starting from the output of one of - * the getDataset* methods above, if there are existing libraries that - * can convert between these formats. Note that, since Exporters can be - * replaced, relying on this method could cause your Exporter to - * malfunction, e.g. if you depend on format "ddi" and a third party - * Exporter is configured to replace the internal ddi Exporter in - * Dataverse. - * @param context - supplies optional parameters. - */ - default Optional getPrerequisiteInputStream(ExportDataContext... context) { - return Optional.empty(); - } - - } diff --git a/modules/dataverse-spi/src/main/java/io/gdcc/spi/export/ExportException.java b/modules/dataverse-spi/src/main/java/io/gdcc/spi/export/ExportException.java deleted file mode 100644 index c816a605860..00000000000 --- a/modules/dataverse-spi/src/main/java/io/gdcc/spi/export/ExportException.java +++ /dev/null @@ -1,13 +0,0 @@ -package io.gdcc.spi.export; - -import java.io.IOException; - -public class ExportException extends IOException { - public ExportException(String message) { - super(message); - } - - public ExportException(String message, Throwable cause) { - super(message, cause); - } -} diff --git a/modules/dataverse-spi/src/main/java/io/gdcc/spi/export/Exporter.java b/modules/dataverse-spi/src/main/java/io/gdcc/spi/export/Exporter.java deleted file mode 100644 index 7132e74641b..00000000000 --- a/modules/dataverse-spi/src/main/java/io/gdcc/spi/export/Exporter.java +++ /dev/null @@ -1,109 +0,0 @@ -package io.gdcc.spi.export; - -import java.io.OutputStream; -import java.util.Locale; -import java.util.Optional; - - -/** - * Dataverse allows new metadata export formats to be dynamically added a running instance. This is done by - * deploying new classes that implement this Exporter interface. - */ - -public interface Exporter { - - - /** - * When this method is called, the Exporter should write the metadata to the given OutputStream. - * - * @apiNote When implementing exportDataset, when done writing content, please make sure - * to flush() the outputStream, but NOT close() it! This way an exporter can be - * used to insert the produced metadata into the body of an HTTP response, etc. - * (for example, to insert it into the body of an OAI response, where more XML - * needs to be written, for the outer OAI-PMH record). -- L.A. 4.5 - * - * @param dataProvider - the @see ExportDataProvider interface includes several methods that can be used to retrieve the dataset metadata in different formats. An Exporter should use one or more of these to obtain the values needed to generate metadata in the format it supports. - * @param outputStream - the OutputStream to write the metadata to - * @throws ExportException - if there is an error writing the metadata - */ - void exportDataset(ExportDataProvider dataProvider, OutputStream outputStream) throws ExportException; - - /** - * This method should return the name of the metadata format this Exporter - * provides. - * - * @apiNote Format names are unique identifiers for the formats supported in - * Dataverse. Reusing the same format name as another Exporter will - * result only one implementation being available. Exporters packaged - * as an external Jar file have precedence over the default - * implementations in Dataverse. Hence re-using one of the existing - * format names will result in the Exporter replacing the internal one - * with the same name. The precedence between two external Exporters - * using the same format name is not defined. - * Current format names used internally by Dataverse are: - * Datacite - * dcterms - * ddi - * oai_dc - * html - * dataverse_json - * oai_ddi - * OAI_ORE - * oai_datacite - * schema.org - * - * @return - the unique name of the metadata format this Exporter - */ - String getFormatName(); - - /** - * This method should return the display name of the metadata format this - * Exporter provides. Display names are used in the UI, specifically in the menu - * of avaiable Metadata Exports on the dataset page/metadata tab to identify the - * format. - */ - String getDisplayName(Locale locale); - - /** - * Exporters can specify that they require, as input, the output of another - * exporter. This is done by providing the name of that format in response to a - * call to this method. - * - * @implNote The one current example where this is done is with the html(display - * name "DDI html codebook") exporter which starts from the XML-based - * ddi format produced by that exporter. - * @apiNote - The Exporter can expect that the metadata produced by its - * prerequisite exporter (as defined with this method) will be - * available via the ExportDataProvider.getPrerequisiteInputStream() - * method. The default implementation of this method returns an empty - * value which means the getPrerequisiteInputStream() method of the - * ExportDataProvider sent in the exportDataset method will return an - * empty Optional. - * - */ - default Optional getPrerequisiteFormatName() { - return Optional.empty(); - } - - /** - * Harvestable Exporters will be available as options in Dataverse's Harvesting mechanism. - * @return true to make this exporter available as a harvesting option. - */ - Boolean isHarvestable(); - - /** - * If an Exporter is available to users, its format will be generated for every - * published dataset and made available via the dataset page/metadata - * tab/Metadata Exports menu item and via the API. - * @return true to make this exporter available to users. - */ - Boolean isAvailableToUsers(); - - /** - * To support effective downloads of metadata in this Exporter's format, the Exporter should specify an appropriate mime type. - * @apiNote - It is recommended to used the @see javax.ws.rs.core.MediaType enum to specify the mime type. - * @return The mime type, e.g. "application/json", "text/plain", etc. - */ - String getMediaType(); - -} diff --git a/modules/dataverse-spi/src/main/java/io/gdcc/spi/export/XMLExporter.java b/modules/dataverse-spi/src/main/java/io/gdcc/spi/export/XMLExporter.java deleted file mode 100644 index 3c3fa35c69d..00000000000 --- a/modules/dataverse-spi/src/main/java/io/gdcc/spi/export/XMLExporter.java +++ /dev/null @@ -1,37 +0,0 @@ -package io.gdcc.spi.export; - -import jakarta.ws.rs.core.MediaType; - -/** - * XML Exporter is an extension of the base Exporter interface that adds the - * additional methods needed for generating XML metadata export formats. - */ -public interface XMLExporter extends Exporter { - - /** - * @implNote for the ddi exporter, this method returns "ddi:codebook:2_5" - * @return - the name space of the XML schema - */ - String getXMLNameSpace(); - - /** - * @apiNote According to the XML specification, the value must be a URI - * @implNote for the ddi exporter, this method returns - * "https://ddialliance.org/Specification/DDI-Codebook/2.5/XMLSchema/codebook.xsd" - * @return - the location of the XML schema as a String (must be a valid URI) - */ - String getXMLSchemaLocation(); - - /** - * @implNote for the ddi exporter, this method returns "2.5" - * @return - the version of the XML schema - */ - String getXMLSchemaVersion(); - - /** - * @return - should always be MediaType.APPLICATION_XML - */ - public default String getMediaType() { - return MediaType.APPLICATION_XML; - }; -} diff --git a/pom.xml b/pom.xml index 50c44586ca3..daeb4da5d7b 100644 --- a/pom.xml +++ b/pom.xml @@ -20,7 +20,7 @@ false false - integration,migration + integration,migration,performance -Ddummy.jacoco.property=true @@ -694,7 +694,7 @@ io.gdcc dataverse-spi - 2.0.0 + 2.1.0-SNAPSHOT javax.cache diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/DatasetServiceBean.java index dc09d8948a8..4c3db2b61b8 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DatasetServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/DatasetServiceBean.java @@ -775,7 +775,7 @@ private void exportAllDatasets(boolean forceReExport, Date reExportDate) { || dataset.getLastExportTime().before(publicationDate))))) { countAll++; try { - recordService.exportAllFormatsInNewTransaction(dataset); + recordService.exportFormatsInNewTransaction(dataset, null); exportLogger.info("Success exporting dataset: " + dataset.getDisplayName() + " " + dataset.getGlobalId().asString()); countSuccess++; } catch (Exception ex) { @@ -803,13 +803,18 @@ private void exportAllDatasets(boolean forceReExport, Date reExportDate) { @Asynchronous public void reExportDatasetAsync(Dataset dataset) { - exportDataset(dataset, true); + exportDataset(dataset, true, null); + } + + @Asynchronous + public void reExportDatasetAsync(Dataset dataset, List formatNames) { + exportDataset(dataset, true, formatNames); } - public void exportDataset(Dataset dataset, boolean forceReExport) { + private void exportDataset(Dataset dataset, boolean forceReExport, List formatNames) { if (dataset != null) { // Note that the logic for handling a dataset is similar to what is implemented in exportAllDatasets, - // but when only one dataset is exported we do not log in a separate export logging file + // but when only one dataset is exported we do not use a dedicated log file if (dataset.isReleased() && dataset.getReleasedVersion() != null && !dataset.isDeaccessioned()) { // can't trust dataset.getPublicationDate(), no. @@ -818,7 +823,7 @@ public void exportDataset(Dataset dataset, boolean forceReExport) { && (dataset.getLastExportTime() == null || dataset.getLastExportTime().before(publicationDate)))) { try { - recordService.exportAllFormatsInNewTransaction(dataset); + recordService.exportFormatsInNewTransaction(dataset, formatNames); logger.info("Success exporting dataset: " + dataset.getDisplayName() + " " + dataset.getGlobalId().asString()); } catch (Exception ex) { logger.log(Level.INFO, "Error exporting dataset: " + dataset.getDisplayName() + " " + dataset.getGlobalId().asString() + "; " + ex.getMessage(), ex); diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetVersion.java b/src/main/java/edu/harvard/iq/dataverse/DatasetVersion.java index c45ddcd62cb..7b807811848 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DatasetVersion.java +++ b/src/main/java/edu/harvard/iq/dataverse/DatasetVersion.java @@ -136,7 +136,7 @@ public enum VersionState { private Dataset dataset; @OneToMany(mappedBy = "datasetVersion", cascade = {CascadeType.REMOVE, CascadeType.MERGE, CascadeType.PERSIST}) - @OrderBy("label") // this is not our preferred ordering, which is with the AlphaNumericComparator, but does allow the files to be grouped by category + @OrderBy("label, id") // this is not our preferred ordering, which is with the AlphaNumericComparator, but does allow the files to be grouped by category; adding id, to avoid ambiguity when there are duplicate filenames. (L.A., 05-2026) private List fileMetadatas = new ArrayList(); @OneToOne(cascade = {CascadeType.MERGE, CascadeType.PERSIST, CascadeType.REMOVE}, orphanRemoval=true) diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetVersionFilesServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/DatasetVersionFilesServiceBean.java index 27c91e8b312..74c3e833753 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DatasetVersionFilesServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/DatasetVersionFilesServiceBean.java @@ -181,6 +181,61 @@ public List getFileMetadatas(DatasetVersion datasetVersion, Intege return typedQuery.getResultList(); } + /** + * Similar to the above, but dedicated for retrieving FileMetadatas of only + * tabular datafiles in the specified DatasetVersion. Used in the metadata + * export subsystem. + * + * @param datasetVersion the DatasetVersion to access + * @param limit for pagination, can be null + * @param offset for pagination, can be null + * @param publicFilesOnly skip restricted, embargoed etc. files + * @return a FileMetadata list from the specified DatasetVersion + */ + public List getTabularDataFileMetadatas(DatasetVersion datasetVersion, Integer limit, Integer offset, boolean publicFilesOnly) { + CriteriaBuilder criteriaBuilder = em.getCriteriaBuilder(); + CriteriaQuery criteriaQuery = criteriaBuilder.createQuery(FileMetadata.class); + + Root fileMetadataRoot = criteriaQuery.from(FileMetadata.class); + Predicate basePredicate = criteriaBuilder.equal(fileMetadataRoot.get("datasetVersion").get("id"), datasetVersion.getId()); + + Root dataTableRoot = criteriaQuery.from(DataTable.class); + Predicate tabularPredicate = criteriaBuilder.equal(dataTableRoot.get("dataFile"), fileMetadataRoot.get("dataFile")); + + Predicate combinedPredicate; + + if (publicFilesOnly) { + combinedPredicate = criteriaBuilder.and(basePredicate, tabularPredicate); + } else { + combinedPredicate = criteriaBuilder.and(basePredicate, + tabularPredicate, + createSearchCriteriaAccessStatusPredicate(FileSearchCriteria.FileAccessStatus.Public, + criteriaBuilder, + fileMetadataRoot)); + } + + List orderList = new ArrayList<>(); + + // Ordering the resulting fileMetadatas by label AND id, to avoid any ambiguity when there are duplicate filenames in the version + orderList.add(criteriaBuilder.asc(fileMetadataRoot.get("label"))); + orderList.add(criteriaBuilder.asc(fileMetadataRoot.get("id"))); + + criteriaQuery + .select(fileMetadataRoot) + .where(combinedPredicate) + .orderBy(orderList); + + TypedQuery typedQuery = em.createQuery(criteriaQuery); + if (limit != null) { + typedQuery.setMaxResults(limit); + } + if (offset != null) { + typedQuery.setFirstResult(offset); + } + + return typedQuery.getResultList(); + } + /** * Returns the total download size of all files for a particular DatasetVersion * @@ -225,6 +280,16 @@ public boolean isDataFilePresentInDatasetVersion(DatasetVersion datasetVersion, return count != null && count > 0; } + /** + * This is strictly for use in IT tests! + * + * @param em EntityManager, such as dataverse.util.testing.performance.JpaEntityManagerService.createEntityManager() + * + */ + public void injectEntityManager(EntityManager em) { + this.em = em; + } + private void addAccessStatusCountToTotal(DatasetVersion datasetVersion, Map totalCounts, FileAccessStatus dataFileAccessStatus, FileSearchCriteria searchCriteria) { long fileMetadataCount = getFileMetadataCountByAccessStatus(datasetVersion, dataFileAccessStatus, searchCriteria); if (fileMetadataCount > 0) { diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java index 56ed1b88cb0..6513afef312 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java @@ -569,7 +569,7 @@ public Response getVersion(@Context ContainerRequestContext crc, } JsonObjectBuilder jsonBuilder = json(requestedDatasetVersion, null, includeFiles, - returnOwners, includeMetadataBlocks, ignoreSettingExcludeEmailFromExport); + returnOwners, includeMetadataBlocks, false, ignoreSettingExcludeEmailFromExport); return ok(jsonBuilder); }, getRequestUser(crc)); diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Files.java b/src/main/java/edu/harvard/iq/dataverse/api/Files.java index 687ccadc36f..6f07fcbb45e 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Files.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Files.java @@ -1133,7 +1133,7 @@ public Response getFileDataTables(@Context ContainerRequestContext crc, if (!dataFile.isTabularData()) { return badRequest(BundleUtil.getStringFromBundle("files.api.only.tabular.supported")); } - return ok(jsonDT(dataFile.getDataTables())); + return ok(jsonDT(dataFile.getDataTables(), true)); } @POST diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Metadata.java b/src/main/java/edu/harvard/iq/dataverse/api/Metadata.java index b57dc18412c..d590925f61b 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Metadata.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Metadata.java @@ -17,6 +17,9 @@ import edu.harvard.iq.dataverse.harvest.server.OAISetServiceBean; import edu.harvard.iq.dataverse.harvest.server.OAISet; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; import org.eclipse.microprofile.openapi.annotations.Operation; import org.eclipse.microprofile.openapi.annotations.parameters.Parameter; import org.eclipse.microprofile.openapi.annotations.tags.Tag; @@ -86,10 +89,15 @@ public Response reExportAll( description = "Starts a background metadata re-export for the specified dataset.") public Response indexDatasetByPersistentId( @Parameter(description = "Dataset id or persistent identifier to re-export.", required = true) - @PathParam("id") String id) { + @PathParam("id") String id, + @QueryParam("formats") String formats) { try { Dataset dataset = findDatasetOrDie(id); - datasetService.reExportDatasetAsync(dataset); + List formatNames = null; + if (formats != null) { + formatNames = new ArrayList<>(Arrays.asList(formats.split(","))); + } + datasetService.reExportDatasetAsync(dataset, formatNames); return ok("export started"); } catch (WrappedResponse wr) { return wr.getResponse(); @@ -116,8 +124,7 @@ public Response clearExportTimestamps() { description = "Marks the specified OAI set as updating and starts its metadata export in the background.") public Response exportOaiSet( @Parameter(description = "OAI set specification name to export.", required = true) - @PathParam("specname") String spec) - { + @PathParam("specname") String spec) { // assuming this belongs here (because it's a metadata export), but open to moving it elsewhere OAISet set = null; try diff --git a/src/main/java/edu/harvard/iq/dataverse/api/dto/DataFileDTO.java b/src/main/java/edu/harvard/iq/dataverse/api/dto/DataFileDTO.java index 318bad4f3a3..f4537b0b7ad 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/dto/DataFileDTO.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/dto/DataFileDTO.java @@ -20,6 +20,7 @@ public class DataFileDTO { private String md5; private String description; private String pidURL; + private List tabularTags; public String getPidURL() { return pidURL; @@ -119,5 +120,11 @@ public void setDescription(String description) { this.description = description; } + public List getTabularTags() { + return tabularTags; + } + public void setTabularTags(List tabularTags) { + this.tabularTags = tabularTags; + } } diff --git a/src/main/java/edu/harvard/iq/dataverse/export/DCTermsExporter.java b/src/main/java/edu/harvard/iq/dataverse/export/DCTermsExporter.java index f82c0d9ad3d..75d9360efd3 100644 --- a/src/main/java/edu/harvard/iq/dataverse/export/DCTermsExporter.java +++ b/src/main/java/edu/harvard/iq/dataverse/export/DCTermsExporter.java @@ -8,6 +8,7 @@ import io.gdcc.spi.export.Exporter; import io.gdcc.spi.export.XMLExporter; import edu.harvard.iq.dataverse.util.BundleUtil; +//import io.gdcc.spi.export.ExportDataContext; import java.io.OutputStream; import java.util.Locale; import java.util.Optional; @@ -38,7 +39,8 @@ public String getDisplayName(Locale locale) { @Override public void exportDataset(ExportDataProvider dataProvider, OutputStream outputStream) throws ExportException { try { - DublinCoreExportUtil.datasetJson2dublincore(dataProvider.getDatasetJson(), outputStream, DublinCoreExportUtil.DC_FLAVOR_DCTERMS); + // @todo add + DublinCoreExportUtil.datasetJson2dublincore(dataProvider.getDatasetJson(/*ExportDataContext.context().withDatasetMetadataOnly()*/), outputStream, DublinCoreExportUtil.DC_FLAVOR_DCTERMS); } catch (XMLStreamException xse) { throw new ExportException("Caught XMLStreamException performing DCTERMS export", xse); } diff --git a/src/main/java/edu/harvard/iq/dataverse/export/DDIExporter.java b/src/main/java/edu/harvard/iq/dataverse/export/DDIExporter.java index 0130c18b22b..e03363a155e 100644 --- a/src/main/java/edu/harvard/iq/dataverse/export/DDIExporter.java +++ b/src/main/java/edu/harvard/iq/dataverse/export/DDIExporter.java @@ -48,7 +48,7 @@ public String getDisplayName(Locale locale) { @Override public void exportDataset(ExportDataProvider dataProvider, OutputStream outputStream) throws ExportException { try { - DdiExportUtil.datasetJson2ddi(dataProvider.getDatasetJson(), dataProvider.getDatasetFileDetails(), + DdiExportUtil.datasetJson2ddi(dataProvider.getDatasetJson(), dataProvider, outputStream); } catch (XMLStreamException xse) { throw new ExportException("Caught XMLStreamException performing DDI export", xse); diff --git a/src/main/java/edu/harvard/iq/dataverse/export/DublinCoreExporter.java b/src/main/java/edu/harvard/iq/dataverse/export/DublinCoreExporter.java index 0fa32dd4bfa..d368fbe214b 100644 --- a/src/main/java/edu/harvard/iq/dataverse/export/DublinCoreExporter.java +++ b/src/main/java/edu/harvard/iq/dataverse/export/DublinCoreExporter.java @@ -5,6 +5,7 @@ import edu.harvard.iq.dataverse.export.dublincore.DublinCoreExportUtil; import io.gdcc.spi.export.ExportDataProvider; import io.gdcc.spi.export.ExportException; +//import io.gdcc.spi.export.ExportDataContext; import io.gdcc.spi.export.Exporter; import io.gdcc.spi.export.XMLExporter; import edu.harvard.iq.dataverse.util.BundleUtil; @@ -38,7 +39,7 @@ public String getDisplayName(Locale locale) { @Override public void exportDataset(ExportDataProvider dataProvider, OutputStream outputStream) throws ExportException { try { - DublinCoreExportUtil.datasetJson2dublincore(dataProvider.getDatasetJson(), outputStream, + DublinCoreExportUtil.datasetJson2dublincore(dataProvider.getDatasetJson(/*ExportDataContext.context().withDatasetMetadataOnly()*/), outputStream, DublinCoreExportUtil.DC_FLAVOR_OAI); } catch (XMLStreamException xse) { throw new ExportException("Caught XMLStreamException performing DC export", xse); diff --git a/src/main/java/edu/harvard/iq/dataverse/export/ExportService.java b/src/main/java/edu/harvard/iq/dataverse/export/ExportService.java index e7bcf17d44b..5d5863798aa 100644 --- a/src/main/java/edu/harvard/iq/dataverse/export/ExportService.java +++ b/src/main/java/edu/harvard/iq/dataverse/export/ExportService.java @@ -64,7 +64,7 @@ public class ExportService { private Map exporterMap = new HashMap<>(); private static final Logger logger = Logger.getLogger(ExportService.class.getCanonicalName()); - + private ExportService() { /* * Step 1 - find the EXPORTERS dir and add all jar files there to a class loader @@ -319,16 +319,72 @@ public void exportAllFormats(Dataset dataset) throws ExportException { } } + + public void exportFormats(Dataset dataset, List formatNames) throws ExportException { + try { + if (formatNames == null) { + clearAllCachedFormats(dataset); + } else { + clearCachedFormats(dataset, formatNames); + } + } catch (IOException ex) { + Logger.getLogger(ExportService.class.getName()).log(Level.SEVERE, null, ex); + } - public void clearAllCachedFormats(Dataset dataset) throws IOException { try { + DatasetVersion releasedVersion = dataset.getReleasedVersion(); + if (releasedVersion == null) { + throw new ExportException("No released version for dataset " + dataset.getGlobalId().toString()); + } + InternalExportDataProvider dataProvider = new InternalExportDataProvider(releasedVersion); for (Exporter e : exporterMap.values()) { String formatName = e.getFormatName(); - clearCachedExport(dataset, formatName); + if (formatNames == null || formatNames.contains(formatName)) { + if (e.getPrerequisiteFormatName().isPresent()) { + String prereqFormatName = e.getPrerequisiteFormatName().get(); + try (InputStream preReqStream = getExport(dataset.getReleasedVersion(), prereqFormatName)) { + dataProvider.setPrerequisiteInputStream(preReqStream); + cacheExport(dataset, dataProvider, formatName, e); + dataProvider.setPrerequisiteInputStream(null); + } catch (IOException ioe) { + throw new ExportException("Could not get prerequisite " + e.getPrerequisiteFormatName() + " to create " + formatName + "export for dataset " + dataset.getId(), ioe); + } + } else { + cacheExport(dataset, dataProvider, formatName, e); + } + } } + // Finally, if we have been able to successfully export in all available + // formats, we'll increment the "last exported" time stamp: + dataset.setLastExportTime(new Timestamp(new Date().getTime())); + + } catch (ServiceConfigurationError serviceError) { + throw new ExportException("Service configuration error during export. " + serviceError.getMessage()); + } catch (RuntimeException e) { + logger.log(Level.FINE, e.getMessage(), e); + throw new ExportException( + "Unknown runtime exception exporting metadata. " + (e.getMessage() == null ? "" : e.getMessage())); + } + } - dataset.setLastExportTime(null); + public void clearAllCachedFormats(Dataset dataset) throws IOException { + List formatNames = new ArrayList<>(); + + for (Exporter e : exporterMap.values()) { + String formatName = e.getFormatName(); + formatNames.add(formatName); + clearCachedExport(dataset, formatName); + } + clearCachedFormats(dataset, formatNames); + dataset.setLastExportTime(null); + } + + public void clearCachedFormats(Dataset dataset, List formatNames) throws IOException { + try { + for (String formatName : formatNames) { + clearCachedExport(dataset, formatName); + } } catch (IOException ex) { // not fatal } @@ -379,7 +435,7 @@ public void exportFormat(Dataset dataset, String formatName) throws ExportExcept } } - + public Exporter getExporter(String formatName) throws ExportException { Exporter e = exporterMap.get(formatName); if (e != null) { diff --git a/src/main/java/edu/harvard/iq/dataverse/export/InternalExportDataProvider.java b/src/main/java/edu/harvard/iq/dataverse/export/InternalExportDataProvider.java index f0d77eb8b52..2e7c22f1134 100644 --- a/src/main/java/edu/harvard/iq/dataverse/export/InternalExportDataProvider.java +++ b/src/main/java/edu/harvard/iq/dataverse/export/InternalExportDataProvider.java @@ -3,6 +3,7 @@ import java.io.InputStream; import java.util.Optional; +import jakarta.enterprise.inject.spi.CDI; import jakarta.json.Json; import jakarta.json.JsonArray; import jakarta.json.JsonArrayBuilder; @@ -11,45 +12,88 @@ import edu.harvard.iq.dataverse.DataCitation; import edu.harvard.iq.dataverse.DataFile; import edu.harvard.iq.dataverse.DatasetVersion; +import edu.harvard.iq.dataverse.DatasetVersionFilesServiceBean; import edu.harvard.iq.dataverse.FileMetadata; import edu.harvard.iq.dataverse.pidproviders.doi.datacite.DOIDataCiteRegisterService; import io.gdcc.spi.export.ExportDataProvider; import edu.harvard.iq.dataverse.util.bagit.OREMap; import edu.harvard.iq.dataverse.util.json.JsonPrinter; import edu.harvard.iq.dataverse.util.json.JsonUtil; +import io.gdcc.spi.export.ExportException; +import io.gdcc.spi.export.DatasetExportQuery; +import io.gdcc.spi.export.DatasetMetadataPredicates; +import io.gdcc.spi.export.FileExportQuery; +import io.gdcc.spi.export.FileMetadataPredicates; +import io.gdcc.spi.export.PageRequest; +import java.io.IOException; +import java.io.StringReader; +import java.util.List; +import java.util.Set; +import java.util.stream.Stream; +import javax.xml.parsers.DocumentBuilder; +import javax.xml.parsers.DocumentBuilderFactory; +import javax.xml.parsers.ParserConfigurationException; +import org.w3c.dom.Document; +import org.xml.sax.InputSource; +import org.xml.sax.SAXException; /** * Provides all data necessary to create an export - * + * */ public class InternalExportDataProvider implements ExportDataProvider { - private DatasetVersion dv; + private final DatasetVersion dv; private JsonObject jsonRepresentation = null; + private JsonObject jsonRepresentationNoFiles = null; private JsonObject schemaDotOrgRepresentation = null; private JsonObject oreRepresentation = null; + private JsonArray fileAndDataDetails = null; private InputStream is = null; + private DatasetVersionFilesServiceBean datasetVersionFilesService = null; InternalExportDataProvider(DatasetVersion dv) { this.dv = dv; } - + InternalExportDataProvider(DatasetVersion dv, InputStream is) { this.dv = dv; this.is=is; } + + /** + * This constructor is exclusively for use in IT tests + * @param dv + * @param versionFilesService + */ + InternalExportDataProvider(DatasetVersion dv, DatasetVersionFilesServiceBean versionFilesService) { + this.dv = dv; + this.datasetVersionFilesService = versionFilesService; + } @Override - public JsonObject getDatasetJson() { + public JsonObject getDatasetJson(DatasetExportQuery query) { + if (isOnlyDatasetLevelMetadataRequested(query)) { + // If we already have the "full" Json representation (with files) + // generated, should we return it (potentially moving MUCH more json + // than the client needs, or spend extra cycles generating the short + // form from scratch? - I'm choosing to go with latter. + if (jsonRepresentationNoFiles == null) { + final JsonObjectBuilder datasetAsJsonBuilder = JsonPrinter.datasetAsJsonForDTO(dv, false); + jsonRepresentationNoFiles = datasetAsJsonBuilder.build(); + } + return jsonRepresentationNoFiles; + } + if (jsonRepresentation == null) { - final JsonObjectBuilder datasetAsJsonBuilder = JsonPrinter.jsonAsDatasetDto(dv); + final JsonObjectBuilder datasetAsJsonBuilder = JsonPrinter.datasetAsJsonForDTO(dv); jsonRepresentation = datasetAsJsonBuilder.build(); } return jsonRepresentation; } @Override - public JsonObject getDatasetSchemaDotOrg() { + public JsonObject getDatasetSchemaDotOrg(DatasetExportQuery query) { if (schemaDotOrgRepresentation == null) { String jsonLdAsString = dv.getJsonLd(); schemaDotOrgRepresentation = JsonUtil.getJsonObject(jsonLdAsString); @@ -58,7 +102,7 @@ public JsonObject getDatasetSchemaDotOrg() { } @Override - public JsonObject getDatasetORE() { + public JsonObject getDatasetORE(DatasetExportQuery query) { if (oreRepresentation == null) { oreRepresentation = new OREMap(dv).getOREMap(); } @@ -67,26 +111,176 @@ public JsonObject getDatasetORE() { @Override public String getDataCiteXml() { + // @todo Is this the best way to obtain the metadata? - as opposed to + // going through the normal Export framework? (it may be, if it needs + // to be version-specific - ?) return DOIDataCiteRegisterService.getMetadataFromDvObject( dv.getDataset().getGlobalId().asString(), new DataCitation(dv).getDataCiteMetadata(), dv.getDataset()); } - + + @Override + public Document getDataCiteXml(DatasetExportQuery query) { + // Note that the query parameter is ignored, for now + String dataciteXmlString = getDataCiteXml(); + + DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance(); + + try { + DocumentBuilder builder = factory.newDocumentBuilder(); + + return builder.parse(new InputSource(new StringReader(dataciteXmlString))); + } catch (ParserConfigurationException | SAXException | IOException px) { + return null; + } + + } + @Override public JsonArray getDatasetFileDetails() { + if (fileAndDataDetails == null) { + JsonArrayBuilder jab = Json.createArrayBuilder(); + for (FileMetadata fileMetadata : dv.getFileMetadatas()) { + DataFile dataFile = fileMetadata.getDataFile(); + jab.add(JsonPrinter.json(dataFile, fileMetadata, true, false, true)); + } + fileAndDataDetails = jab.build(); + } + return fileAndDataDetails; + } + + @Override + public Stream getDatasetFileDetails(FileExportQuery query) { + if (fileAndDataDetails == null) { + JsonArrayBuilder jab = Json.createArrayBuilder(); + for (FileMetadata fileMetadata : dv.getFileMetadatas()) { + DataFile dataFile = fileMetadata.getDataFile(); + jab.add(JsonPrinter.json(dataFile, fileMetadata, true, false, true)); + } + fileAndDataDetails = jab.build(); + } + return fileAndDataDetails.stream().map(jsonValue -> jsonValue.asJsonObject()); + } + + @Override + /** + * This new (as of dataverse-spi 2.1.0) method will attempt to retrieve + * the requested tabular metadata more efficiently, by calling the + * DatasetVersionFilesServiceBean method directly. Which, among other things, + * allows to retrieve this information in batches. If for whatever reason + * that fails - if, for example, the EJB is not available in this context, + * we will throw an ExportException, giving the exporter a chance to try and + * retrieve this information using the traditional all-at-once method via + * getDatasetFileDetails(); + * + */ + public Stream getDatasetFileDetails(FileExportQuery query, PageRequest pageRequest) { JsonArrayBuilder jab = Json.createArrayBuilder(); - for (FileMetadata fileMetadata : dv.getFileMetadatas()) { - DataFile dataFile = fileMetadata.getDataFile(); - jab.add(JsonPrinter.json(dataFile, fileMetadata, true)); + + if (datasetVersionFilesService == null) { + try { + datasetVersionFilesService = CDI.current().select(DatasetVersionFilesServiceBean.class).get(); + } catch (java.lang.IllegalArgumentException | IllegalStateException ie) { + throw new ExportException("EJB DatasetVersionFilesService is not available; " + ie.getMessage()); + } + } + + if (datasetVersionFilesService == null) { + throw new ExportException("EJB DatasetVersionFilesService is not available"); + } + + if (isOnlyTabularMetadataRequested(query) && isDataVariableMetadataRequested(query)) { + + for (FileMetadata fileMetadata : datasetVersionFilesService.getTabularDataFileMetadatas(dv, + pageRequest.getLimit(), + pageRequest.getOffset(), + isOnlyPublicMetadataRequested(query))) { + DataFile dataFile = fileMetadata.getDataFile(); + jab.add(JsonPrinter.jsonDatafileWithDatatableForExport(dataFile, fileMetadata)); + } + + return jab.build().stream().map(jsonValue -> jsonValue.asJsonObject()); + } else { + throw new ExportException("This implementation of getDatasetFileDetails() (paginated version) " + + "only supports request for detailed DataVariable metadata, for tabular DataFiles only"); } - return jab.build(); } - + @Override - public Optional getPrerequisiteInputStream() { + public Optional getPrerequisiteInputStream(DatasetExportQuery query) { return Optional.ofNullable(is); } public void setPrerequisiteInputStream(InputStream prereqStream) { this.is=prereqStream; } + + /** + * Only one context object is supported + * @param DatasetExportQuery + * @return + */ + private boolean isOnlyDatasetLevelMetadataRequested(DatasetExportQuery query) { + + Set predicates = query.getDatasetPredicates(); + + for (DatasetMetadataPredicates p : predicates) { + // @todo This is pending on adding a dedicated DATASET_LEVEL_ONLY predicate + // to the enum + //if (p.equals(DatasetMetadataPredicates.DATASET_LEVEL_ONLY)) return true; + } + + // The default assumption is we pack both the Dataset, and the File-level + // metadata in the Json + return false; + } + + /** + * Are we skipping non-public, restricted and embargoed files? + * + * @param FileExportQuery + * @return yes or no + */ + private boolean isOnlyPublicMetadataRequested(FileExportQuery query) { + return checkForPredicate(query, FileMetadataPredicates.ONLY_PUBLIC_FILES); + } + + /** + * Is this metadata request only for ingested tabular files (i.e., files + * with linked DataTable objects) + * + * @param FileExportQuery + * @return yes or no + */ + private boolean isOnlyTabularMetadataRequested(FileExportQuery query) { + return checkForPredicate(query, FileMetadataPredicates.ONLY_TABULAR_FILES); + } + + /** + * Is detailed information about DataVariable objects associated with the + * tabular DataTable requested? + * + * @param FileExportQuery + * @return yes or no + */ + private boolean isDataVariableMetadataRequested(FileExportQuery query) { + return checkForPredicate(query, FileMetadataPredicates.INCLUDE_TABULAR_DATA_VARIABLES); + } + + /** + * Service method for checking a FileExportQuery for a specific predicate specified. + * + * @param query + * @param predicate + * @return + */ + private boolean checkForPredicate(FileExportQuery query, FileMetadataPredicates predicate) { + + Set predicates = query.getFilePredicates(); + + for (FileMetadataPredicates p : predicates) { + if (p.equals(predicate)) return true; + } + + return false; + } } diff --git a/src/main/java/edu/harvard/iq/dataverse/export/ddi/DdiExportUtil.java b/src/main/java/edu/harvard/iq/dataverse/export/ddi/DdiExportUtil.java index f5cc86bf8ee..10b11f502a7 100644 --- a/src/main/java/edu/harvard/iq/dataverse/export/ddi/DdiExportUtil.java +++ b/src/main/java/edu/harvard/iq/dataverse/export/ddi/DdiExportUtil.java @@ -6,6 +6,7 @@ import edu.harvard.iq.dataverse.DatasetFieldConstant; import edu.harvard.iq.dataverse.DvObjectContainer; import edu.harvard.iq.dataverse.GlobalId; +import edu.harvard.iq.dataverse.api.dto.DataTableDTO; import edu.harvard.iq.dataverse.api.dto.MetadataBlockDTO; import edu.harvard.iq.dataverse.api.dto.DatasetDTO; import edu.harvard.iq.dataverse.api.dto.DatasetVersionDTO; @@ -31,6 +32,12 @@ import edu.harvard.iq.dataverse.util.xml.XmlPrinter; import edu.harvard.iq.dataverse.util.xml.XmlUtil; import edu.harvard.iq.dataverse.util.xml.XmlWriterUtil; +//import io.gdcc.spi.export.ExportDataContext; +import io.gdcc.spi.export.PageRequest; +import io.gdcc.spi.export.FileExportQuery; +import io.gdcc.spi.export.ExportDataProvider; +import io.gdcc.spi.export.ExportException; +import io.gdcc.spi.export.FileMetadataPredicates; import java.io.ByteArrayOutputStream; import java.io.IOException; @@ -64,6 +71,7 @@ import javax.xml.transform.stream.StreamSource; import javax.xml.transform.stream.StreamResult; import java.io.InputStream; +import java.util.stream.Stream; public class DdiExportUtil { @@ -75,12 +83,13 @@ public class DdiExportUtil { public static final String LEVEL_DV = "dv"; - + static SettingsServiceBean settingsService; - + public static final String NOTE_TYPE_CONTENTTYPE = "DATAVERSE:CONTENTTYPE"; public static final String NOTE_SUBJECT_CONTENTTYPE = "Content/MIME Type"; public static final String CITATION_BLOCK_NAME = "citation"; + public static final int DATAVARIABLES_BATCH_SIZE = 10000; //Some tests don't send real PIDs that can be parsed //Use constant empty PID in these cases @@ -96,7 +105,7 @@ public static String datasetDtoAsJson2ddi(String datasetDtoAsJson) { return null; } } - + // "short" ddi, without the "" and "/" sections: public static void datasetJson2ddi(JsonObject datasetDtoAsJson, OutputStream outputStream) throws XMLStreamException { logger.fine(JsonUtil.prettyPrint(datasetDtoAsJson.toString())); @@ -104,14 +113,14 @@ public static void datasetJson2ddi(JsonObject datasetDtoAsJson, OutputStream out DatasetDTO datasetDto = gson.fromJson(datasetDtoAsJson.toString(), DatasetDTO.class); dtoddi(datasetDto, outputStream); } - + private static String dto2ddi(DatasetDTO datasetDto) throws XMLStreamException { OutputStream outputStream = new ByteArrayOutputStream(); dtoddi(datasetDto, outputStream); String xml = outputStream.toString(); return XmlPrinter.prettyPrintXml(xml); } - + private static void dtoddi(DatasetDTO datasetDto, OutputStream outputStream) throws XMLStreamException { XMLStreamWriter xmlw = null; try { @@ -125,7 +134,13 @@ private static void dtoddi(DatasetDTO datasetDto, OutputStream outputStream) thr xmlw.writeAttribute("xml:lang", datasetDto.getMetadataLanguage()); } createStdyDscr(xmlw, datasetDto); - createOtherMats(xmlw, datasetDto.getDatasetVersion().getFiles()); + if (datasetDto.getDatasetVersion().getFiles() != null) { + // We create "otherMat" sections with skipTabularFiles = false, because + // this is the short version of the DDI where all the files, whether + // ingested or not, are encoded as otherMats: + + createOtherMats(xmlw, datasetDto.getDatasetVersion().getFiles(), false); + } xmlw.writeEndElement(); // codeBook xmlw.flush(); } finally { @@ -140,13 +155,13 @@ private static void dtoddi(DatasetDTO datasetDto, OutputStream outputStream) thr } } - + // "full" ddi, with the the "" and "/" sections: - public static void datasetJson2ddi(JsonObject datasetDtoAsJson, JsonArray fileDetails, OutputStream outputStream) throws XMLStreamException { + public static void datasetJson2ddi(JsonObject datasetDtoAsJson, ExportDataProvider dataProvider, OutputStream outputStream) throws XMLStreamException { logger.fine(JsonUtil.prettyPrint(datasetDtoAsJson.toString())); Gson gson = new Gson(); DatasetDTO datasetDto = gson.fromJson(datasetDtoAsJson.toString(), DatasetDTO.class); - + XMLStreamWriter xmlw = null; try { xmlw = XMLOutputFactory.newInstance().createXMLStreamWriter(outputStream); @@ -160,9 +175,38 @@ public static void datasetJson2ddi(JsonObject datasetDtoAsJson, JsonArray fileDe xmlw.writeAttribute("xml:lang", datasetDto.getMetadataLanguage()); } createStdyDscr(xmlw, datasetDto); - createFileDscr(xmlw, fileDetails); - createDataDscr(xmlw, fileDetails); - createOtherMatsFromFileMetadatas(xmlw, fileDetails); + + // If there are no files in this dataset, we can stop here + if (datasetDto.getDatasetVersion().getFiles() != null) { + + // The Files and Data section, for the rich metadata describing + // the "ingested" tabular data files. + // Note that as of 6.8, we are generating the fileDscr from the DTOs + // supplied by ExportDataProvider.ExportDataProvider.getDatasetJson() + List varQuantityMap = createFileDscrs(xmlw, datasetDto.getDatasetVersion().getFiles()); + + if (varQuantityMap != null && !varQuantityMap.isEmpty()) { + // Now that we know that there is 1 or more ingested tabular file + // in the dataset, we can try and produce the dataDscr section. + // A dataset with a large number + // of ingested files may contain more of such metadata than is + // practical or desirable to pass around as a single chunk of json. + // As of the ExportDataProvider v2.1.0 a more efficient method is + // provided for retrieving this information in chunks of length-offset + // datatables-worth at a time. + //if (tabularFilesTotal <= DATATABLES_BATCH_SIZE) { + if (isVarQuantityLimitExceeded(varQuantityMap)) { + createDataDscrInBatches(xmlw, varQuantityMap, dataProvider); + } else { + createDataDscr(xmlw, dataProvider.getDatasetFileDetails()); + } + } + // otherMats section: + // Note that we are asking createOtherMats() to skip tabular files, + // since we have already created the fileDscr and dataDscr sections + // for those. + createOtherMats(xmlw, datasetDto.getDatasetVersion().getFiles(), true); + } xmlw.writeEndElement(); // codeBook xmlw.flush(); } finally { @@ -177,6 +221,18 @@ public static void datasetJson2ddi(JsonObject datasetDtoAsJson, JsonArray fileDe } } + private static boolean isVarQuantityLimitExceeded(List varQuantityMap) { + if (varQuantityMap != null) { + long varQuantityCount = 0; + for (long varQuantity : varQuantityMap) { + varQuantityCount += varQuantity; + if (varQuantityCount > DATAVARIABLES_BATCH_SIZE) { + return true; + } + } + } + return false; + } /** * @todo This is just a stub, copied from DDIExportServiceBean. It should * produce valid DDI based on @@ -213,25 +269,25 @@ private static void createStdyDscr(XMLStreamWriter xmlw, DatasetDTO datasetDto) } else if ("doi".equals(persistentAgency)) { persistentAgency = "DOI"; } - + //docDesc Block writeDocDescElement (xmlw, datasetDto); //stdyDesc Block xmlw.writeStartElement("stdyDscr"); xmlw.writeStartElement("citation"); xmlw.writeStartElement("titlStmt"); - + XmlWriterUtil.writeFullElement(xmlw, "titl", XmlWriterUtil.dto2Primitive(version, DatasetFieldConstant.title), datasetDto.getMetadataLanguage()); XmlWriterUtil.writeFullElement(xmlw, "subTitl", XmlWriterUtil.dto2Primitive(version, DatasetFieldConstant.subTitle)); FieldDTO altField = dto2FieldDTO( version, DatasetFieldConstant.alternativeTitle, "citation" ); if (altField != null) { writeMultipleElement(xmlw, "altTitl", altField, datasetDto.getMetadataLanguage()); } - + xmlw.writeStartElement("IDNo"); XmlWriterUtil.writeAttribute(xmlw, "agency", persistentAgency); - - + + xmlw.writeCharacters(pidString); xmlw.writeEndElement(); // IDNo writeOtherIdElement(xmlw, version); @@ -239,7 +295,7 @@ private static void createStdyDscr(XMLStreamWriter xmlw, DatasetDTO datasetDto) writeAuthorsElement(xmlw, version); writeProducersElement(xmlw, version); - + xmlw.writeStartElement("distStmt"); //The default is to add Dataverse Repository as a distributor. The excludeinstallationifset setting turns that off if there is a distributor defined in the metadata boolean distributorSet=false; @@ -249,7 +305,7 @@ private static void createStdyDscr(XMLStreamWriter xmlw, DatasetDTO datasetDto) distributorSet=true; } } - + boolean excludeRepository = settingsService.isTrueForKey(SettingsServiceBean.Key.ExportInstallationAsDistributorOnlyWhenNotSet, false); if (!StringUtils.isEmpty(datasetDto.getPublisher()) && !(excludeRepository && distributorSet)) { xmlw.writeStartElement("distrbtr"); @@ -271,14 +327,14 @@ private static void createStdyDscr(XMLStreamWriter xmlw, DatasetDTO datasetDto) xmlw.writeStartElement("holdings"); XmlWriterUtil.writeAttribute(xmlw, "URI", pidUri); xmlw.writeEndElement(); //holdings - + xmlw.writeEndElement(); // citation //End Citation Block - + //Start Study Info Block // Study Info xmlw.writeStartElement("stdyInfo"); - + writeSubjectElement(xmlw, version, datasetDto.getMetadataLanguage()); //Subject and Keywords writeAbstractElement(xmlw, version, datasetDto.getMetadataLanguage()); // Description writeSummaryDescriptionElement(xmlw, version, datasetDto.getMetadataLanguage()); @@ -291,7 +347,7 @@ private static void createStdyDscr(XMLStreamWriter xmlw, DatasetDTO datasetDto) writeOtherStudyMaterial(xmlw , version); XmlWriterUtil.writeFullElement(xmlw, "notes", XmlWriterUtil.dto2Primitive(version, DatasetFieldConstant.datasetLevelErrorNotes)); - + xmlw.writeEndElement(); // stdyDscr } @@ -325,7 +381,7 @@ private static void writeOtherStudyMaterial(XMLStreamWriter xmlw , DatasetVersio */ private static void writeDataAccess(XMLStreamWriter xmlw , DatasetVersionDTO version) throws XMLStreamException { xmlw.writeStartElement("dataAccs"); - + xmlw.writeStartElement("setAvail"); XmlWriterUtil.writeFullElement(xmlw, "accsPlac", version.getDataAccessPlace()); XmlWriterUtil.writeFullElement(xmlw, "origArch", version.getOriginalArchive()); @@ -333,7 +389,7 @@ private static void writeDataAccess(XMLStreamWriter xmlw , DatasetVersionDTO ver XmlWriterUtil.writeFullElement(xmlw, "collSize", version.getSizeOfCollection()); XmlWriterUtil.writeFullElement(xmlw, "complete", version.getStudyCompletion()); xmlw.writeEndElement(); //setAvail - + xmlw.writeStartElement("useStmt"); XmlWriterUtil.writeFullElement(xmlw, "confDec", version.getConfidentialityDeclaration()); XmlWriterUtil.writeFullElement(xmlw, "specPerm", version.getSpecialPermissions()); @@ -376,7 +432,7 @@ private static void writeDataAccess(XMLStreamWriter xmlw , DatasetVersionDTO ver } xmlw.writeEndElement(); //dataAccs } - + private static void writeDocDescElement (XMLStreamWriter xmlw, DatasetDTO datasetDto) throws XMLStreamException { DatasetVersionDTO version = datasetDto.getDatasetVersion(); String persistentProtocol = datasetDto.getProtocol(); @@ -390,7 +446,7 @@ private static void writeDocDescElement (XMLStreamWriter xmlw, DatasetDTO datase } else if ("doi".equals(persistentAgency)) { persistentAgency = "DOI"; } - + String persistentAuthority = datasetDto.getAuthority(); String persistentId = datasetDto.getIdentifier(); GlobalId pid = PidUtil.parseAsGlobalID(persistentProtocol, persistentAuthority, persistentId); @@ -419,7 +475,7 @@ private static void writeDocDescElement (XMLStreamWriter xmlw, DatasetDTO datase xmlw.writeEndElement(); // distrbtr } XmlWriterUtil.writeFullElement(xmlw, "distDate", datasetDto.getPublicationDate()); - + xmlw.writeEndElement(); // diststmt writeVersionStatement(xmlw, version); xmlw.writeStartElement("biblCit"); @@ -427,9 +483,9 @@ private static void writeDocDescElement (XMLStreamWriter xmlw, DatasetDTO datase xmlw.writeEndElement(); // biblCit xmlw.writeEndElement(); // citation xmlw.writeEndElement(); // docDscr - + } - + private static void writeVersionStatement(XMLStreamWriter xmlw, DatasetVersionDTO datasetVersionDTO) throws XMLStreamException{ xmlw.writeStartElement("verStmt"); xmlw.writeAttribute("source","archive"); @@ -447,10 +503,10 @@ private static void writeVersionStatement(XMLStreamWriter xmlw, DatasetVersionDT xmlw.writeCharacters(datasetVersionDTO.getVersionNote()); xmlw.writeEndElement(); // notes } - + xmlw.writeEndElement(); // verStmt } - + /* From the DDI 2.5 schema: @@ -669,14 +725,14 @@ private static void writeSummaryDescriptionElement(XMLStreamWriter xmlw, Dataset xmlw.writeEndElement(); //sumDscr } - + private static void writeMultipleElement(XMLStreamWriter xmlw, String element, FieldDTO fieldDTO, String lang) throws XMLStreamException { for (String value : fieldDTO.getMultiplePrimitive()) { //Write multiple lang vals for controlled vocab, otherwise don't include any lang tag XmlWriterUtil.writeFullElement(xmlw, element, value, fieldDTO.isControlledVocabularyField() ? lang : null); } } - + private static void writeDateElement(XMLStreamWriter xmlw, String element, String cycle, String event, String dateIn) throws XMLStreamException { xmlw.writeStartElement(element); @@ -687,7 +743,7 @@ private static void writeDateElement(XMLStreamWriter xmlw, String element, Strin xmlw.writeEndElement(); } - + /** * Again, is an xs:sequence - order is important and must follow * the schema. -L.A. @@ -745,7 +801,7 @@ private static void writeMethodElement(XMLStreamWriter xmlw , DatasetVersionDTO XmlWriterUtil.writeI18NElement(xmlw, "srcDocu", version, DatasetFieldConstant.accessToSources, lang); xmlw.writeEndElement(); //sources - + XmlWriterUtil.writeI18NElement(xmlw, "collSitu", version, DatasetFieldConstant.dataCollectionSituation, lang); XmlWriterUtil.writeI18NElement(xmlw, "actMin", version, DatasetFieldConstant.actionsToMinimizeLoss, lang); /* "" has the uppercase C: */ @@ -763,12 +819,12 @@ private static void writeMethodElement(XMLStreamWriter xmlw , DatasetVersionDTO XmlWriterUtil.writeI18NElement(xmlw, "EstSmpErr", version, DatasetFieldConstant.samplingErrorEstimates, lang); XmlWriterUtil.writeI18NElement(xmlw, "dataAppr", version, DatasetFieldConstant.otherDataAppraisal, lang); xmlw.writeEndElement(); //anlyInfo - + xmlw.writeEndElement();//method } - + private static void writeSubjectElement(XMLStreamWriter xmlw, DatasetVersionDTO datasetVersionDTO, String lang) throws XMLStreamException{ - + //Key Words and Topic Classification Locale defaultLocale = Locale.getDefault(); xmlw.writeStartElement("subject"); @@ -946,7 +1002,7 @@ private static void writeAuthorsElement(XMLStreamWriter xmlw, DatasetVersionDTO } } - + private static void writeContactsElement(XMLStreamWriter xmlw, DatasetVersionDTO datasetVersionDTO) throws XMLStreamException { for (Map.Entry entry : datasetVersionDTO.getMetadataBlocks().entrySet()) { @@ -985,7 +1041,7 @@ private static void writeContactsElement(XMLStreamWriter xmlw, DatasetVersionDTO } } } - + private static void writeProducersElement(XMLStreamWriter xmlw, DatasetVersionDTO version) throws XMLStreamException { xmlw.writeStartElement("prodStmt"); for (Map.Entry entry : version.getMetadataBlocks().entrySet()) { @@ -1025,7 +1081,7 @@ private static void writeProducersElement(XMLStreamWriter xmlw, DatasetVersionDT xmlw.writeEndElement(); //AuthEnty } } - + } } } @@ -1040,11 +1096,11 @@ private static void writeProducersElement(XMLStreamWriter xmlw, DatasetVersionDT writeMultipleElement(xmlw, "prodPlac", prodPlac, null); } writeSoftwareElement(xmlw, version); - + writeGrantElement(xmlw, version); xmlw.writeEndElement(); //prodStmt } - + private static void writeDistributorsElement(XMLStreamWriter xmlw, DatasetVersionDTO datasetVersionDTO, String lang) throws XMLStreamException { for (Map.Entry entry : datasetVersionDTO.getMetadataBlocks().entrySet()) { String key = entry.getKey(); @@ -1092,7 +1148,7 @@ private static void writeDistributorsElement(XMLStreamWriter xmlw, DatasetVersio } } } - + private static void writeRelPublElement(XMLStreamWriter xmlw, DatasetVersionDTO datasetVersionDTO) throws XMLStreamException { for (Map.Entry entry : datasetVersionDTO.getMetadataBlocks().entrySet()) { String key = entry.getKey(); @@ -1167,7 +1223,7 @@ private static void writeRelPublElement(XMLStreamWriter xmlw, DatasetVersionDTO } } } - + private static String appendCommaSeparatedValue(String inVal, String next) { if (!next.isEmpty()) { if (!inVal.isEmpty()) { @@ -1178,7 +1234,7 @@ private static String appendCommaSeparatedValue(String inVal, String next) { } return inVal; } - + private static void writeAbstractElement(XMLStreamWriter xmlw, DatasetVersionDTO datasetVersionDTO, String lang) throws XMLStreamException { for (Map.Entry entry : datasetVersionDTO.getMetadataBlocks().entrySet()) { String key = entry.getKey(); @@ -1245,7 +1301,7 @@ private static void writeGrantElement(XMLStreamWriter xmlw, DatasetVersionDTO da } } } - + private static void writeOtherIdElement(XMLStreamWriter xmlw, DatasetVersionDTO datasetVersionDTO) throws XMLStreamException { for (Map.Entry entry : datasetVersionDTO.getMetadataBlocks().entrySet()) { String key = entry.getKey(); @@ -1277,7 +1333,7 @@ private static void writeOtherIdElement(XMLStreamWriter xmlw, DatasetVersionDTO } } } - + private static void writeSoftwareElement(XMLStreamWriter xmlw, DatasetVersionDTO datasetVersionDTO) throws XMLStreamException { for (Map.Entry entry : datasetVersionDTO.getMetadataBlocks().entrySet()) { String key = entry.getKey(); @@ -1309,7 +1365,7 @@ private static void writeSoftwareElement(XMLStreamWriter xmlw, DatasetVersionDTO } } } - + private static void writeSeriesElement(XMLStreamWriter xmlw, DatasetVersionDTO datasetVersionDTO) throws XMLStreamException { for (Map.Entry entry : datasetVersionDTO.getMetadataBlocks().entrySet()) { String key = entry.getKey(); @@ -1347,7 +1403,7 @@ private static void writeSeriesElement(XMLStreamWriter xmlw, DatasetVersionDTO d } } } - + private static void writeTargetSampleElement(XMLStreamWriter xmlw, DatasetVersionDTO datasetVersionDTO) throws XMLStreamException { for (Map.Entry entry : datasetVersionDTO.getMetadataBlocks().entrySet()) { String key = entry.getKey(); @@ -1379,14 +1435,14 @@ private static void writeTargetSampleElement(XMLStreamWriter xmlw, DatasetVersio xmlw.writeCharacters(sizeFormula); xmlw.writeEndElement(); //sampleSizeFormula } - + xmlw.writeEndElement(); // targetSampleSize } } } } } - + private static void writeNotesElement(XMLStreamWriter xmlw, DatasetVersionDTO datasetVersionDTO) throws XMLStreamException { for (Map.Entry entry : datasetVersionDTO.getMetadataBlocks().entrySet()) { String key = entry.getKey(); @@ -1422,20 +1478,20 @@ private static void writeNotesElement(XMLStreamWriter xmlw, DatasetVersionDTO da } } } - + // TODO: // see if there's more information that we could encode in this otherMat. // contentType? Unfs and such? (in the "short" DDI that is being used for // harvesting *all* files are encoded as otherMats; even tabular ones. - private static void createOtherMats(XMLStreamWriter xmlw, List fileDtos) throws XMLStreamException { + private static void createOtherMats(XMLStreamWriter xmlw, List fileDtos, boolean skipTabularFiles) throws XMLStreamException { // The preferred URL for this dataverse, for cooking up the file access API links: String dataverseUrl = SystemConfig.getDataverseSiteUrlStatic(); - + for (FileDTO fileDTo : fileDtos) { // We'll continue using the scheme we've used before, in DVN2-3: non-tabular files are put into otherMat, // tabular ones - in fileDscr sections. (fileDscr sections have special fields for numbers of variables // and observations, etc.) - if (fileDTo.getDataFile().getDataTables() == null || fileDTo.getDataFile().getDataTables().isEmpty()) { + if (!(skipTabularFiles && isTabularData(fileDTo))) { xmlw.writeStartElement("otherMat"); XmlWriterUtil.writeAttribute(xmlw, "ID", "f" + fileDTo.getDataFile().getId()); String pidURL = fileDTo.getDataFile().getPidURL(); @@ -1465,70 +1521,20 @@ private static void createOtherMats(XMLStreamWriter xmlw, List fileDtos } } } - - // An alternative version of the createOtherMats method - this one is used - // when a "full" DDI is being cooked; just like the fileDscr and data/var sections methods, - // it operates on the list of FileMetadata entities, not on File DTOs. This is because - // DTOs do not support "tabular", variable-level metadata yet. And we need to be able to - // tell if this file is in fact tabular data - so that we know if it needs an - // otherMat, or a fileDscr section. - // -- L.A. 4.5 - - private static void createOtherMatsFromFileMetadatas(XMLStreamWriter xmlw, JsonArray fileDetails) throws XMLStreamException { - // The preferred URL for this dataverse, for cooking up the file access API links: - String dataverseUrl = SystemConfig.getDataverseSiteUrlStatic(); - - for (int i=0;i dto2PrimitiveList(DatasetVersionDTO datasetVersionDTO, String datasetFieldTypeName) { for (Map.Entry entry : datasetVersionDTO.getMetadataBlocks().entrySet()) { MetadataBlockDTO value = entry.getValue(); @@ -1548,7 +1554,7 @@ private static List dto2PrimitiveList(DatasetVersionDTO datasetVersionDT } return null; } - + private static FieldDTO dto2FieldDTO(DatasetVersionDTO datasetVersionDTO, String datasetFieldTypeName, String metadataBlockName) { MetadataBlockDTO block = datasetVersionDTO.getMetadataBlocks().get(metadataBlockName); if (block != null) { @@ -1572,10 +1578,10 @@ private static boolean StringUtilisEmpty(String str) { private static void saveJsonToDisk(String datasetVersionAsJson) throws IOException { Files.write(Paths.get("/tmp/out.json"), datasetVersionAsJson.getBytes()); } - - - - + + + + // Methods specific to the tabular data ("") section. // Note that these do NOT operate on DTO objects, but instead directly // on Dataverse DataVariable, DataTable, etc. objects. @@ -1587,14 +1593,14 @@ private static void saveJsonToDisk(String datasetVersionAsJson) throws IOExcepti // can go through the same DTO state... But we don't have time for it now; // plus, the structure of file-level metadata is currently being re-designed, // so we probably should not invest any time into it right now). -- L.A. 4.5 - + public static void createDataDscr(XMLStreamWriter xmlw, JsonArray fileDetails) throws XMLStreamException { if (fileDetails.isEmpty()) { return; } - boolean tabularData = false; + boolean dataDscrWritten = false; // we're not writing the opening tag until we find an actual // tabular datafile. @@ -1608,44 +1614,140 @@ public static void createDataDscr(XMLStreamWriter xmlw, JsonArray fileDetails) t * should instead use the "Data Variable Metadata Access" endpoint.) * These days we skip restricted files to avoid this exposure. */ - if (fileJson.containsKey("restricted") && fileJson.getBoolean("restricted")) { + if (isFileRestricted(fileJson)) { continue; } - if(fileJson.containsKey("embargo")) { - String dateString = fileJson.getJsonObject("embargo").getString("dateAvailable"); - LocalDate endDate = LocalDate.parse(dateString); - if (endDate != null && endDate.isAfter(LocalDate.now())) { - //Embargo is active so skip - continue; - } - } - + if (fileJson.containsKey("dataTables")) { - if (!tabularData) { + if (!dataDscrWritten) { xmlw.writeStartElement("dataDscr"); - tabularData = true; + dataDscrWritten = true; } - if(fileJson.containsKey("varGroups")) { - JsonArray varGroups = fileJson.getJsonArray("varGroups"); - for (int j=0;j varQuantityMap, ExportDataProvider exportDataProvider) throws XMLStreamException { + createDataDscrInBatches(xmlw, varQuantityMap, exportDataProvider, DATAVARIABLES_BATCH_SIZE); + } + + /** + * + * This public version of the method exists solely so that it can be called + * with a custom batch size; primarily for tests. + * @param xmlw XML stream writer + * @varQuantityMap mapping of ordered datatables -> number of variables in each + * @exportDataProvider data provider instance + * @variablesBatchSize number of variables to use in forming processing batches + */ + public static void createDataDscrInBatches(XMLStreamWriter xmlw, List varQuantityMap, ExportDataProvider exportDataProvider, int variablesBatchSize) throws XMLStreamException { + boolean dataDscrWritten = false; + + try { + int dataTableStart = 0; + int dataTablesThisBatch = 0; + int varQuantityThisBatch = 0; + + for (int dataTableCurrent = 0; dataTableCurrent < varQuantityMap.size(); dataTableCurrent++) { + varQuantityThisBatch += varQuantityMap.get(dataTableCurrent); + dataTablesThisBatch++; + + if (varQuantityThisBatch >= variablesBatchSize || dataTableCurrent == varQuantityMap.size() - 1) { + FileExportQuery exportQuery = FileExportQuery.builder() + .addFilePredicate(FileMetadataPredicates.ONLY_PUBLIC_FILES) + .addFilePredicate(FileMetadataPredicates.ONLY_TABULAR_FILES) + .addFilePredicate(FileMetadataPredicates.INCLUDE_TABULAR_DATA_VARIABLES) + .build(); + PageRequest paginationRequest = PageRequest.of(dataTableStart, dataTablesThisBatch); + Stream tabularFileDetails = exportDataProvider.getDatasetFileDetails(exportQuery, paginationRequest); + logger.fine("total number of variables in this batch: " + varQuantityThisBatch); + + int count = 0; + Iterator it = tabularFileDetails.iterator(); + while (it.hasNext()) { + JsonObject fileJson = it.next(); + + if (isFileRestricted(fileJson)) { + // This should not really happen - since we are explicitly + // requesting public files only; but, better safe ... + continue; + } + + if (fileJson.containsKey("dataTables")) { + if (!dataDscrWritten) { + xmlw.writeStartElement("dataDscr"); + dataDscrWritten = true; + } + + int howmany = createVariablesForDataFile(xmlw, fileJson); + // let's confirm here that the number of variables + // we got is what we expected; a mismatch here would + // indicate that the dataset and/or files in it have + // somehow changed since the initial lookup, and therefore + // the export should be aborted. + int howmanyExpected = varQuantityMap.get(dataTableStart + count); + if (howmanyExpected != howmany) { + throw new XMLStreamException("Number of variables mismatch. Expected: " + + howmanyExpected + + "; processed from datatable: " + + howmany); + } + } + count++; } + + logger.fine("requested: " + dataTablesThisBatch + " tabular file data entries; retrieved: " + count); + + + dataTableStart += dataTablesThisBatch; + dataTablesThisBatch = 0; + varQuantityThisBatch = 0; } } + if (dataDscrWritten) { + xmlw.writeEndElement(); // dataDscr + } + + } catch (ExportException ee) { + if (dataDscrWritten) { + // Unfortunately, we've already written some output by the time + // this exception was caught. We have no other choice but to + // give up + throw new XMLStreamException("Failed to write dataDscr variable-level section using exportDataProvider.getTabularData()"); + } else { + // Looks like we haven't written anything out yet. We can try + // and produce the dataDscr section using the classic, "all-at-once" + // approach instead. + createDataDscr(xmlw, exportDataProvider.getDatasetFileDetails()); + } } + } - if (tabularData) { - xmlw.writeEndElement(); // dataDscr + private static int createVariablesForDataFile(XMLStreamWriter xmlw, JsonObject fileJson) throws XMLStreamException { + if (fileJson.containsKey("varGroups")) { + JsonArray varGroups = fileJson.getJsonArray("varGroups"); + for (int j = 0; j < varGroups.size(); j++) { + createVarGroupDDI(xmlw, varGroups.getJsonObject(j)); + } + } + JsonObject dataTable = fileJson.getJsonArray("dataTables").getJsonObject(0); + JsonArray vars = dataTable.getJsonArray("dataVariables"); + logger.fine(vars.size() + " variables retrieved for file " + fileJson.getJsonNumber("id")); + if (vars != null) { + for (int j = 0; j < vars.size(); j++) { + createVarDDI(xmlw, vars.getJsonObject(j), fileJson.getJsonNumber("id").toString(), + fileJson.getJsonNumber("fileMetadataId").toString()); + } } + return vars.size(); } + private static void createVarGroupDDI(XMLStreamWriter xmlw, JsonObject varGrp) throws XMLStreamException { xmlw.writeStartElement("varGrp"); xmlw.writeAttribute("ID", "VG" + varGrp.getJsonNumber("id").toString()); @@ -1666,7 +1768,7 @@ private static void createVarGroupDDI(XMLStreamWriter xmlw, JsonObject varGrp) t xmlw.writeEndElement(); //varGrp } - + private static void createVarDDI(XMLStreamWriter xmlw, JsonObject dvar, String fileId, String fileMetadataId) throws XMLStreamException { xmlw.writeStartElement("var"); xmlw.writeAttribute("ID", "v" + dvar.getJsonNumber("id").toString()); @@ -1908,46 +2010,57 @@ private static void createVarDDI(XMLStreamWriter xmlw, JsonObject dvar, String f xmlw.writeEndElement(); //var } - - private static void createFileDscr(XMLStreamWriter xmlw, JsonArray fileDetails) throws XMLStreamException { + + private static List createFileDscrs(XMLStreamWriter xmlw, List fileDtos) throws XMLStreamException { + List ret = new ArrayList<>(); + + logger.fine("total " + fileDtos.size() + " file DTOs to process for fileDscr"); String dataverseUrl = SystemConfig.getDataverseSiteUrlStatic(); - for (int i =0;i field) - if (fileJson.containsKey("description")) { + if (fileDTo.getDataFile().getDescription() != null) { xmlw.writeStartElement("notes"); xmlw.writeAttribute("level", LEVEL_FILE); xmlw.writeAttribute("type", NOTE_TYPE_FILEDESCRIPTION); xmlw.writeAttribute("subject", NOTE_SUBJECT_FILEDESCRIPTION); - xmlw.writeCharacters(fileJson.getString("description")); + xmlw.writeCharacters(fileDTo.getDataFile().getDescription()); xmlw.writeEndElement(); // notes } // TODO: add the remaining fileDscr elements! xmlw.writeEndElement(); // fileDscr + counter++; } } + logger.fine("produced " + counter + " fileDscr entries; total number of variables found: " + totalVarQuantity); + return ret; } - - - - public static void datasetHtmlDDI(InputStream datafile, OutputStream outputStream) throws XMLStreamException { @@ -2025,7 +2137,7 @@ public static void datasetHtmlDDI(InputStream datafile, OutputStream outputStrea // Set secure processing feature tFactory.setFeature(XMLConstants.FEATURE_SECURE_PROCESSING, true); tFactory.setAttribute(XMLConstants.ACCESS_EXTERNAL_STYLESHEET, ""); - + StreamSource stylesource = new StreamSource(styleSheetInput); Transformer transformer = tFactory.newTransformer(stylesource); @@ -2052,4 +2164,34 @@ public static void injectSettingsService(SettingsServiceBean settingsSvc) { settingsService=settingsSvc; } + private static boolean isTabularData(FileDTO fileDTO) { + return !(fileDTO.getDataFile().getDataTables() == null || fileDTO.getDataFile().getDataTables().isEmpty()); + } + + /** + * Previously (in Dataverse 5.3 and below) the dataDscr section was included + * for restricted files but that meant that summary statistics were exposed. + * (To get at these statistics, API users should instead use the "Data + * Variable Metadata Access" endpoint.) These days we skip restricted files + * to avoid this exposure. + * @param fileJson - a JsonObject representing one datafile/datatable-worth + * of tabular data. + */ + private static boolean isFileRestricted(JsonObject fileJson) { + if (fileJson.containsKey("restricted") && fileJson.getBoolean("restricted")) { + return true; + } + if (fileJson.containsKey("embargo")) { + String dateString = fileJson.getJsonObject("embargo").getString("dateAvailable"); + LocalDate endDate = LocalDate.parse(dateString); + if (endDate != null && endDate.isAfter(LocalDate.now())) { + //Embargo is active so skip + return true; + } + } + return false; + } + + + } diff --git a/src/main/java/edu/harvard/iq/dataverse/harvest/server/OAIRecordServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/harvest/server/OAIRecordServiceBean.java index 1128746c06b..d0fa306927b 100644 --- a/src/main/java/edu/harvard/iq/dataverse/harvest/server/OAIRecordServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/harvest/server/OAIRecordServiceBean.java @@ -260,10 +260,15 @@ public void exportAllFormats(Dataset dataset) { @TransactionAttribute(REQUIRES_NEW) public void exportAllFormatsInNewTransaction(Dataset dataset) throws ExportException { + exportFormatsInNewTransaction(dataset, null); + } + + @TransactionAttribute(REQUIRES_NEW) + public void exportFormatsInNewTransaction(Dataset dataset, List formatNames) throws ExportException { try { ExportService exportServiceInstance = ExportService.getInstance(); - exportServiceInstance.exportAllFormats(dataset); - datasetService.setLastExportTimeInNewTransaction(dataset.getId(), dataset.getLastExportTime()); + exportServiceInstance.exportFormats(dataset, formatNames); + datasetService.setLastExportTimeInNewTransaction(dataset.getId(), dataset.getLastExportTime()); } catch (OptimisticLockException ole) { datasetService.setLastExportTimeInNewTransaction(dataset.getId(), dataset.getLastExportTime()); } catch (Exception e) { diff --git a/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java b/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java index 55247f7d2af..2a1e0b49cac 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java @@ -47,6 +47,11 @@ import java.util.stream.Collector; import java.util.stream.Collectors; +import jakarta.ejb.EJB; +import jakarta.ejb.Singleton; +import jakarta.json.JsonArray; +import jakarta.json.JsonObject; +import java.math.BigDecimal; import static edu.harvard.iq.dataverse.util.json.FileVersionDifferenceJsonPrinter.jsonFileVersionDifference; import static edu.harvard.iq.dataverse.util.json.NullSafeJsonBuilder.jsonObjectBuilder; import static java.util.stream.Collectors.toList; @@ -646,17 +651,22 @@ public static JsonObjectBuilder json(FileDetailsHolder ds) { } public static JsonObjectBuilder json(DatasetVersion dsv, boolean includeFiles) { - return json(dsv, null, includeFiles, false, true, false); + return json(dsv, null, includeFiles, false, true, false, false); } public static JsonObjectBuilder json(DatasetVersion dsv, boolean includeFiles, boolean includeMetadataBlocks) { - return json(dsv, null, includeFiles, false, includeMetadataBlocks, false); + return json(dsv, null, includeFiles, false, includeMetadataBlocks, false, false); } public static JsonObjectBuilder json(DatasetVersion dsv, List anonymizedFieldTypeNamesList, boolean includeFiles, boolean returnOwners) { - return json(dsv, anonymizedFieldTypeNamesList, includeFiles, returnOwners, true, false); + return json(dsv, anonymizedFieldTypeNamesList, includeFiles, returnOwners, true, false, false); + } + public static JsonObjectBuilder json(DatasetVersion dsv, List anonymizedFieldTypeNamesList, + boolean includeFiles, boolean returnOwners, boolean includeMetadataBlocks) { + return json(dsv, anonymizedFieldTypeNamesList, includeFiles, returnOwners, includeMetadataBlocks, false, false); } + public static JsonObjectBuilder json(DatasetVersion dsv, List anonymizedFieldTypeNamesList, - boolean includeFiles, boolean returnOwners, boolean includeMetadataBlocks, boolean ignoreSettingExcludeEmailFromExport) { + boolean includeFiles, boolean returnOwners, boolean includeMetadataBlocks, boolean forExportDataProvider, boolean ignoreSettingExcludeEmailFromExport) { Dataset dataset = dsv.getDataset(); JsonObjectBuilder bld = jsonObjectBuilder() .add("id", dsv.getId()).add("datasetId", dataset.getId()) @@ -715,7 +725,7 @@ public static JsonObjectBuilder json(DatasetVersion dsv, List anonymized bld.add("isPartOf", getOwnersFromDvObject(dataset)); } if (includeFiles) { - bld.add("files", jsonFileMetadatas(dsv.getFileMetadatas())); + bld.add("files", jsonFileMetadatas(dsv.getFileMetadatas(), forExportDataProvider)); } return bld; @@ -740,6 +750,22 @@ public static JsonObjectBuilder jsonDataFileList(List dataFiles){ return bld; } + public static JsonObjectBuilder datasetAsJsonForDTO(DatasetVersion dsv) { + return datasetAsJsonForDTO(dsv, true); + } + + /** + * Same as above, but gives an option to skip the file-level info + * @param dsv + * @param includeFiles + * @return + */ + public static JsonObjectBuilder datasetAsJsonForDTO(DatasetVersion dsv, boolean includeFiles) { + JsonObjectBuilder jsonForDTO = JsonPrinter.json(dsv.getDataset()); + jsonForDTO.add("datasetVersion", versionAsJsonForDTO(dsv, includeFiles)); + return jsonForDTO; + } + /** * Export formats such as DDI require the citation to be included. See * https://github.com/IQSS/dataverse/issues/2579 for more on DDI export. @@ -748,34 +774,20 @@ public static JsonObjectBuilder jsonDataFileList(List dataFiles){ * to the regular `json` method for DatasetVersion? Will anything break? * Unit tests for that method could not be found. */ - public static JsonObjectBuilder jsonWithCitation(DatasetVersion dsv, boolean includeFiles) { - JsonObjectBuilder dsvWithCitation = JsonPrinter.json(dsv, includeFiles); + private static JsonObjectBuilder versionAsJsonForDTO(DatasetVersion dsv, boolean includeFiles) { + JsonObjectBuilder dsvWithCitation = JsonPrinter.json(dsv, null, includeFiles, false, true, true, false); dsvWithCitation.add("citation", dsv.getCitation()); return dsvWithCitation; } - /** - * Export formats such as DDI require the persistent identifier components - * such as "protocol", "authority" and "identifier" to be included so we - * create a JSON object we can convert to a DatasetDTO which can include a - * DatasetVersionDTO, which has all the metadata fields we need to export. - * See https://github.com/IQSS/dataverse/issues/2579 for more on DDI export. - * - * @todo Instead of having this separate method, should "datasetVersion" be - * added to the regular `json` method for Dataset? Will anything break? Unit - * tests for that method could not be found. If we keep this method as-is - * should the method be renamed? - */ - public static JsonObjectBuilder jsonAsDatasetDto(DatasetVersion dsv) { - JsonObjectBuilder datasetDtoAsJson = JsonPrinter.json(dsv.getDataset()); - datasetDtoAsJson.add("datasetVersion", jsonWithCitation(dsv, true)); - return datasetDtoAsJson; - } - public static JsonArrayBuilder jsonFileMetadatas(Collection fmds) { + return jsonFileMetadatas(fmds, false); + } + + public static JsonArrayBuilder jsonFileMetadatas(Collection fmds, boolean forExportDataProvider) { JsonArrayBuilder filesArr = Json.createArrayBuilder(); for (FileMetadata fmd : fmds) { - filesArr.add(JsonPrinter.json(fmd)); + filesArr.add(JsonPrinter.json(fmd, false, false, forExportDataProvider)); } return filesArr; @@ -998,6 +1010,10 @@ public static JsonObjectBuilder json(FileMetadata fmd){ } public static JsonObjectBuilder json(FileMetadata fmd, boolean returnOwners, boolean printDatasetVersion) { + return json(fmd, returnOwners, printDatasetVersion, false); + } + + public static JsonObjectBuilder json(FileMetadata fmd, boolean returnOwners, boolean printDatasetVersion, boolean forExportDataProvider) { NullSafeJsonBuilder builder = jsonObjectBuilder(); // deprecated: .add("category", fmd.getCategory()) @@ -1013,7 +1029,7 @@ public static JsonObjectBuilder json(FileMetadata fmd, boolean returnOwners, boo .add("version", fmd.getVersion()) .add("datasetVersionId", fmd.getDatasetVersion().getId()) .add("categories", getFileCategories(fmd)) - .add("dataFile", JsonPrinter.json(fmd.getDataFile(), fmd, false, returnOwners)); + .add("dataFile", JsonPrinter.json(fmd.getDataFile(), fmd, forExportDataProvider, returnOwners)); if (printDatasetVersion) { builder.add("datasetVersion", json(fmd.getDatasetVersion(), false)); @@ -1044,14 +1060,14 @@ public static JsonObjectBuilder json(DataFile df, FileMetadata fileMetadata, boo } public static JsonObjectBuilder json(DataFile df, FileMetadata fileMetadata, boolean forExportDataProvider, boolean returnOwners) { - // File names are no longer stored in the DataFile entity; - // (they are instead in the FileMetadata (as "labels") - this way - // the filename can change between versions... - // It does appear that for some historical purpose we still need the - // filename in the file DTO (?)... We rely on it to be there for the - // DDI export, for example. So we need to make sure this is is the - // *correct* file name - i.e., that it comes from the right version. - // (TODO...? L.A. 4.5, Aug 7 2016) + return json(df, fileMetadata, forExportDataProvider, returnOwners, false); + } + + public static JsonObjectBuilder jsonDatafileWithDatatableForExport(DataFile df, FileMetadata fileMetadata) { + return json(df, fileMetadata, true, false, true); + } + + public static JsonObjectBuilder json(DataFile df, FileMetadata fileMetadata, boolean forExportDataProvider, boolean returnOwners, boolean includeVariables) { String fileName = null; if (fileMetadata == null){ @@ -1115,14 +1131,18 @@ public static JsonObjectBuilder json(DataFile df, FileMetadata fileMetadata, boo * The restricted state was not included prior to #9175 so to avoid backward * incompatability, it is now only added when generating json for the * InternalExportDataProvider fileDetails. + * [update]: more fields have been added below that are only there + * when the json is requested by the InternalExportDataProvider. */ if (forExportDataProvider) { builder.add("restricted", df.isRestricted()) - .add("fileMetadataId", fileMetadata.getId()) - .add("dataTables", df.getDataTables().isEmpty() ? null : JsonPrinter.jsonDT(df.getDataTables())) - .add("varGroups", fileMetadata.getVarGroups().isEmpty() - ? JsonPrinter.jsonVarGroup(fileMetadata.getVarGroups()) - : null); + .add("fileMetadataId", fileMetadata.getId()) + .add("dataTables", df.getDataTables().isEmpty() ? null : jsonDT(df.getDataTables(), includeVariables)); + if (includeVariables) { + builder.add("varGroups", fileMetadata.getVarGroups().isEmpty() + ? JsonPrinter.jsonVarGroup(fileMetadata.getVarGroups()) + : null); + } } if (returnOwners){ builder.add("isPartOf", getOwnersFromDvObject(df, fileMetadata.getDatasetVersion())); @@ -1131,22 +1151,24 @@ public static JsonObjectBuilder json(DataFile df, FileMetadata fileMetadata, boo } //Started from https://github.com/RENCI-NRIG/dataverse/, i.e. https://github.com/RENCI-NRIG/dataverse/commit/2b5a1225b42cf1caba85e18abfeb952171c6754a - public static JsonArrayBuilder jsonDT(List ldt) { + public static JsonArrayBuilder jsonDT(List ldt, boolean includeVariables) { JsonArrayBuilder ldtArr = Json.createArrayBuilder(); for(DataTable dt: ldt){ - ldtArr.add(JsonPrinter.json(dt)); + ldtArr.add(JsonPrinter.json(dt, includeVariables)); } return ldtArr; } - public static JsonObjectBuilder json(DataTable dt) { - return jsonObjectBuilder() + public static JsonObjectBuilder json(DataTable dt, boolean includeVariables) { + JsonObjectBuilder builder = jsonObjectBuilder() .add("varQuantity", dt.getVarQuantity()) .add("caseQuantity", dt.getCaseQuantity()) .add("recordsPerCase", dt.getRecordsPerCase()) - .add("UNF", dt.getUnf()) - .add("dataVariables", JsonPrinter.jsonDV(dt.getDataVariables())) - ; + .add("UNF", dt.getUnf()); + if (includeVariables) { + builder.add("dataVariables", JsonPrinter.jsonDV(dt.getDataVariables())); + } + return builder; } public static JsonArrayBuilder jsonDV(List dvl) { diff --git a/src/test/java/edu/harvard/iq/dataverse/export/CroissantExporterSlimTest.java b/src/test/java/edu/harvard/iq/dataverse/export/CroissantExporterSlimTest.java index fcbc9611818..f06354831d5 100644 --- a/src/test/java/edu/harvard/iq/dataverse/export/CroissantExporterSlimTest.java +++ b/src/test/java/edu/harvard/iq/dataverse/export/CroissantExporterSlimTest.java @@ -1,8 +1,11 @@ package edu.harvard.iq.dataverse.export; +import io.gdcc.spi.export.DatasetExportQuery; import static org.junit.jupiter.api.Assertions.*; import io.gdcc.spi.export.ExportDataProvider; +import io.gdcc.spi.export.FileExportQuery; +import io.gdcc.spi.export.PageRequest; import jakarta.json.Json; import jakarta.json.JsonArray; import jakarta.json.JsonObject; @@ -23,9 +26,11 @@ import java.nio.file.Paths; import java.util.HashMap; import java.util.Map; +import java.util.stream.Stream; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; import org.skyscreamer.jsonassert.JSONAssert; +import org.w3c.dom.Document; public class CroissantExporterSlimTest { @@ -109,6 +114,38 @@ public String getDataCiteXml() { return null; } } + + // extra methods in the new-and-improved ExportDataProvider interface + // (that we do not need for our current purposes) + + public JsonObject getDatasetJson(DatasetExportQuery q) { + return null; + } + + @Override + public JsonObject getDatasetORE(DatasetExportQuery q) { + return null; + } + + @Override + public Stream getDatasetFileDetails(FileExportQuery q) { + return null; + } + + @Override + public Stream getDatasetFileDetails(FileExportQuery q, PageRequest p) { + return null; + } + + @Override + public JsonObject getDatasetSchemaDotOrg(DatasetExportQuery q) { + return null; + } + + @Override + public Document getDataCiteXml(DatasetExportQuery q) { + return null; + } }; outputStreamMax = new ByteArrayOutputStream(); @@ -173,6 +210,38 @@ public String getDataCiteXml() { return null; } } + + // extra methods in the new-and-improved ExportDataProvider interface + // (that we do not need for our current purposes) + + public JsonObject getDatasetJson(DatasetExportQuery q) { + return null; + } + + @Override + public JsonObject getDatasetORE(DatasetExportQuery q) { + return null; + } + + @Override + public Stream getDatasetFileDetails(FileExportQuery q) { + return null; + } + + @Override + public Stream getDatasetFileDetails(FileExportQuery q, PageRequest p) { + return null; + } + + @Override + public JsonObject getDatasetSchemaDotOrg(DatasetExportQuery q) { + return null; + } + + @Override + public Document getDataCiteXml(DatasetExportQuery q) { + return null; + } }; outputStreamCars = new ByteArrayOutputStream(); @@ -237,6 +306,38 @@ public String getDataCiteXml() { return null; } } + + // extra methods in the new-and-improved ExportDataProvider interface + // (that we do not need for our current purposes) + + public JsonObject getDatasetJson(DatasetExportQuery q) { + return null; + } + + @Override + public JsonObject getDatasetORE(DatasetExportQuery q) { + return null; + } + + @Override + public Stream getDatasetFileDetails(FileExportQuery q) { + return null; + } + + @Override + public Stream getDatasetFileDetails(FileExportQuery q, PageRequest p) { + return null; + } + + @Override + public JsonObject getDatasetSchemaDotOrg(DatasetExportQuery q) { + return null; + } + + @Override + public Document getDataCiteXml(DatasetExportQuery q) { + return null; + } }; outputStreamRestricted = new ByteArrayOutputStream(); @@ -301,6 +402,38 @@ public String getDataCiteXml() { return null; } } + + // extra methods in the new-and-improved ExportDataProvider interface + // (that we do not need for our current purposes) + + public JsonObject getDatasetJson(DatasetExportQuery q) { + return null; + } + + @Override + public JsonObject getDatasetORE(DatasetExportQuery q) { + return null; + } + + @Override + public Stream getDatasetFileDetails(FileExportQuery q) { + return null; + } + + @Override + public Stream getDatasetFileDetails(FileExportQuery q, PageRequest p) { + return null; + } + + @Override + public JsonObject getDatasetSchemaDotOrg(DatasetExportQuery q) { + return null; + } + + @Override + public Document getDataCiteXml(DatasetExportQuery q) { + return null; + } }; outputStreamJunk = new ByteArrayOutputStream(); @@ -365,6 +498,38 @@ public String getDataCiteXml() { return null; } } + + // extra methods in the new-and-improved ExportDataProvider interface + // (that we do not need for our current purposes) + + public JsonObject getDatasetJson(DatasetExportQuery q) { + return null; + } + + @Override + public JsonObject getDatasetORE(DatasetExportQuery q) { + return null; + } + + @Override + public Stream getDatasetFileDetails(FileExportQuery q) { + return null; + } + + @Override + public Stream getDatasetFileDetails(FileExportQuery q, PageRequest p) { + return null; + } + + @Override + public JsonObject getDatasetSchemaDotOrg(DatasetExportQuery q) { + return null; + } + + @Override + public Document getDataCiteXml(DatasetExportQuery q) { + return null; + } }; outputStreamDraft = new ByteArrayOutputStream(); @@ -429,7 +594,39 @@ public String getDataCiteXml() { return null; } } - }; + + // extra methods in the new-and-improved ExportDataProvider interface + // (that we do not need for our current purposes) + + public JsonObject getDatasetJson(DatasetExportQuery q) { + return null; + } + + @Override + public JsonObject getDatasetORE(DatasetExportQuery q) { + return null; + } + + @Override + public Stream getDatasetFileDetails(FileExportQuery q) { + return null; + } + + @Override + public Stream getDatasetFileDetails(FileExportQuery q, PageRequest p) { + return null; + } + + @Override + public JsonObject getDatasetSchemaDotOrg(DatasetExportQuery q) { + return null; + } + + @Override + public Document getDataCiteXml(DatasetExportQuery q) { + return null; + } + }; } @Test diff --git a/src/test/java/edu/harvard/iq/dataverse/export/CroissantExporterTest.java b/src/test/java/edu/harvard/iq/dataverse/export/CroissantExporterTest.java index 6c6da792d4e..826441dfb70 100644 --- a/src/test/java/edu/harvard/iq/dataverse/export/CroissantExporterTest.java +++ b/src/test/java/edu/harvard/iq/dataverse/export/CroissantExporterTest.java @@ -3,6 +3,9 @@ import static org.junit.jupiter.api.Assertions.*; import io.gdcc.spi.export.ExportDataProvider; +import io.gdcc.spi.export.DatasetExportQuery; +import io.gdcc.spi.export.FileExportQuery; +import io.gdcc.spi.export.PageRequest; import jakarta.json.Json; import jakarta.json.JsonArray; import jakarta.json.JsonObject; @@ -23,9 +26,11 @@ import java.nio.file.Paths; import java.util.HashMap; import java.util.Map; +import java.util.stream.Stream; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; import org.skyscreamer.jsonassert.JSONAssert; +import org.w3c.dom.Document; public class CroissantExporterTest { @@ -109,6 +114,38 @@ public String getDataCiteXml() { return null; } } + + // extra methods in the new-and-improved ExportDataProvider interface + // (that we do not need for our current purposes) + + public JsonObject getDatasetJson(DatasetExportQuery q) { + return null; + } + + @Override + public JsonObject getDatasetORE(DatasetExportQuery q) { + return null; + } + + @Override + public Stream getDatasetFileDetails(FileExportQuery q) { + return null; + } + + @Override + public Stream getDatasetFileDetails(FileExportQuery q, PageRequest p) { + return null; + } + + @Override + public JsonObject getDatasetSchemaDotOrg(DatasetExportQuery q) { + return null; + } + + @Override + public Document getDataCiteXml(DatasetExportQuery q) { + return null; + } }; outputStreamMax = new ByteArrayOutputStream(); @@ -173,6 +210,38 @@ public String getDataCiteXml() { return null; } } + + // extra methods in the new-and-improved ExportDataProvider interface + // (that we do not need for our current purposes) + + public JsonObject getDatasetJson(DatasetExportQuery q) { + return null; + } + + @Override + public JsonObject getDatasetORE(DatasetExportQuery q) { + return null; + } + + @Override + public Stream getDatasetFileDetails(FileExportQuery q) { + return null; + } + + @Override + public Stream getDatasetFileDetails(FileExportQuery q, PageRequest p) { + return null; + } + + @Override + public JsonObject getDatasetSchemaDotOrg(DatasetExportQuery q) { + return null; + } + + @Override + public Document getDataCiteXml(DatasetExportQuery q) { + return null; + } }; outputStreamCars = new ByteArrayOutputStream(); @@ -237,6 +306,38 @@ public String getDataCiteXml() { return null; } } + + // extra methods in the new-and-improved ExportDataProvider interface + // (that we do not need for our current purposes) + + public JsonObject getDatasetJson(DatasetExportQuery q) { + return null; + } + + @Override + public JsonObject getDatasetORE(DatasetExportQuery q) { + return null; + } + + @Override + public Stream getDatasetFileDetails(FileExportQuery q) { + return null; + } + + @Override + public Stream getDatasetFileDetails(FileExportQuery q, PageRequest p) { + return null; + } + + @Override + public JsonObject getDatasetSchemaDotOrg(DatasetExportQuery q) { + return null; + } + + @Override + public Document getDataCiteXml(DatasetExportQuery q) { + return null; + } }; outputStreamRestricted = new ByteArrayOutputStream(); @@ -301,6 +402,38 @@ public String getDataCiteXml() { return null; } } + + // extra methods in the new-and-improved ExportDataProvider interface + // (that we do not need for our current purposes) + + public JsonObject getDatasetJson(DatasetExportQuery q) { + return null; + } + + @Override + public JsonObject getDatasetORE(DatasetExportQuery q) { + return null; + } + + @Override + public Stream getDatasetFileDetails(FileExportQuery q) { + return null; + } + + @Override + public Stream getDatasetFileDetails(FileExportQuery q, PageRequest p) { + return null; + } + + @Override + public JsonObject getDatasetSchemaDotOrg(DatasetExportQuery q) { + return null; + } + + @Override + public Document getDataCiteXml(DatasetExportQuery q) { + return null; + } }; outputStreamJunk = new ByteArrayOutputStream(); @@ -365,6 +498,38 @@ public String getDataCiteXml() { return null; } } + + // extra methods in the new-and-improved ExportDataProvider interface + // (that we do not need for our current purposes) + + public JsonObject getDatasetJson(DatasetExportQuery q) { + return null; + } + + @Override + public JsonObject getDatasetORE(DatasetExportQuery q) { + return null; + } + + @Override + public Stream getDatasetFileDetails(FileExportQuery q) { + return null; + } + + @Override + public Stream getDatasetFileDetails(FileExportQuery q, PageRequest p) { + return null; + } + + @Override + public JsonObject getDatasetSchemaDotOrg(DatasetExportQuery q) { + return null; + } + + @Override + public Document getDataCiteXml(DatasetExportQuery q) { + return null; + } }; outputStreamDraft = new ByteArrayOutputStream(); @@ -429,7 +594,39 @@ public String getDataCiteXml() { return null; } } - }; + + // extra methods in the new-and-improved ExportDataProvider interface + // (that we do not need for our current purposes) + + public JsonObject getDatasetJson(DatasetExportQuery q) { + return null; + } + + @Override + public JsonObject getDatasetORE(DatasetExportQuery q) { + return null; + } + + @Override + public Stream getDatasetFileDetails(FileExportQuery q) { + return null; + } + + @Override + public Stream getDatasetFileDetails(FileExportQuery q, PageRequest p) { + return null; + } + + @Override + public JsonObject getDatasetSchemaDotOrg(DatasetExportQuery q) { + return null; + } + + @Override + public Document getDataCiteXml(DatasetExportQuery q) { + return null; + } + }; } @Test diff --git a/src/test/java/edu/harvard/iq/dataverse/export/HugeDatasetExportPerformanceIT.java b/src/test/java/edu/harvard/iq/dataverse/export/HugeDatasetExportPerformanceIT.java index 1cf4c17495d..63bf826167d 100644 --- a/src/test/java/edu/harvard/iq/dataverse/export/HugeDatasetExportPerformanceIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/export/HugeDatasetExportPerformanceIT.java @@ -60,7 +60,7 @@ static void setUp() { @Test void shouldExportLargeDataset() { - Long datasetVersionId = regularFilesDataset.getId(); + Long datasetVersionId = regularFilesDataset.getVersions().get(0).getId(); QueryCountHolder.clear(); Instant start = Instant.now(); diff --git a/src/test/java/edu/harvard/iq/dataverse/export/TabularDataExportIT.java b/src/test/java/edu/harvard/iq/dataverse/export/TabularDataExportIT.java new file mode 100644 index 00000000000..b28f7b52376 --- /dev/null +++ b/src/test/java/edu/harvard/iq/dataverse/export/TabularDataExportIT.java @@ -0,0 +1,357 @@ +package edu.harvard.iq.dataverse.export; + +import edu.harvard.iq.dataverse.DataFile; +import edu.harvard.iq.dataverse.Dataset; +import edu.harvard.iq.dataverse.DatasetVersion; +import edu.harvard.iq.dataverse.DatasetVersionFilesServiceBean; +import edu.harvard.iq.dataverse.export.ddi.DdiExportUtil; +import edu.harvard.iq.dataverse.util.testing.fixtures.DatasetFixtureBuilder; +import edu.harvard.iq.dataverse.util.testing.performance.JpaEntityManagerService; +import edu.harvard.iq.dataverse.util.testing.performance.JpaPerformanceTest; +import edu.harvard.iq.dataverse.util.testing.recipes.DatasetRecipe; +import edu.harvard.iq.dataverse.util.testing.recipes.DatasetTypeRecipe; +import edu.harvard.iq.dataverse.util.testing.recipes.FileRecipe; +import edu.harvard.iq.dataverse.util.testing.recipes.VersionRecipe; +import edu.harvard.iq.dataverse.util.testing.recipes.VariableSetRecipe; +import edu.harvard.iq.dataverse.util.xml.XmlUtil; +import io.gdcc.spi.export.FileExportQuery; +import io.gdcc.spi.export.FileMetadataPredicates; +import io.gdcc.spi.export.PageRequest; +import jakarta.json.Json; +import jakarta.json.JsonArray; +import jakarta.json.JsonArrayBuilder; +import jakarta.json.JsonObject; +import java.io.ByteArrayOutputStream; +import java.io.StringReader; +import net.ttddyy.dsproxy.QueryCount; +import net.ttddyy.dsproxy.QueryCountHolder; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.nio.charset.StandardCharsets; + +import java.time.Instant; +import java.time.temporal.ChronoUnit; +import java.util.ArrayList; +import java.util.HashSet; + +import java.util.Iterator; +import java.util.List; +import java.util.Set; +import java.util.stream.Stream; +import javax.xml.stream.XMLInputFactory; +import javax.xml.stream.XMLOutputFactory; +import javax.xml.stream.XMLStreamConstants; +import javax.xml.stream.XMLStreamException; +import javax.xml.stream.XMLStreamReader; +import javax.xml.stream.XMLStreamWriter; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assumptions.assumeTrue; + +@JpaPerformanceTest +class TabularDataExportIT { + + static JpaEntityManagerService jpa; + + static Dataset tabularFilesDataset; + static int numberOfFiles = 20; + static int numberOfVariables = 100; + static int numberOfBatches = 4; + static List varQuantityMap = new ArrayList<>(); + + @BeforeAll + static void setUp() { + jpa.start(); + + DatasetTypeRecipe datasetType = DatasetTypeRecipe.dataset(); + + DatasetRecipe tabularFiles = DatasetRecipe.of( + datasetType, + VersionRecipe.of( + FileRecipe.tabular(numberOfFiles, VariableSetRecipe.uniform(numberOfVariables)) + ) + ); + + // Build the fixture + var tabularFixture = DatasetFixtureBuilder.builder().recipe(tabularFiles).build(); + + // Some save the type entity in the database and let the ORM create the mappings + jpa.inTransactionVoid(em -> em.persist(tabularFixture.datasetType())); + + // Persist the actual dataset + tabularFilesDataset = tabularFixture.dataset(); + jpa.inTransactionVoid(em -> { + // DataFile has no cascade path from Dataset, so each file must be persisted explicitly before + // the dataset graph is flushed. + for (DataFile dataFile : tabularFixture.dataFiles()) { + em.persist(dataFile); + } + em.persist(tabularFilesDataset); + }); + + for (int i = 0; i < numberOfFiles; i++) { + varQuantityMap.add(numberOfVariables); + // Something to consider in the future - add a more complex recipe, + // with varying numbers of variables in each file. + } + } + + @Test + void exportTabularMetadata() { + Long datasetVersionId = tabularFilesDataset.getVersions().get(0).getId(); + System.out.println("version id: " + datasetVersionId); + + QueryCountHolder.clear(); + Instant start = Instant.now(); + + // First, obtain the entire mess of the tabular data in the dataset in + // in one go, using the legacy .getDatasetFileDetails() method: + String json = jpa.inTransaction(em -> { + var datasetVersion = em.find(DatasetVersion.class, datasetVersionId); + assumeTrue(datasetVersion != null, "No dataset version available in DB. Check fixtures!"); + + InternalExportDataProvider provider = new InternalExportDataProvider(datasetVersion); + JsonArray details = provider.getDatasetFileDetails(); + // We want to parse this json and make sure that the expected number + // of files, datatables and variables has been returned. + + assertEquals(numberOfFiles, details.size(), "Number of tabular files retrieved does not match the number used in the recipe"); + + int total = 0; + for (int i = 0; i < details.size(); i++) { + JsonObject fileJson = details.getJsonObject(i); + JsonObject dataTable = fileJson.getJsonArray("dataTables").getJsonObject(0); + JsonArray vars = dataTable.getJsonArray("dataVariables"); + System.out.println(vars.size() + " variables retrieved for file " + fileJson.getJsonNumber("id")); + total += vars.size(); + } + assertEquals(numberOfFiles * numberOfVariables, total, "Failed to retrieve and parse the expected total number of variables"); + return details.toString(); + }); + + assertNotNull(json); + + System.out.println("test json produced: " + json); + + // Calculate the md5 of the complete output, for verification further down + String md5 = calculateMD5(json); + assertNotNull(md5); + System.out.println("MD5 Hash: " + md5); + + Instant end = Instant.now(); + long elapsed = start.until(end, ChronoUnit.MILLIS); + + QueryCount count = QueryCountHolder.getGrandTotal(); + + long queriesTotal = count.getTotal(); + long queriesSelect = count.getSelect(); + + System.out.println("Elapsed ms: " + elapsed); + System.out.println("Total queries: " + queriesTotal); + System.out.println("Select queries: " + queriesSelect); + + // And now acquire the same content using the new, paginated methods; + // then compare the checksums, to ensure the 2 methods produce the same + // exact result. + start = Instant.now(); + + json = jpa.inTransaction(em -> { + var datasetVersion = em.find(DatasetVersion.class, datasetVersionId); + assumeTrue(datasetVersion != null, "No dataset version available in DB. Check fixtures!"); + + DatasetVersionFilesServiceBean versionFilesService = new DatasetVersionFilesServiceBean(); + versionFilesService.injectEntityManager(em); + InternalExportDataProvider provider = new InternalExportDataProvider(datasetVersion, versionFilesService); + + JsonArrayBuilder jab = Json.createArrayBuilder(); + int filesPerBatch = numberOfFiles / numberOfBatches; + + for (int i = 0; i < numberOfBatches; i++) { + + FileExportQuery exportQuery = FileExportQuery.builder() + .addFilePredicate(FileMetadataPredicates.ONLY_PUBLIC_FILES) + .addFilePredicate(FileMetadataPredicates.ONLY_TABULAR_FILES) + .addFilePredicate(FileMetadataPredicates.INCLUDE_TABULAR_DATA_VARIABLES) + .build(); + PageRequest paginationRequest = PageRequest.of(filesPerBatch * i, filesPerBatch); + + Stream details = provider.getDatasetFileDetails(exportQuery, paginationRequest); + + Iterator it = details.iterator(); + int filesRetrieved = 0; + while (it.hasNext()) { + JsonObject fileJson = it.next(); + jab.add(fileJson); + filesRetrieved++; + } + assertEquals(filesPerBatch, filesRetrieved, "Failed to retrieve the expected number of tabular files in batch number " + i); + } + + // Parse the combined json, count the variables, confirm: + JsonArray jsonCombined = jab.build().asJsonArray(); + + int total = 0; + for (int i = 0; i < jsonCombined.size(); i++) { + JsonObject fileJson = jsonCombined.getJsonObject(i); + JsonObject dataTable = fileJson.getJsonArray("dataTables").getJsonObject(0); + JsonArray vars = dataTable.getJsonArray("dataVariables"); + total += vars.size(); + } + assertEquals(numberOfFiles * numberOfVariables, total, "Failed to retrieve and parse the expected total number of variables using new, paginated methods"); + + return jsonCombined.toString(); + }); + + end = Instant.now(); + elapsed = start.until(end, ChronoUnit.MILLIS); + + System.out.println("test json produced, paginated: " + json); + + String md5paginated = calculateMD5(json); + assertNotNull(md5paginated); + System.out.println("MD5 Hash: " + md5paginated); + + assertEquals(md5, md5paginated, "MD5 Hash mismatch between json fragments produced by the legacy, and paginated ExportDataProvder methods"); + + count = QueryCountHolder.getGrandTotal(); + System.out.println("Elapsed ms using paginated methods: " + elapsed); + System.out.println("Total queries using paginated methods: " + (count.getTotal() - queriesTotal)); + System.out.println("Select queries using paginated methods: " + (count.getSelect() - queriesSelect)); + + // And now try to export the dataVariable-level metadata as the + // fragment of DDI xml. + // (we are not interested in exporting a full DDI for this imaginary + // dataset since there are dedicated tests for the dataset-level DDI + // exports elsewhere). + String xml = jpa.inTransaction(em -> { + var datasetVersion = em.find(DatasetVersion.class, datasetVersionId); + assumeTrue(datasetVersion != null, "No dataset version available in DB. Check fixtures!"); + + DatasetVersionFilesServiceBean versionFilesService = new DatasetVersionFilesServiceBean(); + versionFilesService.injectEntityManager(em); + InternalExportDataProvider provider = new InternalExportDataProvider(datasetVersion, versionFilesService); + + ByteArrayOutputStream byteOutputStream = new ByteArrayOutputStream(); + + XMLStreamWriter xmlw = null; + try { + xmlw = XMLOutputFactory.newInstance().createXMLStreamWriter(byteOutputStream); + DdiExportUtil.createDataDscrInBatches(xmlw, varQuantityMap, provider, numberOfFiles * numberOfVariables / numberOfBatches); + xmlw.flush(); + } catch (XMLStreamException xse) { + assertTrue(false, "Failed to export the dataDscr DDI section, XMLStreamException: " + xse.getMessage()); + } finally { + if (xmlw != null) { + try { + xmlw.close(); + } catch (XMLStreamException e) { + // we don't care at this point + } + } + } + + return byteOutputStream.toString(StandardCharsets.UTF_8); + }); + + System.out.println("test DDI xml produced: " + xml); + + // Finally, let's parse the resulting XML and confirm that the expected + // numbers of unique datatables and variables have been exported. + // There are obvious ways in which the test can be made more thorough. + // For example, we can trace each variable by name and confirm that each + // one is properly exported in both the json and ddi xml formats. + StringReader reader = new StringReader(xml); + XMLStreamReader xmlr = null; + XMLInputFactory xmlFactory = XmlUtil.getSecureXMLInputFactory(); + + int dataDscrVariablesTotal = 0; + boolean dataDscrComplete = false; + boolean dataDscrVarElementOpen = false; + Set dataDscrDistinctFiles = new HashSet<>(); + + try { + xmlr = xmlFactory.createXMLStreamReader(reader); + } catch (XMLStreamException xse) { + assertTrue(false, "Failed to parse the produced dataDscr fragment as valid xml, XMLStreamException: " + xse.getMessage()); + } + try { + xmlr.nextTag(); + xmlr.require(XMLStreamConstants.START_ELEMENT, null, "dataDscr"); + } catch (XMLStreamException xse) { + assertTrue(false, "The produced xml fragment does not start with a dataDscr tag"); + } + + try { + for (int event = xmlr.next(); event != XMLStreamConstants.END_DOCUMENT; event = xmlr.next()) { + if (event == XMLStreamConstants.START_ELEMENT) { + if (xmlr.getLocalName().equals("var")) { + assertFalse(dataDscrVarElementOpen, "Out of order, nested tag encountered"); + dataDscrVarElementOpen = true; + } else if (xmlr.getLocalName().equals("location")) { + assertTrue(dataDscrVarElementOpen, "Out of order tag encountered"); + String fileId = xmlr.getAttributeValue(null, "fileid"); + assertNotNull(fileId, " element without a valid fileid attribute encountered"); + dataDscrDistinctFiles.add(fileId); + } + } else if (event == XMLStreamConstants.END_ELEMENT) { + if (xmlr.getLocalName().equals("var")) { + assertTrue(dataDscrVarElementOpen, "Out of order closing tag encountered"); + dataDscrVariablesTotal++; + dataDscrVarElementOpen = false; + } else if (xmlr.getLocalName().equals("dataDscr")) { + dataDscrComplete = true; + } + } + } + } catch (XMLStreamException xse) { + assertTrue(false, "Unexpected XMLStreamException when attempting to parse the dataDscr section: " + xse.getMessage()); + } finally { + if (xmlr != null) { + try {xmlr.close();} catch (XMLStreamException e) {} + } + } + + assertTrue(dataDscrComplete, " section not closed properly"); + assertFalse(dataDscrVarElementOpen, "an extra, unterminated section in the "); + assertEquals(numberOfFiles * numberOfVariables, dataDscrVariablesTotal, "Failed to parse the expected total number of variables in the generated section"); + assertEquals(numberOfFiles, dataDscrDistinctFiles.size(), "Invalid number of distinct tabular datafiles in the exported ddi fragment"); + + // In all of the tests above, the final result - the entire dataset-worth + // of exported tabular data - is passed around as a string; in the last + // method, this combined string then get re-parsed as xml for validation + // purposes. This is working adequately for the numbers of files and + // variables used; but if we want to use this code for true stress-testing + // with gigantic amounts of metadata, it will need to be rewritten to + // to stream the data in real time, for writing and reading, to avoid + // having to keep the whole mess in memory at once. + } + + private String calculateMD5(String source) { + String md5 = null; + + try { + MessageDigest md = MessageDigest.getInstance("MD5"); + md.update(source.getBytes(StandardCharsets.UTF_8)); + + byte[] hashBytes = md.digest(); + + StringBuilder hexString = new StringBuilder(); + for (byte b : hashBytes) { + String hex = Integer.toHexString(0xff & b); + if (hex.length() == 1) { + hexString.append('0'); + } + hexString.append(hex); + } + md5 = hexString.toString(); + } catch (NoSuchAlgorithmException e) { + System.err.println("MD5 algorithm not found!"); + //e.printStackTrace(); + } + return md5; + } +} diff --git a/src/test/java/edu/harvard/iq/dataverse/util/json/JsonPrinterTest.java b/src/test/java/edu/harvard/iq/dataverse/util/json/JsonPrinterTest.java index 40d6e02f9c2..95a3fbef20e 100644 --- a/src/test/java/edu/harvard/iq/dataverse/util/json/JsonPrinterTest.java +++ b/src/test/java/edu/harvard/iq/dataverse/util/json/JsonPrinterTest.java @@ -555,7 +555,7 @@ public void testDatasetWithGuestbook() { dataset.setGuestbook(guestbook); // verify that the guestbook id is in the dataset response - var jsob = JsonPrinter.json(dataset.getLatestVersion(), null, false, false, false, false).build(); + var jsob = JsonPrinter.json(dataset.getLatestVersion(), null, false, false, false, false, false).build(); System.out.println(jsob); var gbID = jsob.getInt("guestbookId"); assertEquals(1, gbID);