diff --git a/.gitignore b/.gitignore index ef3a8dc1f65..64433a06aaa 100644 --- a/.gitignore +++ b/.gitignore @@ -46,6 +46,7 @@ docs/.pnp.js tmp/images integration-tests/mail/output/ +# Generated spreadsheet writer outputs (under files/ historically; now also under output/) integration-tests/spreadsheet/files/exel-part-001.xlsx integration-tests/spreadsheet/files/exel-part-002.xlsx integration-tests/spreadsheet/files/exel-header-test.xlsx @@ -55,10 +56,20 @@ integration-tests/spreadsheet/files/exel-multi-part_1.xlsx integration-tests/spreadsheet/files/exel-multi-part_2.xlsx integration-tests/spreadsheet/files/exel-multi-part_3.xlsx integration-tests/spreadsheet/files/exel-multi-part_4.xlsx +/integration-tests/spreadsheet/files/exelwriter-testfile +integration-tests/spreadsheet/output/ integration-tests/json/output integration-tests/mdi/files/excelwriter-mdi-test.xlsx +integration-tests/mdi/files/jsonoutput-mdi-test_0.json +integration-tests/mdi/output/ +integration-tests/http/files/http-action-output +integration-tests/http/output/ integration-tests/spreadsheet/files/sample-file-append.xlsx +integration-tests/spreadsheet/files/sample-file-append-test.xlsx integration-tests/spark-native/output +integration-tests/transforms/files/parquet-test*.parquet +# MDI may still leave injected pipelines in the project root on older paths +integration-tests/mdi/*-injected.hpl assemblies/debug/audit @@ -71,5 +82,4 @@ screenshots/ # Docker script leaving audit data docker/local-data/ - -/integration-tests/spreadsheet/files/exelwriter-testfile +/docker/marketplace-nexus/.env diff --git a/integration-tests/http/files/http-action-output b/integration-tests/http/files/http-action-output deleted file mode 100644 index ec80ada209e..00000000000 --- a/integration-tests/http/files/http-action-output +++ /dev/null @@ -1,29 +0,0 @@ - - - - - - - Example - - -

This is an example of a simple HTML page.

- - diff --git a/integration-tests/http/main-0006-https-trustore.hwf b/integration-tests/http/main-0006-https-trustore.hwf index 83c8bad79fc..3c48b4dd214 100644 --- a/integration-tests/http/main-0006-https-trustore.hwf +++ b/integration-tests/http/main-0006-https-trustore.hwf @@ -82,7 +82,7 @@ limitations under the License. HTTP ${HTTPS_TEST_URL} - ${PROJECT_HOME}/files/http-action-output + ${PROJECT_HOME}/output/http-action-output N N diff --git a/integration-tests/mdi/0016-excelwriter-parent.hpl b/integration-tests/mdi/0016-excelwriter-parent.hpl index 0724edcfc80..ed6a26651a7 100644 --- a/integration-tests/mdi/0016-excelwriter-parent.hpl +++ b/integration-tests/mdi/0016-excelwriter-parent.hpl @@ -173,7 +173,7 @@ limitations under the License. - ${PROJECT_HOME}/files/excelwriter-mdi-test + ${PROJECT_HOME}/output/excelwriter-mdi-test xlsx diff --git a/integration-tests/mdi/0016-excelwriter-verify.hpl b/integration-tests/mdi/0016-excelwriter-verify.hpl index 3a47e4dffd7..0cd0259a274 100644 --- a/integration-tests/mdi/0016-excelwriter-verify.hpl +++ b/integration-tests/mdi/0016-excelwriter-verify.hpl @@ -139,7 +139,7 @@ limitations under the License. - ${PROJECT_HOME}/files/excelwriter-mdi-test.xlsx + ${PROJECT_HOME}/output/excelwriter-mdi-test.xlsx N diff --git a/integration-tests/mdi/0017-json-output-parent.hpl b/integration-tests/mdi/0017-json-output-parent.hpl index 1de26514b76..cee5362027b 100644 --- a/integration-tests/mdi/0017-json-output-parent.hpl +++ b/integration-tests/mdi/0017-json-output-parent.hpl @@ -197,7 +197,7 @@ limitations under the License. - ${PROJECT_HOME}/files/jsonoutput-mdi-test + ${PROJECT_HOME}/output/jsonoutput-mdi-test json diff --git a/integration-tests/mdi/0017-json-output-verify.hpl b/integration-tests/mdi/0017-json-output-verify.hpl index 0e58e6ce70b..a7a9b1cd7cb 100644 --- a/integration-tests/mdi/0017-json-output-verify.hpl +++ b/integration-tests/mdi/0017-json-output-verify.hpl @@ -79,7 +79,7 @@ limitations under the License. Y - ${PROJECT_HOME}/files/jsonoutput-mdi-test_0.json + ${PROJECT_HOME}/output/jsonoutput-mdi-test_0.json N diff --git a/integration-tests/mdi/files/excelwriter-mdi-test.xlsx b/integration-tests/mdi/files/excelwriter-mdi-test.xlsx deleted file mode 100644 index 89dc9f13332..00000000000 Binary files a/integration-tests/mdi/files/excelwriter-mdi-test.xlsx and /dev/null differ diff --git a/integration-tests/mdi/files/jsonoutput-mdi-test_0.json b/integration-tests/mdi/files/jsonoutput-mdi-test_0.json deleted file mode 100644 index 4e6a7c46360..00000000000 --- a/integration-tests/mdi/files/jsonoutput-mdi-test_0.json +++ /dev/null @@ -1 +0,0 @@ -{"data":[{"foo":"bar"}]} \ No newline at end of file diff --git a/integration-tests/mdi/main-0016-excelwriter.hwf b/integration-tests/mdi/main-0016-excelwriter.hwf index 8479522e449..20c7b641400 100644 --- a/integration-tests/mdi/main-0016-excelwriter.hwf +++ b/integration-tests/mdi/main-0016-excelwriter.hwf @@ -53,7 +53,7 @@ limitations under the License. DELETE_FILE - ${PROJECT_HOME}/files/excelwriter-mdi-test.xlsx + ${PROJECT_HOME}/output/excelwriter-mdi-test.xlsx N N 272 diff --git a/integration-tests/mdi/main-0017-json-output.hwf b/integration-tests/mdi/main-0017-json-output.hwf index 30d5dfa8aef..65a2a0b3d5c 100644 --- a/integration-tests/mdi/main-0017-json-output.hwf +++ b/integration-tests/mdi/main-0017-json-output.hwf @@ -53,7 +53,7 @@ limitations under the License. DELETE_FILE - ${PROJECT_HOME}/files/jsonoutput-mdi-test.json + ${PROJECT_HOME}/output/jsonoutput-mdi-test.json N N 208 diff --git a/integration-tests/scripts/run-tests-docker.sh b/integration-tests/scripts/run-tests-docker.sh index 1d56361d104..f6ebeac88bf 100755 --- a/integration-tests/scripts/run-tests-docker.sh +++ b/integration-tests/scripts/run-tests-docker.sh @@ -112,14 +112,29 @@ fi # Detect a real Google Cloud service-account key. The dummy file is a license comment, not # JSON; spreadsheet Google Sheets ITs need a real key (Jenkins: credentials gcp-access-hop). +# Require non-empty file + type=service_account + JSON-looking content so a corrupt/empty +# secret still skips cleanly. When python3 is available, also require parseable JSON. SKIP_GOOGLE_SHEETS="false" +GCP_KEY_OK="true" if [ ! -f "${GCP_KEY_FILE}" ] \ || [[ "${GCP_KEY_FILE}" == *dummyfile* ]] \ - || ! grep -qE '"type"[[:space:]]*:[[:space:]]*"service_account"' "${GCP_KEY_FILE}" 2>/dev/null; then + || [ ! -s "${GCP_KEY_FILE}" ] \ + || ! grep -qE '"type"[[:space:]]*:[[:space:]]*"service_account"' "${GCP_KEY_FILE}" 2>/dev/null \ + || ! grep -qE '\{' "${GCP_KEY_FILE}" 2>/dev/null; then + GCP_KEY_OK="false" +elif command -v python3 >/dev/null 2>&1; then + if ! python3 -c "import json,sys; json.load(open(sys.argv[1]))" "${GCP_KEY_FILE}" 2>/dev/null; then + GCP_KEY_OK="false" + fi +fi +if [ "${GCP_KEY_OK}" = "true" ]; then + echo "GCP service-account JSON present at GCP_KEY_FILE=${GCP_KEY_FILE}; Google Sheets ITs will run" +else SKIP_GOOGLE_SHEETS="true" echo "No valid GCP service-account JSON at GCP_KEY_FILE=${GCP_KEY_FILE}; spreadsheet Google Sheets tests will be skipped" fi export SKIP_GOOGLE_SHEETS +echo "SKIP_GOOGLE_SHEETS=${SKIP_GOOGLE_SHEETS}" if [ -z "${HOP_OPTIONS}" ] ; then HOP_OPTIONS="${HOP_OPTIONS} -Djavax.net.ssl.keyStore=./docker/integration-tests/resource/keystore.jks -Djavax.net.ssl.keyStorePassword=password -Djavax.net.ssl.trustStore=./docker/integration-tests/resource/mail/conf/keystore " @@ -142,31 +157,52 @@ fi mkdir -p "${CURRENT_DIR}"/../surefire-reports/ chmod 777 "${CURRENT_DIR}"/../surefire-reports/ -# Pre-create project output/ dirs on the host and make them world-writable. +# Pre-create project write dirs on the host and make them world-writable. # ASF Jenkins passes a container UID that matches the agent workspace owner, so ownership -# alone is enough there. World-writable output/ is a belt-and-suspenders for: +# alone is often enough. World-writable dirs are belt-and-suspenders for: # - local runs where someone overrides JENKINS_UID to a fixed value # - compose files that hardcode build-arg UIDs when not using this script's --build-arg -# Pipelines that write temp artifacts (Excel/ODS writer, Spark CSV, etc.) use -# ${PROJECT_HOME}/output. +# - residual files from a previous container UID that the host cannot delete/overwrite +# +# Many ITs write under ${PROJECT_HOME}/output (Excel/ODS, Spark CSV, mail), under +# ${PROJECT_HOME}/files (HTTP download, spreadsheet Excel writer, MDI JSON, parquet), +# and some MDI tests write *-injected.hpl into the project root itself. # # Spark/Spark-native leave untracked part-*.csv / .crc trees under output// owned by -# the previous container UID (often 755). Later runs with a different JENKINS_UID cannot -# delete them. Only remove *untracked* residual files — never wipe git-tracked content +# the previous container UID. Later runs with a different JENKINS_UID cannot delete them. +# Only remove *untracked* residual files under output/ — never wipe git-tracked content # (ldap stores setup pipelines under output/*.hpl). REPO_ROOT="$(cd "${CURRENT_DIR}/../.." && pwd)" +IT_ROOT="$(cd "${CURRENT_DIR}/.." && pwd)" + +# Best-effort: re-own the whole integration-tests tree as the container user so bind-mount +# writes succeed even when a previous run left root/other-UID residuals (needs Docker). +# Ownership only — do not chmod -R the tree (that dirties git file modes on 644 fixtures). +if docker info >/dev/null 2>&1; then + echo "Ensuring integration-tests/ is owned by ${JENKINS_UID}:${JENKINS_GID} (container identity)" + docker run --rm -v "${IT_ROOT}:/files" alpine:3.19 \ + sh -c "chown -R ${JENKINS_UID}:${JENKINS_GID} /files 2>/dev/null; true" \ + 2>/dev/null || true +fi + for d in "${CURRENT_DIR}"/../${PROJECT_NAME}/; do if [[ "$d" != *"scripts/" ]] && [[ "$d" != *"surefire-reports/" ]] && [[ "$d" != *"hopweb/" ]]; then if [ -d "$d" ] && [ ! -f "$d/disabled.txt" ]; then - mkdir -p "$d/output" - chmod 777 "$d/output" 2>/dev/null || true + # Project root: MDI target_file=…-injected.hpl writes here. + chmod a+rwx "$d" 2>/dev/null || true + + mkdir -p "$d/output" "$d/files" + chmod 777 "$d/output" "$d/files" 2>/dev/null || true + abs_out="$(cd "$d/output" && pwd)" rel_out="${abs_out#"${REPO_ROOT}"/}" + abs_files="$(cd "$d/files" && pwd)" + rel_files="${abs_files#"${REPO_ROOT}"/}" - # 1) Untracked leftovers only when the host can delete them (safe for ldap/*.hpl). + # 1) Untracked leftovers under output/ only when the host can delete them (safe for ldap/*.hpl). git -C "${REPO_ROOT}" clean -fd -- "${rel_out}" 2>/dev/null || true - # 2) Other-UID Spark residuals: chmod as root, then host git clean again. + # 2) Other-UID residuals under output/: chmod as root, then host git clean again. # Never blanket-delete under output/ (that wiped ldap/output/*.hpl). if find "$d/output" \( -name 'part-*' -o -name '_SUCCESS' -o -name '*.crc' \) 2>/dev/null | grep -q .; then echo "Fixing permissions on Spark residual files under ${rel_out}" @@ -178,6 +214,47 @@ for d in "${CURRENT_DIR}"/../${PROJECT_NAME}/; do sh -c 'find /out \( -name "part-*" -o -name "_SUCCESS" -o -name "*.crc" \) -exec rm -rf {} + 2>/dev/null; find /out -type d -empty -delete 2>/dev/null; true' \ 2>/dev/null || true fi + + # 3) files/ must stay writable for overwrite (HopVfs deletes then recreates). + # Re-assert modes; do not wipe fixtures. Only remove *untracked* generated leftovers + # (never rm a path that is still in git — e.g. http/files/exel-part-001.xlsx is a fixture). + if [ -d "$d/files" ]; then + chmod 777 "$d/files" 2>/dev/null || true + for residual in \ + http-action-output \ + jsonoutput-mdi-test_0.json \ + excelwriter-mdi-test.xlsx \ + exelwriter-testfile \ + exelwriter-testfile.xls \ + exel-header-test.xlsx \ + exel-part-001.xlsx \ + exel-part-002.xlsx \ + exel-multi-part_0.xlsx \ + exel-multi-part_1.xlsx \ + exel-multi-part_2.xlsx \ + exel-multi-part_3.xlsx \ + exel-multi-part_4.xlsx \ + sample-file-append-test.xlsx \ + parquet-test-00-0001.parquet; do + residual_path="$d/files/${residual}" + residual_rel="${residual_path#"${REPO_ROOT}"/}" + # Skip if tracked by git (fixture), only drop untracked leftovers + if [ -e "${residual_path}" ] \ + && ! git -C "${REPO_ROOT}" ls-files --error-unmatch "${residual_rel}" >/dev/null 2>&1; then + rm -f "${residual_path}" 2>/dev/null || true + fi + done + # If host still cannot write into files/, force a+rwx via root. + if ! touch "$d/files/.hop-it-write-check" 2>/dev/null; then + echo "Fixing permissions on ${rel_files} (container/host UID mismatch)" + docker run --rm -v "${abs_files}:/filesdir" alpine:3.19 \ + sh -c 'chmod -R a+rwx /filesdir 2>/dev/null; true' 2>/dev/null || true + else + rm -f "$d/files/.hop-it-write-check" 2>/dev/null || true + fi + chmod 777 "$d/files" 2>/dev/null || true + fi + chmod 777 "$d/output" 2>/dev/null || true fi fi diff --git a/integration-tests/scripts/run-tests.sh b/integration-tests/scripts/run-tests.sh index a63d2e8933c..de7da572ba7 100755 --- a/integration-tests/scripts/run-tests.sh +++ b/integration-tests/scripts/run-tests.sh @@ -255,15 +255,20 @@ for d in "${CURRENT_DIR}"/../${PROJECT_NAME}/; do # Create New Project export HOP_CONFIG_FOLDER="$d" - # Project output/ is often written by pipelines (CSV, Excel/ODS temp files, etc.). + # Project output/ and files/ are written by pipelines (CSV, Excel/ODS, HTTP downloads, + # MDI JSON, parquet, etc.). MDI also writes *-injected.hpl into the project root. # On ASF Jenkins the container UID matches the agent workspace owner (Jenkinsfile.daily - # passes id -u / id -g), so writes succeed by ownership. When UIDs differ, output/ is + # passes id -u / id -g), so writes succeed by ownership. When UIDs differ, dirs are # pre-created and chmod'd world-writable by run-tests-docker.sh on the host; here we # only best-effort reinforce that (mkdir/chmod may no-op if not owner). - mkdir -p "$d/output" 2>/dev/null || true + chmod a+rwx "$d" 2>/dev/null || true + mkdir -p "$d/output" "$d/files" 2>/dev/null || true if [ -d "$d/output" ]; then chmod 777 "$d/output" 2>/dev/null || true fi + if [ -d "$d/files" ]; then + chmod 777 "$d/files" 2>/dev/null || true + fi # Default pipeline run configuration name used by hop-run and the suite runner. # Beam projects name their Beam engine "local" and keep a native Local engine as "hop-local". diff --git a/integration-tests/spreadsheet/0003-write-xlsx.hpl b/integration-tests/spreadsheet/0003-write-xlsx.hpl index 06de996baea..e5956600d0b 100644 --- a/integration-tests/spreadsheet/0003-write-xlsx.hpl +++ b/integration-tests/spreadsheet/0003-write-xlsx.hpl @@ -130,7 +130,7 @@ limitations under the License. N Y - ${PROJECT_HOME}/files/exelwriter-testfile + ${PROJECT_HOME}/output/exelwriter-testfile xlsx N N diff --git a/integration-tests/spreadsheet/0004-test-header-check.hpl b/integration-tests/spreadsheet/0004-test-header-check.hpl index 6b0730d59cb..13d68190165 100644 --- a/integration-tests/spreadsheet/0004-test-header-check.hpl +++ b/integration-tests/spreadsheet/0004-test-header-check.hpl @@ -222,7 +222,7 @@ limitations under the License. - ${PROJECT_HOME}/files/exel-header-test.xlsx + ${PROJECT_HOME}/output/exel-header-test.xlsx N diff --git a/integration-tests/spreadsheet/0004-test-header.hpl b/integration-tests/spreadsheet/0004-test-header.hpl index 78a8f92780b..d76b419f36a 100644 --- a/integration-tests/spreadsheet/0004-test-header.hpl +++ b/integration-tests/spreadsheet/0004-test-header.hpl @@ -113,7 +113,7 @@ limitations under the License. N xlsx - ${PROJECT_HOME}/files/exel-header-test + ${PROJECT_HOME}/output/exel-header-test N reuse diff --git a/integration-tests/spreadsheet/0005-check-multiple-files.hpl b/integration-tests/spreadsheet/0005-check-multiple-files.hpl index 90ce641e830..191f9b557e0 100644 --- a/integration-tests/spreadsheet/0005-check-multiple-files.hpl +++ b/integration-tests/spreadsheet/0005-check-multiple-files.hpl @@ -90,7 +90,7 @@ limitations under the License. - ${PROJECT_HOME}/files/ + ${PROJECT_HOME}/output/ ^exel-part-.* N diff --git a/integration-tests/spreadsheet/0005-write-multiple-files.hpl b/integration-tests/spreadsheet/0005-write-multiple-files.hpl index 56332beb8f9..76f5fbb05c8 100644 --- a/integration-tests/spreadsheet/0005-write-multiple-files.hpl +++ b/integration-tests/spreadsheet/0005-write-multiple-files.hpl @@ -76,31 +76,31 @@ limitations under the License. a - ${PROJECT_HOME}/files/exel-part-001 + ${PROJECT_HOME}/output/exel-part-001 b - ${PROJECT_HOME}/files/exel-part-001 + ${PROJECT_HOME}/output/exel-part-001 d - ${PROJECT_HOME}/files/exel-part-002 + ${PROJECT_HOME}/output/exel-part-002 e - ${PROJECT_HOME}/files/exel-part-002 + ${PROJECT_HOME}/output/exel-part-002 f - ${PROJECT_HOME}/files/exel-part-002 + ${PROJECT_HOME}/output/exel-part-002 c - ${PROJECT_HOME}/files/exel-part-001 + ${PROJECT_HOME}/output/exel-part-001 g - ${PROJECT_HOME}/files/exel-part-002 + ${PROJECT_HOME}/output/exel-part-002 diff --git a/integration-tests/spreadsheet/0006-multi-part-fixed-name-check.hpl b/integration-tests/spreadsheet/0006-multi-part-fixed-name-check.hpl index c897ee29969..b250b73171f 100644 --- a/integration-tests/spreadsheet/0006-multi-part-fixed-name-check.hpl +++ b/integration-tests/spreadsheet/0006-multi-part-fixed-name-check.hpl @@ -126,7 +126,7 @@ limitations under the License. - ${PROJECT_HOME}/files/ + ${PROJECT_HOME}/output/ ^exel-multi-part.* N diff --git a/integration-tests/spreadsheet/0006-multi-part-fixed-name.hpl b/integration-tests/spreadsheet/0006-multi-part-fixed-name.hpl index f3828676f1b..21691514156 100644 --- a/integration-tests/spreadsheet/0006-multi-part-fixed-name.hpl +++ b/integration-tests/spreadsheet/0006-multi-part-fixed-name.hpl @@ -130,7 +130,7 @@ limitations under the License. Y xlsx - ${PROJECT_HOME}/files/exel-multi-part + ${PROJECT_HOME}/output/exel-multi-part N new diff --git a/integration-tests/spreadsheet/0007-append-to-xlsx-validate.hpl b/integration-tests/spreadsheet/0007-append-to-xlsx-validate.hpl index 3829b3ce4df..6f0959d80d8 100644 --- a/integration-tests/spreadsheet/0007-append-to-xlsx-validate.hpl +++ b/integration-tests/spreadsheet/0007-append-to-xlsx-validate.hpl @@ -207,7 +207,7 @@ limitations under the License. N N - ${PROJECT_HOME}/files/sample-file-append-test.xlsx + ${PROJECT_HOME}/output/sample-file-append-test.xlsx
Y
diff --git a/integration-tests/spreadsheet/0007-append-to-xlsx.hpl b/integration-tests/spreadsheet/0007-append-to-xlsx.hpl index e06dd39c693..e1fe9630669 100644 --- a/integration-tests/spreadsheet/0007-append-to-xlsx.hpl +++ b/integration-tests/spreadsheet/0007-append-to-xlsx.hpl @@ -203,7 +203,7 @@ limitations under the License. N reuse reuse - ${PROJECT_HOME}/files/sample-file-append-test + ${PROJECT_HOME}/output/sample-file-append-test N diff --git a/integration-tests/spreadsheet/datasets/golden-multi-file-excel.csv b/integration-tests/spreadsheet/datasets/golden-multi-file-excel.csv index 1ca551cfc1a..f748c27df93 100644 --- a/integration-tests/spreadsheet/datasets/golden-multi-file-excel.csv +++ b/integration-tests/spreadsheet/datasets/golden-multi-file-excel.csv @@ -1,8 +1,8 @@ value,filename -a,${PROJECT_HOME}/files/exel-part-001 -b,${PROJECT_HOME}/files/exel-part-001 -c,${PROJECT_HOME}/files/exel-part-001 -d,${PROJECT_HOME}/files/exel-part-002 -e,${PROJECT_HOME}/files/exel-part-002 -f,${PROJECT_HOME}/files/exel-part-002 -g,${PROJECT_HOME}/files/exel-part-002 +a,${PROJECT_HOME}/output/exel-part-001 +b,${PROJECT_HOME}/output/exel-part-001 +c,${PROJECT_HOME}/output/exel-part-001 +d,${PROJECT_HOME}/output/exel-part-002 +e,${PROJECT_HOME}/output/exel-part-002 +f,${PROJECT_HOME}/output/exel-part-002 +g,${PROJECT_HOME}/output/exel-part-002 diff --git a/integration-tests/spreadsheet/main-0007-append-file.hwf b/integration-tests/spreadsheet/main-0007-append-file.hwf index a572f54f477..94b8b82c7a8 100644 --- a/integration-tests/spreadsheet/main-0007-append-file.hwf +++ b/integration-tests/spreadsheet/main-0007-append-file.hwf @@ -106,7 +106,7 @@ limitations under the License. DELETE_FILE N - ${PROJECT_HOME}/files/sample-file-append-test.xlsx + ${PROJECT_HOME}/output/sample-file-append-test.xlsx N 880 48 @@ -124,12 +124,12 @@ limitations under the License. N N Y - N + Y EMPTY_SOURCE_URL-0-${PROJECT_HOME}/files/sample-file-append.xlsx STATIC-SOURCE-FILE-0 - EMPTY_DEST_URL-0-${PROJECT_HOME}/files/sample-file-append-test.xlsx + EMPTY_DEST_URL-0-${PROJECT_HOME}/output/sample-file-append-test.xlsx STATIC-DEST-FILE-0 diff --git a/integration-tests/transforms/0072-prepare-parquet-file.hpl b/integration-tests/transforms/0072-prepare-parquet-file.hpl index 30e45d33b33..4756294b297 100644 --- a/integration-tests/transforms/0072-prepare-parquet-file.hpl +++ b/integration-tests/transforms/0072-prepare-parquet-file.hpl @@ -259,7 +259,7 @@ limitations under the License. Int_Number - ${PROJECT_HOME}/files/parquet-test + ${PROJECT_HOME}/output/parquet-test Y yyyyMMdd-HHmmss parquet diff --git a/integration-tests/transforms/0072-read-parquet-file.hpl b/integration-tests/transforms/0072-read-parquet-file.hpl index 49d0c631684..ec58354513e 100644 --- a/integration-tests/transforms/0072-read-parquet-file.hpl +++ b/integration-tests/transforms/0072-read-parquet-file.hpl @@ -133,7 +133,7 @@ limitations under the License. -1 file - files/parquet-test-00-0001.parquet + output/parquet-test-00-0001.parquet -1 N String diff --git a/integration-tests/transforms/main-0072-parquet-input-test-delete.hwf b/integration-tests/transforms/main-0072-parquet-input-test-delete.hwf index 8ad4d17b976..81f5f5f60b8 100644 --- a/integration-tests/transforms/main-0072-parquet-input-test-delete.hwf +++ b/integration-tests/transforms/main-0072-parquet-input-test-delete.hwf @@ -106,7 +106,7 @@ limitations under the License. DELETE_FILE N - ${PROJECT_HOME}/files/parquet-test-00-0001.parquet + ${PROJECT_HOME}/output/parquet-test-00-0001.parquet N 864 192 diff --git a/plugins/transforms/execprocess/src/main/java/org/apache/hop/pipeline/transforms/execprocess/ExecProcess.java b/plugins/transforms/execprocess/src/main/java/org/apache/hop/pipeline/transforms/execprocess/ExecProcess.java index d275946a6a1..922ba648033 100644 --- a/plugins/transforms/execprocess/src/main/java/org/apache/hop/pipeline/transforms/execprocess/ExecProcess.java +++ b/plugins/transforms/execprocess/src/main/java/org/apache/hop/pipeline/transforms/execprocess/ExecProcess.java @@ -23,6 +23,7 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.List; +import java.util.StringTokenizer; import java.util.concurrent.CompletableFuture; import java.util.concurrent.CompletionException; import java.util.concurrent.CountDownLatch; @@ -218,7 +219,10 @@ private void execProcess(String[] process, ProcessResult processresult) throws H // execute process try { if (!meta.isArgumentsInFields()) { - p = data.runtime.exec(new String[] {process[0]}); + // Match historical Runtime.exec(String) whitespace tokenization without using the + // deprecated String overload. A single-element String[] would treat the whole command + // line (e.g. "/bin/echo hop-single") as the executable name. + p = data.runtime.exec(tokenizeCommandLine(process[0])); } else { p = data.runtime.exec(process); } @@ -323,6 +327,19 @@ private void execProcess(String[] process, ProcessResult processresult) throws H } } + /** + * Tokenize a command line the same way Runtime.exec(String) historically did (whitespace via + * {@link StringTokenizer}). + */ + static String[] tokenizeCommandLine(String command) { + StringTokenizer st = new StringTokenizer(command); + String[] cmdArray = new String[st.countTokens()]; + for (int i = 0; st.hasMoreTokens(); i++) { + cmdArray[i] = st.nextToken(); + } + return cmdArray; + } + private String getOutputString(BufferedReader b) throws IOException { StringBuilder returnValueBuffer = new StringBuilder(); String line; diff --git a/plugins/transforms/execprocess/src/test/java/org/apache/hop/pipeline/transforms/execprocess/ExecProcessTest.java b/plugins/transforms/execprocess/src/test/java/org/apache/hop/pipeline/transforms/execprocess/ExecProcessTest.java index 7cfd1e7c7a4..832673cbcde 100644 --- a/plugins/transforms/execprocess/src/test/java/org/apache/hop/pipeline/transforms/execprocess/ExecProcessTest.java +++ b/plugins/transforms/execprocess/src/test/java/org/apache/hop/pipeline/transforms/execprocess/ExecProcessTest.java @@ -25,6 +25,8 @@ import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.when; +import java.util.Arrays; +import java.util.List; import org.apache.hop.core.HopEnvironment; import org.apache.hop.core.IRowSet; import org.apache.hop.core.QueueRowSet; @@ -215,6 +217,14 @@ void processRow_failWhenNotSuccess_setsErrors() throws HopException { assertEquals(1, transform.getErrors()); } + @Test + void tokenizeCommandLine_splitsOnWhitespaceLikeRuntimeExecString() { + assertEquals( + Arrays.asList("/bin/echo", "hop-single"), + Arrays.asList(ExecProcess.tokenizeCommandLine("/bin/echo hop-single"))); + assertEquals(List.of("cmd"), Arrays.asList(ExecProcess.tokenizeCommandLine("cmd"))); + } + @Test void processRow_emptyProcessField_throws() throws HopException { HopEnvironment.init();