From 2550a3a81b49ac35b6acc8fdf44cd80286399ca9 Mon Sep 17 00:00:00 2001 From: mattcasters Date: Mon, 20 Jul 2026 17:23:40 +0200 Subject: [PATCH 1/2] issue #7582 : fix IT write permissions and ExecProcess single-command exec Harden integration-test Docker prep so project output/, files/, and roots are writable under the container UID, and move generated spreadsheet/MDI/ parquet/http artifacts into world-writable output/ paths (with golden data and gitignore updates). Also restore Runtime.exec(String) for ExecProcess single-command mode so command lines are tokenized again (#7565 regression). --- .gitignore | 14 ++- .../http/files/http-action-output | 29 ----- .../http/main-0006-https-trustore.hwf | 2 +- .../mdi/0016-excelwriter-parent.hpl | 2 +- .../mdi/0016-excelwriter-verify.hpl | 2 +- .../mdi/0017-json-output-parent.hpl | 2 +- .../mdi/0017-json-output-verify.hpl | 2 +- .../mdi/files/excelwriter-mdi-test.xlsx | Bin 3310 -> 0 bytes .../mdi/files/jsonoutput-mdi-test_0.json | 1 - .../mdi/main-0016-excelwriter.hwf | 2 +- .../mdi/main-0017-json-output.hwf | 2 +- integration-tests/scripts/run-tests-docker.sh | 99 ++++++++++++++++-- integration-tests/scripts/run-tests.sh | 11 +- .../spreadsheet/0003-write-xlsx.hpl | 2 +- .../spreadsheet/0004-test-header-check.hpl | 2 +- .../spreadsheet/0004-test-header.hpl | 2 +- .../spreadsheet/0005-check-multiple-files.hpl | 2 +- .../spreadsheet/0005-write-multiple-files.hpl | 14 +-- .../0006-multi-part-fixed-name-check.hpl | 2 +- .../0006-multi-part-fixed-name.hpl | 2 +- .../0007-append-to-xlsx-validate.hpl | 2 +- .../spreadsheet/0007-append-to-xlsx.hpl | 2 +- .../datasets/golden-multi-file-excel.csv | 14 +-- .../spreadsheet/main-0007-append-file.hwf | 6 +- .../transforms/0072-prepare-parquet-file.hpl | 2 +- .../transforms/0072-read-parquet-file.hpl | 2 +- .../main-0072-parquet-input-test-delete.hwf | 2 +- .../transforms/execprocess/ExecProcess.java | 5 +- .../execprocess/ExecProcessTest.java | 4 +- 29 files changed, 150 insertions(+), 83 deletions(-) delete mode 100644 integration-tests/http/files/http-action-output delete mode 100644 integration-tests/mdi/files/excelwriter-mdi-test.xlsx delete mode 100644 integration-tests/mdi/files/jsonoutput-mdi-test_0.json diff --git a/.gitignore b/.gitignore index ef3a8dc1f65..64433a06aaa 100644 --- a/.gitignore +++ b/.gitignore @@ -46,6 +46,7 @@ docs/.pnp.js tmp/images integration-tests/mail/output/ +# Generated spreadsheet writer outputs (under files/ historically; now also under output/) integration-tests/spreadsheet/files/exel-part-001.xlsx integration-tests/spreadsheet/files/exel-part-002.xlsx integration-tests/spreadsheet/files/exel-header-test.xlsx @@ -55,10 +56,20 @@ integration-tests/spreadsheet/files/exel-multi-part_1.xlsx integration-tests/spreadsheet/files/exel-multi-part_2.xlsx integration-tests/spreadsheet/files/exel-multi-part_3.xlsx integration-tests/spreadsheet/files/exel-multi-part_4.xlsx +/integration-tests/spreadsheet/files/exelwriter-testfile +integration-tests/spreadsheet/output/ integration-tests/json/output integration-tests/mdi/files/excelwriter-mdi-test.xlsx +integration-tests/mdi/files/jsonoutput-mdi-test_0.json +integration-tests/mdi/output/ +integration-tests/http/files/http-action-output +integration-tests/http/output/ integration-tests/spreadsheet/files/sample-file-append.xlsx +integration-tests/spreadsheet/files/sample-file-append-test.xlsx integration-tests/spark-native/output +integration-tests/transforms/files/parquet-test*.parquet +# MDI may still leave injected pipelines in the project root on older paths +integration-tests/mdi/*-injected.hpl assemblies/debug/audit @@ -71,5 +82,4 @@ screenshots/ # Docker script leaving audit data docker/local-data/ - -/integration-tests/spreadsheet/files/exelwriter-testfile +/docker/marketplace-nexus/.env diff --git a/integration-tests/http/files/http-action-output b/integration-tests/http/files/http-action-output deleted file mode 100644 index ec80ada209e..00000000000 --- a/integration-tests/http/files/http-action-output +++ /dev/null @@ -1,29 +0,0 @@ - - - - - - - Example - - -

This is an example of a simple HTML page.

- - diff --git a/integration-tests/http/main-0006-https-trustore.hwf b/integration-tests/http/main-0006-https-trustore.hwf index 83c8bad79fc..3c48b4dd214 100644 --- a/integration-tests/http/main-0006-https-trustore.hwf +++ b/integration-tests/http/main-0006-https-trustore.hwf @@ -82,7 +82,7 @@ limitations under the License. HTTP ${HTTPS_TEST_URL} - ${PROJECT_HOME}/files/http-action-output + ${PROJECT_HOME}/output/http-action-output N N diff --git a/integration-tests/mdi/0016-excelwriter-parent.hpl b/integration-tests/mdi/0016-excelwriter-parent.hpl index 0724edcfc80..ed6a26651a7 100644 --- a/integration-tests/mdi/0016-excelwriter-parent.hpl +++ b/integration-tests/mdi/0016-excelwriter-parent.hpl @@ -173,7 +173,7 @@ limitations under the License. - ${PROJECT_HOME}/files/excelwriter-mdi-test + ${PROJECT_HOME}/output/excelwriter-mdi-test xlsx diff --git a/integration-tests/mdi/0016-excelwriter-verify.hpl b/integration-tests/mdi/0016-excelwriter-verify.hpl index 3a47e4dffd7..0cd0259a274 100644 --- a/integration-tests/mdi/0016-excelwriter-verify.hpl +++ b/integration-tests/mdi/0016-excelwriter-verify.hpl @@ -139,7 +139,7 @@ limitations under the License. - ${PROJECT_HOME}/files/excelwriter-mdi-test.xlsx + ${PROJECT_HOME}/output/excelwriter-mdi-test.xlsx N diff --git a/integration-tests/mdi/0017-json-output-parent.hpl b/integration-tests/mdi/0017-json-output-parent.hpl index 1de26514b76..cee5362027b 100644 --- a/integration-tests/mdi/0017-json-output-parent.hpl +++ b/integration-tests/mdi/0017-json-output-parent.hpl @@ -197,7 +197,7 @@ limitations under the License. - ${PROJECT_HOME}/files/jsonoutput-mdi-test + ${PROJECT_HOME}/output/jsonoutput-mdi-test json diff --git a/integration-tests/mdi/0017-json-output-verify.hpl b/integration-tests/mdi/0017-json-output-verify.hpl index 0e58e6ce70b..a7a9b1cd7cb 100644 --- a/integration-tests/mdi/0017-json-output-verify.hpl +++ b/integration-tests/mdi/0017-json-output-verify.hpl @@ -79,7 +79,7 @@ limitations under the License. Y - ${PROJECT_HOME}/files/jsonoutput-mdi-test_0.json + ${PROJECT_HOME}/output/jsonoutput-mdi-test_0.json N diff --git a/integration-tests/mdi/files/excelwriter-mdi-test.xlsx b/integration-tests/mdi/files/excelwriter-mdi-test.xlsx deleted file mode 100644 index 89dc9f1333285725b99bd39d8840035cbe0ffc80..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 3310 zcmaJ@2{@E%8^+jWJI20mg=C#X*_Wv?WEr9&>tt=lGGyP9CB{yLDBI~cDI`l88lxH# z8pPPfRzjuh#WDX({r@q?f9js=`)204-sk<^=l<^JxgTpNfDuT?%F0R?(U{;wM*}{} z@AD=|1PYEox!8pVz=NGt!Y=tgg1tm^sPXLNh`IaEi=z+>(EA z=kumBi*}xEDtq9H$3?G;4jZ10fsN-(jQOH)&!ak(J}Y_8wkAa@Us#Mh<5qA1Dyj?e z=fGba=yDgOKU}>&Qma3|CB;}RQEVz96J|V`RosKJxRqji1kN%O3VO_^v~@s#xW&76 zVUF!wOMOek_=Mm>QkPJTjRs7)ZsXgjhDnYU#?UEOq|78wynUajE%A|wf<|BxF3?Q} zHX+J+P?(c^O(p6KABOv;Haf92D+dg5dta zsw&ibO%B!^pw5%zBHx24y2QlmGn&SVNgZSn$|X+J7FnDEPqbmMX^s9y*jY7A5?Q?~ z)F2sxfvJ>~$A=6glz9`sj);co=->Zj$d7#=FmsCew4H1FnEpVbtwZ+|%qE{VE#63O z!q54GAw3kA>i!W`3@|?u{D5ziHst|F{p9O-MsfBS8`Z=$qU1<41CcKFuns~71gp9Q z1W;vZnSo`e$kKE<#te^{STr&vfew+yd1f5Ekb!|LwixM=OSZ>V zYLYQcTu~pSZ_la6Lmj47z%SU?>H$ZLOIUL|9F`uXEG(X%NsQt4;+XjWiYxAJnRi^K0>xGXkIwp(C5U}mmYAP@ zIBC%eeHs!jaRtgPlVA*aI@?ug-j+K*v~KwjceLC;_jOu&eUeR^%2pvJZ*G_j*$QcF zYcs!!ire6B;%m~qK4AiRR9 zJep-#Qgs2|ERpG*tO}Kufjs7^VCe@WhWZ1(YHIRlB2D|>Msn-x549%};-fSu5JlSJ*nZ_i8DL31s0v4G5+W zRKSu?U)~HRznIn}q2FY2Y~2=rgThoR(EAq?Muyh@YK|Qz+zOkLKF!RfXDHVdcc&^k z9jEDWX93$?8w7AY?sKb2ql$1@`G7xSYIg8LkfMu=(Bzw0I@i2*CybzHME`3rGPVm*_56`_CPN?kjMd_#A z8`ntAvKYNlDy`MPn!v3A{@iTEbarJHWN7(X8rUIcXjCHLa%x<>PuI6en9|v zSB4F78{IV(GCVId@pQ#4y|>ELhp3ZxQ1(HFJY9ro?gpn=^p)c_@AR1gmiAZ5T+5eCyryc&VG&Hv z60p-e0O1=cZkTfjA45f#k>FQ5+(;2$FmrBy zAN4X$2X?7A{z)L`UA=c1opg~sbD=p<5eFuxGq5tUx5wB!7X&ymC)1^3;@Jf^fA7FMsNvb#f+)Iq{7KH-U1SV#8X|?8YPA+%y2tZ1RQtZChjKFz&(I)GdRyd-y_)lI zbKohk&$poL@G>_<_o)2uY&H%^HAA$a2#)u+BPyT&(tQPW8jQi!42(95^VhyVNf% zc=jzXM&381>D|3`32C_y+*vCxr%t*fy(+yOeM-UTm>u|ld;!UpMCI7v;HBdd6!A@c z7a%8yt?R8W?*+U}Jj~)JYm;Ir%4T0JW z_rSpNJ(RZm_XpD!47F$ML5THx@J~9({(WdgK`pO69A*3a5dT+f)=(y9>Pj}s$AI#h JZn9JV{Rasq8&3cL diff --git a/integration-tests/mdi/files/jsonoutput-mdi-test_0.json b/integration-tests/mdi/files/jsonoutput-mdi-test_0.json deleted file mode 100644 index 4e6a7c46360..00000000000 --- a/integration-tests/mdi/files/jsonoutput-mdi-test_0.json +++ /dev/null @@ -1 +0,0 @@ -{"data":[{"foo":"bar"}]} \ No newline at end of file diff --git a/integration-tests/mdi/main-0016-excelwriter.hwf b/integration-tests/mdi/main-0016-excelwriter.hwf index 8479522e449..20c7b641400 100644 --- a/integration-tests/mdi/main-0016-excelwriter.hwf +++ b/integration-tests/mdi/main-0016-excelwriter.hwf @@ -53,7 +53,7 @@ limitations under the License. DELETE_FILE - ${PROJECT_HOME}/files/excelwriter-mdi-test.xlsx + ${PROJECT_HOME}/output/excelwriter-mdi-test.xlsx N N 272 diff --git a/integration-tests/mdi/main-0017-json-output.hwf b/integration-tests/mdi/main-0017-json-output.hwf index 30d5dfa8aef..65a2a0b3d5c 100644 --- a/integration-tests/mdi/main-0017-json-output.hwf +++ b/integration-tests/mdi/main-0017-json-output.hwf @@ -53,7 +53,7 @@ limitations under the License. DELETE_FILE - ${PROJECT_HOME}/files/jsonoutput-mdi-test.json + ${PROJECT_HOME}/output/jsonoutput-mdi-test.json N N 208 diff --git a/integration-tests/scripts/run-tests-docker.sh b/integration-tests/scripts/run-tests-docker.sh index 1d56361d104..f6ebeac88bf 100755 --- a/integration-tests/scripts/run-tests-docker.sh +++ b/integration-tests/scripts/run-tests-docker.sh @@ -112,14 +112,29 @@ fi # Detect a real Google Cloud service-account key. The dummy file is a license comment, not # JSON; spreadsheet Google Sheets ITs need a real key (Jenkins: credentials gcp-access-hop). +# Require non-empty file + type=service_account + JSON-looking content so a corrupt/empty +# secret still skips cleanly. When python3 is available, also require parseable JSON. SKIP_GOOGLE_SHEETS="false" +GCP_KEY_OK="true" if [ ! -f "${GCP_KEY_FILE}" ] \ || [[ "${GCP_KEY_FILE}" == *dummyfile* ]] \ - || ! grep -qE '"type"[[:space:]]*:[[:space:]]*"service_account"' "${GCP_KEY_FILE}" 2>/dev/null; then + || [ ! -s "${GCP_KEY_FILE}" ] \ + || ! grep -qE '"type"[[:space:]]*:[[:space:]]*"service_account"' "${GCP_KEY_FILE}" 2>/dev/null \ + || ! grep -qE '\{' "${GCP_KEY_FILE}" 2>/dev/null; then + GCP_KEY_OK="false" +elif command -v python3 >/dev/null 2>&1; then + if ! python3 -c "import json,sys; json.load(open(sys.argv[1]))" "${GCP_KEY_FILE}" 2>/dev/null; then + GCP_KEY_OK="false" + fi +fi +if [ "${GCP_KEY_OK}" = "true" ]; then + echo "GCP service-account JSON present at GCP_KEY_FILE=${GCP_KEY_FILE}; Google Sheets ITs will run" +else SKIP_GOOGLE_SHEETS="true" echo "No valid GCP service-account JSON at GCP_KEY_FILE=${GCP_KEY_FILE}; spreadsheet Google Sheets tests will be skipped" fi export SKIP_GOOGLE_SHEETS +echo "SKIP_GOOGLE_SHEETS=${SKIP_GOOGLE_SHEETS}" if [ -z "${HOP_OPTIONS}" ] ; then HOP_OPTIONS="${HOP_OPTIONS} -Djavax.net.ssl.keyStore=./docker/integration-tests/resource/keystore.jks -Djavax.net.ssl.keyStorePassword=password -Djavax.net.ssl.trustStore=./docker/integration-tests/resource/mail/conf/keystore " @@ -142,31 +157,52 @@ fi mkdir -p "${CURRENT_DIR}"/../surefire-reports/ chmod 777 "${CURRENT_DIR}"/../surefire-reports/ -# Pre-create project output/ dirs on the host and make them world-writable. +# Pre-create project write dirs on the host and make them world-writable. # ASF Jenkins passes a container UID that matches the agent workspace owner, so ownership -# alone is enough there. World-writable output/ is a belt-and-suspenders for: +# alone is often enough. World-writable dirs are belt-and-suspenders for: # - local runs where someone overrides JENKINS_UID to a fixed value # - compose files that hardcode build-arg UIDs when not using this script's --build-arg -# Pipelines that write temp artifacts (Excel/ODS writer, Spark CSV, etc.) use -# ${PROJECT_HOME}/output. +# - residual files from a previous container UID that the host cannot delete/overwrite +# +# Many ITs write under ${PROJECT_HOME}/output (Excel/ODS, Spark CSV, mail), under +# ${PROJECT_HOME}/files (HTTP download, spreadsheet Excel writer, MDI JSON, parquet), +# and some MDI tests write *-injected.hpl into the project root itself. # # Spark/Spark-native leave untracked part-*.csv / .crc trees under output// owned by -# the previous container UID (often 755). Later runs with a different JENKINS_UID cannot -# delete them. Only remove *untracked* residual files — never wipe git-tracked content +# the previous container UID. Later runs with a different JENKINS_UID cannot delete them. +# Only remove *untracked* residual files under output/ — never wipe git-tracked content # (ldap stores setup pipelines under output/*.hpl). REPO_ROOT="$(cd "${CURRENT_DIR}/../.." && pwd)" +IT_ROOT="$(cd "${CURRENT_DIR}/.." && pwd)" + +# Best-effort: re-own the whole integration-tests tree as the container user so bind-mount +# writes succeed even when a previous run left root/other-UID residuals (needs Docker). +# Ownership only — do not chmod -R the tree (that dirties git file modes on 644 fixtures). +if docker info >/dev/null 2>&1; then + echo "Ensuring integration-tests/ is owned by ${JENKINS_UID}:${JENKINS_GID} (container identity)" + docker run --rm -v "${IT_ROOT}:/files" alpine:3.19 \ + sh -c "chown -R ${JENKINS_UID}:${JENKINS_GID} /files 2>/dev/null; true" \ + 2>/dev/null || true +fi + for d in "${CURRENT_DIR}"/../${PROJECT_NAME}/; do if [[ "$d" != *"scripts/" ]] && [[ "$d" != *"surefire-reports/" ]] && [[ "$d" != *"hopweb/" ]]; then if [ -d "$d" ] && [ ! -f "$d/disabled.txt" ]; then - mkdir -p "$d/output" - chmod 777 "$d/output" 2>/dev/null || true + # Project root: MDI target_file=…-injected.hpl writes here. + chmod a+rwx "$d" 2>/dev/null || true + + mkdir -p "$d/output" "$d/files" + chmod 777 "$d/output" "$d/files" 2>/dev/null || true + abs_out="$(cd "$d/output" && pwd)" rel_out="${abs_out#"${REPO_ROOT}"/}" + abs_files="$(cd "$d/files" && pwd)" + rel_files="${abs_files#"${REPO_ROOT}"/}" - # 1) Untracked leftovers only when the host can delete them (safe for ldap/*.hpl). + # 1) Untracked leftovers under output/ only when the host can delete them (safe for ldap/*.hpl). git -C "${REPO_ROOT}" clean -fd -- "${rel_out}" 2>/dev/null || true - # 2) Other-UID Spark residuals: chmod as root, then host git clean again. + # 2) Other-UID residuals under output/: chmod as root, then host git clean again. # Never blanket-delete under output/ (that wiped ldap/output/*.hpl). if find "$d/output" \( -name 'part-*' -o -name '_SUCCESS' -o -name '*.crc' \) 2>/dev/null | grep -q .; then echo "Fixing permissions on Spark residual files under ${rel_out}" @@ -178,6 +214,47 @@ for d in "${CURRENT_DIR}"/../${PROJECT_NAME}/; do sh -c 'find /out \( -name "part-*" -o -name "_SUCCESS" -o -name "*.crc" \) -exec rm -rf {} + 2>/dev/null; find /out -type d -empty -delete 2>/dev/null; true' \ 2>/dev/null || true fi + + # 3) files/ must stay writable for overwrite (HopVfs deletes then recreates). + # Re-assert modes; do not wipe fixtures. Only remove *untracked* generated leftovers + # (never rm a path that is still in git — e.g. http/files/exel-part-001.xlsx is a fixture). + if [ -d "$d/files" ]; then + chmod 777 "$d/files" 2>/dev/null || true + for residual in \ + http-action-output \ + jsonoutput-mdi-test_0.json \ + excelwriter-mdi-test.xlsx \ + exelwriter-testfile \ + exelwriter-testfile.xls \ + exel-header-test.xlsx \ + exel-part-001.xlsx \ + exel-part-002.xlsx \ + exel-multi-part_0.xlsx \ + exel-multi-part_1.xlsx \ + exel-multi-part_2.xlsx \ + exel-multi-part_3.xlsx \ + exel-multi-part_4.xlsx \ + sample-file-append-test.xlsx \ + parquet-test-00-0001.parquet; do + residual_path="$d/files/${residual}" + residual_rel="${residual_path#"${REPO_ROOT}"/}" + # Skip if tracked by git (fixture), only drop untracked leftovers + if [ -e "${residual_path}" ] \ + && ! git -C "${REPO_ROOT}" ls-files --error-unmatch "${residual_rel}" >/dev/null 2>&1; then + rm -f "${residual_path}" 2>/dev/null || true + fi + done + # If host still cannot write into files/, force a+rwx via root. + if ! touch "$d/files/.hop-it-write-check" 2>/dev/null; then + echo "Fixing permissions on ${rel_files} (container/host UID mismatch)" + docker run --rm -v "${abs_files}:/filesdir" alpine:3.19 \ + sh -c 'chmod -R a+rwx /filesdir 2>/dev/null; true' 2>/dev/null || true + else + rm -f "$d/files/.hop-it-write-check" 2>/dev/null || true + fi + chmod 777 "$d/files" 2>/dev/null || true + fi + chmod 777 "$d/output" 2>/dev/null || true fi fi diff --git a/integration-tests/scripts/run-tests.sh b/integration-tests/scripts/run-tests.sh index a63d2e8933c..de7da572ba7 100755 --- a/integration-tests/scripts/run-tests.sh +++ b/integration-tests/scripts/run-tests.sh @@ -255,15 +255,20 @@ for d in "${CURRENT_DIR}"/../${PROJECT_NAME}/; do # Create New Project export HOP_CONFIG_FOLDER="$d" - # Project output/ is often written by pipelines (CSV, Excel/ODS temp files, etc.). + # Project output/ and files/ are written by pipelines (CSV, Excel/ODS, HTTP downloads, + # MDI JSON, parquet, etc.). MDI also writes *-injected.hpl into the project root. # On ASF Jenkins the container UID matches the agent workspace owner (Jenkinsfile.daily - # passes id -u / id -g), so writes succeed by ownership. When UIDs differ, output/ is + # passes id -u / id -g), so writes succeed by ownership. When UIDs differ, dirs are # pre-created and chmod'd world-writable by run-tests-docker.sh on the host; here we # only best-effort reinforce that (mkdir/chmod may no-op if not owner). - mkdir -p "$d/output" 2>/dev/null || true + chmod a+rwx "$d" 2>/dev/null || true + mkdir -p "$d/output" "$d/files" 2>/dev/null || true if [ -d "$d/output" ]; then chmod 777 "$d/output" 2>/dev/null || true fi + if [ -d "$d/files" ]; then + chmod 777 "$d/files" 2>/dev/null || true + fi # Default pipeline run configuration name used by hop-run and the suite runner. # Beam projects name their Beam engine "local" and keep a native Local engine as "hop-local". diff --git a/integration-tests/spreadsheet/0003-write-xlsx.hpl b/integration-tests/spreadsheet/0003-write-xlsx.hpl index 06de996baea..e5956600d0b 100644 --- a/integration-tests/spreadsheet/0003-write-xlsx.hpl +++ b/integration-tests/spreadsheet/0003-write-xlsx.hpl @@ -130,7 +130,7 @@ limitations under the License. N Y - ${PROJECT_HOME}/files/exelwriter-testfile + ${PROJECT_HOME}/output/exelwriter-testfile xlsx N N diff --git a/integration-tests/spreadsheet/0004-test-header-check.hpl b/integration-tests/spreadsheet/0004-test-header-check.hpl index 6b0730d59cb..13d68190165 100644 --- a/integration-tests/spreadsheet/0004-test-header-check.hpl +++ b/integration-tests/spreadsheet/0004-test-header-check.hpl @@ -222,7 +222,7 @@ limitations under the License. - ${PROJECT_HOME}/files/exel-header-test.xlsx + ${PROJECT_HOME}/output/exel-header-test.xlsx N diff --git a/integration-tests/spreadsheet/0004-test-header.hpl b/integration-tests/spreadsheet/0004-test-header.hpl index 78a8f92780b..d76b419f36a 100644 --- a/integration-tests/spreadsheet/0004-test-header.hpl +++ b/integration-tests/spreadsheet/0004-test-header.hpl @@ -113,7 +113,7 @@ limitations under the License. N xlsx - ${PROJECT_HOME}/files/exel-header-test + ${PROJECT_HOME}/output/exel-header-test N reuse diff --git a/integration-tests/spreadsheet/0005-check-multiple-files.hpl b/integration-tests/spreadsheet/0005-check-multiple-files.hpl index 90ce641e830..191f9b557e0 100644 --- a/integration-tests/spreadsheet/0005-check-multiple-files.hpl +++ b/integration-tests/spreadsheet/0005-check-multiple-files.hpl @@ -90,7 +90,7 @@ limitations under the License. - ${PROJECT_HOME}/files/ + ${PROJECT_HOME}/output/ ^exel-part-.* N diff --git a/integration-tests/spreadsheet/0005-write-multiple-files.hpl b/integration-tests/spreadsheet/0005-write-multiple-files.hpl index 56332beb8f9..76f5fbb05c8 100644 --- a/integration-tests/spreadsheet/0005-write-multiple-files.hpl +++ b/integration-tests/spreadsheet/0005-write-multiple-files.hpl @@ -76,31 +76,31 @@ limitations under the License. a - ${PROJECT_HOME}/files/exel-part-001 + ${PROJECT_HOME}/output/exel-part-001 b - ${PROJECT_HOME}/files/exel-part-001 + ${PROJECT_HOME}/output/exel-part-001 d - ${PROJECT_HOME}/files/exel-part-002 + ${PROJECT_HOME}/output/exel-part-002 e - ${PROJECT_HOME}/files/exel-part-002 + ${PROJECT_HOME}/output/exel-part-002 f - ${PROJECT_HOME}/files/exel-part-002 + ${PROJECT_HOME}/output/exel-part-002 c - ${PROJECT_HOME}/files/exel-part-001 + ${PROJECT_HOME}/output/exel-part-001 g - ${PROJECT_HOME}/files/exel-part-002 + ${PROJECT_HOME}/output/exel-part-002 diff --git a/integration-tests/spreadsheet/0006-multi-part-fixed-name-check.hpl b/integration-tests/spreadsheet/0006-multi-part-fixed-name-check.hpl index c897ee29969..b250b73171f 100644 --- a/integration-tests/spreadsheet/0006-multi-part-fixed-name-check.hpl +++ b/integration-tests/spreadsheet/0006-multi-part-fixed-name-check.hpl @@ -126,7 +126,7 @@ limitations under the License. - ${PROJECT_HOME}/files/ + ${PROJECT_HOME}/output/ ^exel-multi-part.* N diff --git a/integration-tests/spreadsheet/0006-multi-part-fixed-name.hpl b/integration-tests/spreadsheet/0006-multi-part-fixed-name.hpl index f3828676f1b..21691514156 100644 --- a/integration-tests/spreadsheet/0006-multi-part-fixed-name.hpl +++ b/integration-tests/spreadsheet/0006-multi-part-fixed-name.hpl @@ -130,7 +130,7 @@ limitations under the License. Y xlsx - ${PROJECT_HOME}/files/exel-multi-part + ${PROJECT_HOME}/output/exel-multi-part N new diff --git a/integration-tests/spreadsheet/0007-append-to-xlsx-validate.hpl b/integration-tests/spreadsheet/0007-append-to-xlsx-validate.hpl index 3829b3ce4df..6f0959d80d8 100644 --- a/integration-tests/spreadsheet/0007-append-to-xlsx-validate.hpl +++ b/integration-tests/spreadsheet/0007-append-to-xlsx-validate.hpl @@ -207,7 +207,7 @@ limitations under the License. N N - ${PROJECT_HOME}/files/sample-file-append-test.xlsx + ${PROJECT_HOME}/output/sample-file-append-test.xlsx
Y
diff --git a/integration-tests/spreadsheet/0007-append-to-xlsx.hpl b/integration-tests/spreadsheet/0007-append-to-xlsx.hpl index e06dd39c693..e1fe9630669 100644 --- a/integration-tests/spreadsheet/0007-append-to-xlsx.hpl +++ b/integration-tests/spreadsheet/0007-append-to-xlsx.hpl @@ -203,7 +203,7 @@ limitations under the License. N reuse reuse - ${PROJECT_HOME}/files/sample-file-append-test + ${PROJECT_HOME}/output/sample-file-append-test N diff --git a/integration-tests/spreadsheet/datasets/golden-multi-file-excel.csv b/integration-tests/spreadsheet/datasets/golden-multi-file-excel.csv index 1ca551cfc1a..f748c27df93 100644 --- a/integration-tests/spreadsheet/datasets/golden-multi-file-excel.csv +++ b/integration-tests/spreadsheet/datasets/golden-multi-file-excel.csv @@ -1,8 +1,8 @@ value,filename -a,${PROJECT_HOME}/files/exel-part-001 -b,${PROJECT_HOME}/files/exel-part-001 -c,${PROJECT_HOME}/files/exel-part-001 -d,${PROJECT_HOME}/files/exel-part-002 -e,${PROJECT_HOME}/files/exel-part-002 -f,${PROJECT_HOME}/files/exel-part-002 -g,${PROJECT_HOME}/files/exel-part-002 +a,${PROJECT_HOME}/output/exel-part-001 +b,${PROJECT_HOME}/output/exel-part-001 +c,${PROJECT_HOME}/output/exel-part-001 +d,${PROJECT_HOME}/output/exel-part-002 +e,${PROJECT_HOME}/output/exel-part-002 +f,${PROJECT_HOME}/output/exel-part-002 +g,${PROJECT_HOME}/output/exel-part-002 diff --git a/integration-tests/spreadsheet/main-0007-append-file.hwf b/integration-tests/spreadsheet/main-0007-append-file.hwf index a572f54f477..94b8b82c7a8 100644 --- a/integration-tests/spreadsheet/main-0007-append-file.hwf +++ b/integration-tests/spreadsheet/main-0007-append-file.hwf @@ -106,7 +106,7 @@ limitations under the License. DELETE_FILE N - ${PROJECT_HOME}/files/sample-file-append-test.xlsx + ${PROJECT_HOME}/output/sample-file-append-test.xlsx N 880 48 @@ -124,12 +124,12 @@ limitations under the License. N N Y - N + Y EMPTY_SOURCE_URL-0-${PROJECT_HOME}/files/sample-file-append.xlsx STATIC-SOURCE-FILE-0 - EMPTY_DEST_URL-0-${PROJECT_HOME}/files/sample-file-append-test.xlsx + EMPTY_DEST_URL-0-${PROJECT_HOME}/output/sample-file-append-test.xlsx STATIC-DEST-FILE-0 diff --git a/integration-tests/transforms/0072-prepare-parquet-file.hpl b/integration-tests/transforms/0072-prepare-parquet-file.hpl index 30e45d33b33..4756294b297 100644 --- a/integration-tests/transforms/0072-prepare-parquet-file.hpl +++ b/integration-tests/transforms/0072-prepare-parquet-file.hpl @@ -259,7 +259,7 @@ limitations under the License. Int_Number - ${PROJECT_HOME}/files/parquet-test + ${PROJECT_HOME}/output/parquet-test Y yyyyMMdd-HHmmss parquet diff --git a/integration-tests/transforms/0072-read-parquet-file.hpl b/integration-tests/transforms/0072-read-parquet-file.hpl index 49d0c631684..ec58354513e 100644 --- a/integration-tests/transforms/0072-read-parquet-file.hpl +++ b/integration-tests/transforms/0072-read-parquet-file.hpl @@ -133,7 +133,7 @@ limitations under the License. -1 file - files/parquet-test-00-0001.parquet + output/parquet-test-00-0001.parquet -1 N String diff --git a/integration-tests/transforms/main-0072-parquet-input-test-delete.hwf b/integration-tests/transforms/main-0072-parquet-input-test-delete.hwf index 8ad4d17b976..81f5f5f60b8 100644 --- a/integration-tests/transforms/main-0072-parquet-input-test-delete.hwf +++ b/integration-tests/transforms/main-0072-parquet-input-test-delete.hwf @@ -106,7 +106,7 @@ limitations under the License. DELETE_FILE N - ${PROJECT_HOME}/files/parquet-test-00-0001.parquet + ${PROJECT_HOME}/output/parquet-test-00-0001.parquet N 864 192 diff --git a/plugins/transforms/execprocess/src/main/java/org/apache/hop/pipeline/transforms/execprocess/ExecProcess.java b/plugins/transforms/execprocess/src/main/java/org/apache/hop/pipeline/transforms/execprocess/ExecProcess.java index d275946a6a1..6dc0bd22ace 100644 --- a/plugins/transforms/execprocess/src/main/java/org/apache/hop/pipeline/transforms/execprocess/ExecProcess.java +++ b/plugins/transforms/execprocess/src/main/java/org/apache/hop/pipeline/transforms/execprocess/ExecProcess.java @@ -218,7 +218,10 @@ private void execProcess(String[] process, ProcessResult processresult) throws H // execute process try { if (!meta.isArgumentsInFields()) { - p = data.runtime.exec(new String[] {process[0]}); + // String form tokenizes the command line (spaces), matching the historical + // "single process field" behavior. String[] form would treat the whole line + // as one executable name (regression from #7565). + p = data.runtime.exec(process[0]); } else { p = data.runtime.exec(process); } diff --git a/plugins/transforms/execprocess/src/test/java/org/apache/hop/pipeline/transforms/execprocess/ExecProcessTest.java b/plugins/transforms/execprocess/src/test/java/org/apache/hop/pipeline/transforms/execprocess/ExecProcessTest.java index 7cfd1e7c7a4..1f5672b4a3a 100644 --- a/plugins/transforms/execprocess/src/test/java/org/apache/hop/pipeline/transforms/execprocess/ExecProcessTest.java +++ b/plugins/transforms/execprocess/src/test/java/org/apache/hop/pipeline/transforms/execprocess/ExecProcessTest.java @@ -33,6 +33,7 @@ import org.apache.hop.core.row.IRowMeta; import org.apache.hop.core.row.RowMeta; import org.apache.hop.core.row.value.ValueMetaString; +import org.apache.hop.core.util.TestUtil; import org.apache.hop.junit.rules.RestoreHopEngineEnvironmentExtension; import org.apache.hop.pipeline.transform.TransformMeta; import org.apache.hop.pipeline.transforms.mock.TransformMockHelper; @@ -51,7 +52,8 @@ class ExecProcessTest { TransformMockHelper smh; @BeforeEach - void setUp() { + void setUp() throws HopException { + TestUtil.registerTestPluginTypes(); smh = new TransformMockHelper<>( "Execute a process", ExecProcessMeta.class, ExecProcessData.class); From 056e647b7b8e5cef7063994e6df35e25d19d1037 Mon Sep 17 00:00:00 2001 From: mattcasters Date: Mon, 20 Jul 2026 17:26:20 +0200 Subject: [PATCH 2/2] issue #7582 : adopt StringTokenizer-based ExecProcess fix from #7581 Use explicit whitespace tokenization before Runtime.exec(String[]) for single-command mode (avoids the deprecated String overload), and add a unit test for tokenizeCommandLine. Credit: @leehaut in #7581. --- .../transforms/execprocess/ExecProcess.java | 22 +++++++++++++++---- .../execprocess/ExecProcessTest.java | 14 +++++++++--- 2 files changed, 29 insertions(+), 7 deletions(-) diff --git a/plugins/transforms/execprocess/src/main/java/org/apache/hop/pipeline/transforms/execprocess/ExecProcess.java b/plugins/transforms/execprocess/src/main/java/org/apache/hop/pipeline/transforms/execprocess/ExecProcess.java index 6dc0bd22ace..922ba648033 100644 --- a/plugins/transforms/execprocess/src/main/java/org/apache/hop/pipeline/transforms/execprocess/ExecProcess.java +++ b/plugins/transforms/execprocess/src/main/java/org/apache/hop/pipeline/transforms/execprocess/ExecProcess.java @@ -23,6 +23,7 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.List; +import java.util.StringTokenizer; import java.util.concurrent.CompletableFuture; import java.util.concurrent.CompletionException; import java.util.concurrent.CountDownLatch; @@ -218,10 +219,10 @@ private void execProcess(String[] process, ProcessResult processresult) throws H // execute process try { if (!meta.isArgumentsInFields()) { - // String form tokenizes the command line (spaces), matching the historical - // "single process field" behavior. String[] form would treat the whole line - // as one executable name (regression from #7565). - p = data.runtime.exec(process[0]); + // Match historical Runtime.exec(String) whitespace tokenization without using the + // deprecated String overload. A single-element String[] would treat the whole command + // line (e.g. "/bin/echo hop-single") as the executable name. + p = data.runtime.exec(tokenizeCommandLine(process[0])); } else { p = data.runtime.exec(process); } @@ -326,6 +327,19 @@ private void execProcess(String[] process, ProcessResult processresult) throws H } } + /** + * Tokenize a command line the same way Runtime.exec(String) historically did (whitespace via + * {@link StringTokenizer}). + */ + static String[] tokenizeCommandLine(String command) { + StringTokenizer st = new StringTokenizer(command); + String[] cmdArray = new String[st.countTokens()]; + for (int i = 0; st.hasMoreTokens(); i++) { + cmdArray[i] = st.nextToken(); + } + return cmdArray; + } + private String getOutputString(BufferedReader b) throws IOException { StringBuilder returnValueBuffer = new StringBuilder(); String line; diff --git a/plugins/transforms/execprocess/src/test/java/org/apache/hop/pipeline/transforms/execprocess/ExecProcessTest.java b/plugins/transforms/execprocess/src/test/java/org/apache/hop/pipeline/transforms/execprocess/ExecProcessTest.java index 1f5672b4a3a..832673cbcde 100644 --- a/plugins/transforms/execprocess/src/test/java/org/apache/hop/pipeline/transforms/execprocess/ExecProcessTest.java +++ b/plugins/transforms/execprocess/src/test/java/org/apache/hop/pipeline/transforms/execprocess/ExecProcessTest.java @@ -25,6 +25,8 @@ import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.when; +import java.util.Arrays; +import java.util.List; import org.apache.hop.core.HopEnvironment; import org.apache.hop.core.IRowSet; import org.apache.hop.core.QueueRowSet; @@ -33,7 +35,6 @@ import org.apache.hop.core.row.IRowMeta; import org.apache.hop.core.row.RowMeta; import org.apache.hop.core.row.value.ValueMetaString; -import org.apache.hop.core.util.TestUtil; import org.apache.hop.junit.rules.RestoreHopEngineEnvironmentExtension; import org.apache.hop.pipeline.transform.TransformMeta; import org.apache.hop.pipeline.transforms.mock.TransformMockHelper; @@ -52,8 +53,7 @@ class ExecProcessTest { TransformMockHelper smh; @BeforeEach - void setUp() throws HopException { - TestUtil.registerTestPluginTypes(); + void setUp() { smh = new TransformMockHelper<>( "Execute a process", ExecProcessMeta.class, ExecProcessData.class); @@ -217,6 +217,14 @@ void processRow_failWhenNotSuccess_setsErrors() throws HopException { assertEquals(1, transform.getErrors()); } + @Test + void tokenizeCommandLine_splitsOnWhitespaceLikeRuntimeExecString() { + assertEquals( + Arrays.asList("/bin/echo", "hop-single"), + Arrays.asList(ExecProcess.tokenizeCommandLine("/bin/echo hop-single"))); + assertEquals(List.of("cmd"), Arrays.asList(ExecProcess.tokenizeCommandLine("cmd"))); + } + @Test void processRow_emptyProcessField_throws() throws HopException { HopEnvironment.init();