issue #7486 : A dedicated Spark pipeline execution engine

.gitignore

@@ -57,6 +57,7 @@ integration-tests/spreadsheet/files/exel-multi-part_4.xlsx
 integration-tests/json/output
 integration-tests/mdi/files/excelwriter-mdi-test.xlsx
 integration-tests/spreadsheet/files/sample-file-append.xlsx
+integration-tests/spark-native/output
 
 assemblies/debug/audit
 
@@ -66,11 +67,8 @@ rap/src/main/resources/org/apache/hop/ui/hopgui/monaco/monaco-files.list
 
 # Screenshots captured by SWTbot
 screenshots/
-<<<<<<< Updated upstream
 
 # Docker script leaving audit data
 docker/local-data/
 
-=======
-/docker/local-data/
->>>>>>> Stashed changes
+/integration-tests/spreadsheet/files/exelwriter-testfile

NOTICE

@@ -9,3 +9,13 @@ Hitachi Vantara (https://www.hitachivantara.com//).
 
 This product includes software developed by
 JADAPTIVE (https://www.jadaptive.com/)
+
+This product bundles Delta Lake (io.delta) under the Apache License, Version 2.0.
+See: https://github.com/delta-io/delta
+
+This product bundles Apache Iceberg under the Apache License, Version 2.0.
+See: https://iceberg.apache.org/
+
+This product bundles Unity Catalog client libraries (transitive dependency of
+Delta Lake) under the Apache License, Version 2.0.
+See: https://github.com/unitycatalog/unitycatalog

assemblies/client/pom.xml

@@ -67,6 +67,12 @@
             <version>${project.version}</version>
             <type>zip</type>
         </dependency>
+        <dependency>
+            <groupId>org.apache.hop</groupId>
+            <artifactId>hop-engines-spark</artifactId>
+            <version>${project.version}</version>
+            <type>zip</type>
+        </dependency>
         <dependency>
             <groupId>org.apache.hop</groupId>
             <artifactId>hop-ui</artifactId>

assemblies/debug/pom.xml

@@ -355,6 +355,12 @@
             <version>${project.version}</version>
             <scope>provided</scope>
         </dependency>
+        <dependency>
+            <groupId>org.apache.hop</groupId>
+            <artifactId>hop-engines-spark</artifactId>
+            <version>${project.version}</version>
+            <scope>provided</scope>
+        </dependency>
         <dependency>
             <groupId>org.apache.hop</groupId>
             <artifactId>hop-misc-git</artifactId>

assemblies/plugins/pom.xml

@@ -618,6 +618,12 @@
             <version>${project.version}</version>
             <type>zip</type>
         </dependency>
+        <dependency>
+            <groupId>org.apache.hop</groupId>
+            <artifactId>hop-engines-spark</artifactId>
+            <version>${project.version}</version>
+            <type>zip</type>
+        </dependency>
         <dependency>
             <groupId>org.apache.hop</groupId>
             <artifactId>hop-misc-async</artifactId>

core/src/main/java/org/apache/hop/core/gui/plugin/GuiElementType.java

@@ -20,6 +20,11 @@
 public enum GuiElementType {
   NONE, // To disable default options
   TEXT,
+  /**
+   * Multi-line text widget ({@code SWT.MULTI}). Height in lines is set via {@link
+   * GuiWidgetElement#multiLineTextHeight()}.
+   */
+  MULTI_LINE_TEXT,
   FILENAME, // Text widget with browse button
   FOLDER, // Text widget with browse button
   COMBO,

core/src/main/java/org/apache/hop/core/gui/plugin/GuiElements.java

@@ -49,6 +49,9 @@ public class GuiElements extends BaseGuiElements implements Comparable<GuiElemen
 
   private boolean password;
 
+  /** Height in text lines for MULTI_LINE_TEXT (minimum 1). */
+  private int multiLineTextHeight = 1;
+
   private String fieldName;
 
   private Class<?> fieldClass;
@@ -104,6 +107,7 @@ public GuiElements(GuiWidgetElement guiElement, Field field) {
     this.disabledImage = null;
     this.variablesEnabled = guiElement.variables();
     this.password = guiElement.password();
+    this.multiLineTextHeight = Math.max(1, guiElement.multiLineTextHeight());
     this.ignored = guiElement.ignored();
     this.addingSeparator = guiElement.separator();
     this.label = getTranslation(guiElement.label(), fieldPackageName, field.getDeclaringClass());
@@ -143,6 +147,7 @@ public GuiElements(GuiWidgetElement guiElement, Method method, ClassLoader class
     this.disabledImage = null;
     this.variablesEnabled = guiElement.variables();
     this.password = guiElement.password();
+    this.multiLineTextHeight = Math.max(1, guiElement.multiLineTextHeight());
     this.ignored = guiElement.ignored();
     this.addingSeparator = guiElement.separator();
     this.label = getTranslation(guiElement.label(), methodPackageName, method.getDeclaringClass());
@@ -370,6 +375,22 @@ public void setPassword(boolean password) {
     this.password = password;
   }
 
+  /**
+   * Preferred height for {@link GuiElementType#MULTI_LINE_TEXT}, in text lines (at least 1).
+   *
+   * @return value of multiLineTextHeight
+   */
+  public int getMultiLineTextHeight() {
+    return multiLineTextHeight;
+  }
+
+  /**
+   * @param multiLineTextHeight The multiLineTextHeight to set (clamped to at least 1)
+   */
+  public void setMultiLineTextHeight(int multiLineTextHeight) {
+    this.multiLineTextHeight = Math.max(1, multiLineTextHeight);
+  }
+
   /**
    * Gets fieldName
    *

core/src/main/java/org/apache/hop/core/gui/plugin/GuiWidgetElement.java

@@ -80,6 +80,14 @@
    */
   boolean password() default false;
 
+  /**
+   * Preferred height of a {@link GuiElementType#MULTI_LINE_TEXT} widget, in text lines (not
+   * pixels). Default is 1. Values less than 1 are treated as 1. Ignored for other element types.
+   *
+   * @return height in lines
+   */
+  int multiLineTextHeight() default 1;
+
   /**
    * @return true if the widget supports variables
    */

dist/spark-native-demo/README.md

@@ -0,0 +1,33 @@
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+  http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied.  See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+# Staging location moved
+
+Native Spark demo artifacts default to **`/tmp/spark-demo-dist`** (not under the git tree).
+
+```bash
+./plugins/engines/spark/src/samples/spark-demo/scripts/prepare-dist.sh
+# → /tmp/spark-demo-dist/{hop-native-spark4-submit.jar,spark-demo.zip,hop-data/,…}
+
+docker compose -f docker/integration-tests/integration-tests-spark-native-cluster.yaml \
+  up -d --build --scale spark-worker=2
+# hop-data is bind-mounted to /data/hop-data in the cluster (outputs + executions on the host)
+```
+
+Override with `DIST_DIR` / `HOP_DIST_DIR` / `HOP_DATA_HOST_DIR` if you prefer other paths.

docker/integration-tests/integration-tests-base.yaml

@@ -33,4 +33,6 @@ services:
       - FLASK_ENV=docker
       # Optional workflow basename filter (substring or glob). Set by run-tests-docker.sh.
       - TEST_FILTER=${TEST_FILTER:-}
+      # Skip Google Sheets ITs when no real service-account JSON key is provided.
+      - SKIP_GOOGLE_SHEETS=${SKIP_GOOGLE_SHEETS:-false}
     command: [ "bash", "-c", "/files/scripts/run-tests.sh ${PROJECT_NAME}" ]

docker/integration-tests/integration-tests-beam-base.yaml

@@ -35,4 +35,5 @@ services:
       - HOP_SPARK_CLIENT_VERSION=${HOP_SPARK_CLIENT_VERSION:-}
       # Optional workflow basename filter (substring or glob). Set by run-tests-docker.sh.
       - TEST_FILTER=${TEST_FILTER:-}
+      - SKIP_GOOGLE_SHEETS=${SKIP_GOOGLE_SHEETS:-false}
     command: [ "bash", "-c", "/files/scripts/run-tests.sh ${PROJECT_NAME}" ]

docker/integration-tests/integration-tests-iceberg.yaml

@@ -0,0 +1,27 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# Apache Iceberg table ITs on native SparkPipelineEngine (local[2]).
+# Requires engines-spark with lib/iceberg (default assembly).
+services:
+  hop-iceberg-test:
+    extends:
+      file: integration-tests-beam-base.yaml
+      service: integration_test
+    environment:
+      - FLASK_ENV=docker
+      - HOP_OPTIONS=-Dspark.ui.enabled=false

docker/integration-tests/integration-tests-lakehouse.yaml

@@ -0,0 +1,27 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# Delta Lake table ITs on native SparkPipelineEngine (local[2]).
+# Requires engines-spark with lib/delta (default assembly).
+services:
+  hop-lakehouse-test:
+    extends:
+      file: integration-tests-beam-base.yaml
+      service: integration_test
+    environment:
+      - FLASK_ENV=docker
+      - HOP_OPTIONS=-Dspark.ui.enabled=false

docker/integration-tests/integration-tests-spark-native-cluster.yaml

@@ -0,0 +1,105 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# Native Spark 4.1 standalone cluster for the spark-demo walk-through.
+#
+# Proves multi-node spark-submit with a Hop project package (Simple Mapping /
+# Workflow Executor under ${PROJECT_HOME}) and shared data on a host-visible
+# bind mount (HOP_DATA=file:///data/hop-data inside the containers).
+#
+# Host data plane (default): ${HOP_DIST_DIR:-/tmp/spark-demo-dist}/hop-data
+#   → container /data/hop-data  (CSV seeds, out/, packages/, executions/)
+#
+# From the repository root:
+#
+#   # Prepare fat jar + package zip into /tmp/spark-demo-dist (see sample scripts)
+#   ./plugins/engines/spark/src/samples/spark-demo/scripts/prepare-dist.sh
+#
+#   docker compose -f docker/integration-tests/integration-tests-spark-native-cluster.yaml \
+#     up -d --build --scale spark-worker=2
+#
+#   # Mapping demo (default)
+#   docker compose -f docker/integration-tests/integration-tests-spark-native-cluster.yaml \
+#     exec spark /opt/hop-samples/spark-demo/scripts/spark-submit-demo.sh
+#
+#   # Workflow Executor country folders demo
+#   docker compose -f docker/integration-tests/integration-tests-spark-native-cluster.yaml \
+#     exec spark /opt/hop-samples/spark-demo/scripts/spark-submit-workflows-demo.sh
+#
+#   # Nested Native Spark pipelines (per country) demo
+#   docker compose -f docker/integration-tests/integration-tests-spark-native-cluster.yaml \
+#     exec spark /opt/hop-samples/spark-demo/scripts/spark-submit-pipelines-demo.sh
+#
+#   # Inspect on the host (no docker exec required)
+#   ls -la /tmp/spark-demo-dist/hop-data/out /tmp/spark-demo-dist/hop-data/executions
+#
+# Spark master UI: http://localhost:8080
+# Override SPARK_VERSION if needed (default matches plugins/engines/spark Spark 4.1.x pin).
+# Override data host path: HOP_DATA_HOST_DIR=/somewhere/else docker compose …
+
+services:
+  spark:
+    build:
+      context: ./spark
+      dockerfile: Dockerfile.master
+      args:
+        SPARK_VERSION: ${SPARK_VERSION:-4.1.2}
+        HADOOP_VERSION: ${HADOOP_VERSION:-3}
+        BASE_URL: ${SPARK_BASE_URL:-https://dlcdn.apache.org/spark}
+    hostname: spark
+    environment:
+      - INIT_DAEMON_STEP=setup_spark
+      - SPARK_MASTER_HOST=spark
+    ports:
+      - "8080:8080"
+      - "7077:7077"
+    volumes:
+      # Host-prepared fat jar + project package zip
+      - ${HOP_DIST_DIR:-/tmp/spark-demo-dist}:/opt/hop-dist:ro
+      # Sample project sources + scripts
+      - ../../plugins/engines/spark/src/samples:/opt/hop-samples:ro
+      # Shared data plane (master + workers): bind-mounted so the host can inspect
+      # outputs and execution-info files without docker exec.
+      # Default: /tmp/spark-demo-dist/hop-data  (override HOP_DATA_HOST_DIR; if you change
+      # HOP_DIST_DIR, set HOP_DATA_HOST_DIR=${HOP_DIST_DIR}/hop-data as well)
+      - ${HOP_DATA_HOST_DIR:-/tmp/spark-demo-dist/hop-data}:/data/hop-data
+    healthcheck:
+      test: ["CMD", "curl", "-fsS", "http://127.0.0.1:8080/"]
+      interval: 5s
+      timeout: 5s
+      retries: 24
+      start_period: 40s
+
+  spark-worker:
+    build:
+      context: ./spark
+      dockerfile: Dockerfile.worker
+      args:
+        SPARK_VERSION: ${SPARK_VERSION:-4.1.2}
+        HADOOP_VERSION: ${HADOOP_VERSION:-3}
+        BASE_URL: ${SPARK_BASE_URL:-https://dlcdn.apache.org/spark}
+    depends_on:
+      spark:
+        condition: service_healthy
+    environment:
+      - SPARK_MASTER=spark://spark:7077
+      - SPARK_WORKER_CORES=${SPARK_WORKER_CORES:-2}
+      - SPARK_WORKER_MEMORY=${SPARK_WORKER_MEMORY:-2g}
+    volumes:
+      - ${HOP_DATA_HOST_DIR:-/tmp/spark-demo-dist/hop-data}:/data/hop-data
+    ports:
+      - "8081"

docker/integration-tests/integration-tests-spark-native.yaml

@@ -0,0 +1,29 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# Native SparkPipelineEngine smoke (local[2] only — no Spark cluster).
+# Extends hop-beam-image so /tmp/hop-fatjar.jar is present for future cluster tests
+# (spark-local run config leaves fatJar empty for Spark 4.1 plugin classpath).
+services:
+  hop-spark-native-test:
+    extends:
+      file: integration-tests-beam-base.yaml
+      service: integration_test
+    environment:
+      - FLASK_ENV=docker
+      # Avoid Spark UI / jersey servlet clash on the driver
+      - HOP_OPTIONS=-Dspark.ui.enabled=false

docker/integration-tests/spark/scripts/master.sh

@@ -32,5 +32,6 @@ ln -sf /dev/stdout $SPARK_MASTER_LOG/spark-master.out
 # # start history server and thrift server 
 # /spark/sbin/start-thriftserver.sh
 
+# Spark 3 accepted --ip; Spark 4 Master only accepts --host / -h (see Master --help).
 cd /spark/bin && /spark/sbin/../bin/spark-class org.apache.spark.deploy.master.Master \
-    --ip $SPARK_MASTER_HOST --port $SPARK_MASTER_PORT --webui-port $SPARK_MASTER_WEBUI_PORT >> $SPARK_MASTER_LOG/spark-master.out
+    --host $SPARK_MASTER_HOST --port $SPARK_MASTER_PORT --webui-port $SPARK_MASTER_WEBUI_PORT >> $SPARK_MASTER_LOG/spark-master.out