Alternatively enable a registered project first with {@code -j} (not {@code
+ * --project} / {@code -p}, which only name a project for create/delete/list management):
+ *
+ *
+ */
+@ConfigPlugin(
+ id = "ExportSparkProjectConfigPlugin",
+ description = "Export a Native Spark project package zip (definitions + metadata.json)")
+public class ExportSparkProjectConfigPlugin implements IConfigOptions {
+
+ @CommandLine.Option(
+ names = {"-esp", "--export-spark-project"},
+ description =
+ "Export a Native Spark project package zip (project/ tree + root metadata.json). "
+ + "Skip bulk datasets/. Pair with --export-spark-project-home= or enable a "
+ + "project with -j first (not --project/-p). MainSpark: --HopProjectPackage=.")
+ private String exportSparkProjectZip;
+
+ @CommandLine.Option(
+ names = {"--export-spark-project-home"},
+ description =
+ "Project home folder to package. Defaults to PROJECT_HOME when a project was enabled "
+ + "with -j . Prefer this for scripts so you need not register the project.")
+ private String exportSparkProjectHome;
+
+ @CommandLine.Option(
+ names = {"--export-spark-project-exclude-dirs"},
+ description =
+ "Comma-separated extra directory basenames to skip (merged with defaults and "
+ + "spark-package.json). Example: tmp,screenshots")
+ private String exportExcludeDirs;
+
+ @CommandLine.Option(
+ names = {"--export-spark-project-exclude-globs"},
+ description =
+ "Comma-separated exclude globs relative to project home (e.g. **/*.jar,**/*.zip)")
+ private String exportExcludeGlobs;
+
+ @CommandLine.Option(
+ names = {"--export-spark-project-include"},
+ description =
+ "Comma-separated relative paths to force-include even under skipped dirs "
+ + "(e.g. work/cluster-env.json)")
+ private String exportIncludePaths;
+
+ @CommandLine.Option(
+ names = {"--export-spark-project-replace-default-excludes"},
+ description =
+ "If set, do not apply built-in exclude dirs (work, datasets, target, …); use only "
+ + "spark-package.json and --export-spark-project-exclude-dirs",
+ defaultValue = "false")
+ private boolean exportReplaceDefaultExcludes;
+
+ @Override
+ public boolean handleOption(
+ ILogChannel log, IHasHopMetadataProvider hasHopMetadataProvider, IVariables variables)
+ throws HopException {
+ if (StringUtils.isEmpty(exportSparkProjectZip)) {
+ return false;
+ }
+
+ String projectHome = resolveProjectHome(variables);
+ String zip = variables.resolve(exportSparkProjectZip);
+
+ log.logBasic("Exporting Native Spark project package from home=" + projectHome);
+ log.logBasic("Destination zip: " + zip);
+
+ IHopMetadataProvider metadataProvider =
+ resolveMetadataProvider(projectHome, hasHopMetadataProvider, variables, log);
+
+ PackageExportFilter cliFilter =
+ new PackageExportFilter(
+ PackageExportFilter.splitCsv(exportExcludeDirs),
+ PackageExportFilter.splitCsv(exportExcludeGlobs),
+ PackageExportFilter.splitCsv(exportIncludePaths),
+ exportReplaceDefaultExcludes);
+ SparkProjectPackage.exportProject(projectHome, zip, metadataProvider, variables, cliFilter);
+ log.logBasic(
+ "Spark project package written. Submit with MainSpark "
+ + "--HopProjectPackage="
+ + zip
+ + " --HopPipelinePath= --HopRunConfigurationName=");
+ return true;
+ }
+
+ private String resolveProjectHome(IVariables variables) throws HopException {
+ if (StringUtils.isNotEmpty(exportSparkProjectHome)) {
+ return variables.resolve(exportSparkProjectHome);
+ }
+ String projectHome = variables.getVariable("PROJECT_HOME");
+ if (StringUtils.isNotEmpty(projectHome)) {
+ return variables.resolve(projectHome);
+ }
+ throw new HopException(
+ "No project home for export. Either pass "
+ + "--export-spark-project-home=/path/to/project "
+ + "or enable a registered project first with -j "
+ + "(short option -j). Note: --project / -p only names a project for "
+ + "create/delete/list and does not set PROJECT_HOME.");
+ }
+
+ /**
+ * Prefer {@code /metadata} on disk so export works without enabling a Hop project.
+ * Fall back to the hop-conf metadata provider (used when -j already reconfigured it).
+ */
+ private IHopMetadataProvider resolveMetadataProvider(
+ String projectHome,
+ IHasHopMetadataProvider hasHopMetadataProvider,
+ IVariables variables,
+ ILogChannel log)
+ throws HopException {
+ try {
+ String metaFolder = projectHome;
+ if (!metaFolder.endsWith("/") && !metaFolder.endsWith("\\")) {
+ metaFolder = metaFolder + Const.FILE_SEPARATOR;
+ }
+ metaFolder = metaFolder + "metadata";
+ FileObject metaFo = HopVfs.getFileObject(metaFolder);
+ if (metaFo.exists() && metaFo.getType() == FileType.FOLDER) {
+ log.logBasic("Loading metadata from " + metaFolder);
+ return new JsonMetadataProvider(Encr.getEncoder(), metaFolder, variables);
+ }
+ } catch (HopException e) {
+ throw e;
+ } catch (Exception e) {
+ throw new HopException("Unable to open project metadata folder under " + projectHome, e);
+ }
+
+ if (hasHopMetadataProvider != null && hasHopMetadataProvider.getMetadataProvider() != null) {
+ log.logBasic("Using hop-conf metadata provider (no metadata/ folder under project home)");
+ return hasHopMetadataProvider.getMetadataProvider();
+ }
+ throw new HopException(
+ "No metadata found under "
+ + projectHome
+ + "/metadata and no hop-conf metadata provider is available");
+ }
+}
diff --git a/plugins/engines/spark/src/main/java/org/apache/hop/spark/core/HopMapPartitionsFn.java b/plugins/engines/spark/src/main/java/org/apache/hop/spark/core/HopMapPartitionsFn.java
new file mode 100644
index 00000000000..e310fd2034b
--- /dev/null
+++ b/plugins/engines/spark/src/main/java/org/apache/hop/spark/core/HopMapPartitionsFn.java
@@ -0,0 +1,1110 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hop.spark.core;
+
+import java.io.File;
+import java.io.Serializable;
+import java.net.InetAddress;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Iterator;
+import java.util.List;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.hop.core.Const;
+import org.apache.hop.core.exception.HopException;
+import org.apache.hop.core.exception.HopRuntimeException;
+import org.apache.hop.core.exception.HopTransformException;
+import org.apache.hop.core.logging.LogChannel;
+import org.apache.hop.core.logging.LoggingObject;
+import org.apache.hop.core.metadata.SerializableMetadataProvider;
+import org.apache.hop.core.plugins.PluginRegistry;
+import org.apache.hop.core.plugins.TransformPluginType;
+import org.apache.hop.core.row.IRowMeta;
+import org.apache.hop.core.row.IValueMeta;
+import org.apache.hop.core.row.JsonRowMeta;
+import org.apache.hop.core.variables.IVariables;
+import org.apache.hop.core.variables.Variables;
+import org.apache.hop.core.vfs.HopVfs;
+import org.apache.hop.metadata.api.IHopMetadataProvider;
+import org.apache.hop.pipeline.PipelineHopMeta;
+import org.apache.hop.pipeline.PipelineMeta;
+import org.apache.hop.pipeline.RowProducer;
+import org.apache.hop.pipeline.SingleThreadedPipelineExecutor;
+import org.apache.hop.pipeline.TransformWithMappingMeta;
+import org.apache.hop.pipeline.engines.local.LocalPipelineEngine;
+import org.apache.hop.pipeline.transform.BaseTransform;
+import org.apache.hop.pipeline.transform.ITransform;
+import org.apache.hop.pipeline.transform.ITransformMeta;
+import org.apache.hop.pipeline.transform.RowAdapter;
+import org.apache.hop.pipeline.transform.TransformMeta;
+import org.apache.hop.pipeline.transform.TransformMetaDataCombi;
+import org.apache.hop.pipeline.transforms.dummy.DummyMeta;
+import org.apache.hop.pipeline.transforms.file.BaseFileInputMeta;
+import org.apache.hop.pipeline.transforms.injector.InjectorField;
+import org.apache.hop.pipeline.transforms.injector.InjectorMeta;
+import org.apache.hop.spark.execution.SparkTransformExecutionSampling;
+import org.apache.hop.spark.pkg.SparkProjectPackage;
+import org.apache.hop.spark.util.SparkConst;
+import org.apache.spark.TaskContext;
+import org.apache.spark.api.java.function.MapPartitionsFunction;
+import org.apache.spark.broadcast.Broadcast;
+import org.apache.spark.sql.Row;
+
+/**
+ * Executes a single Hop transform once per Spark partition via {@link
+ * SingleThreadedPipelineExecutor}. Metadata is passed as serializable strings (XML/JSON) so
+ * executors never receive live Hop graph objects.
+ */
+public class HopMapPartitionsFn implements MapPartitionsFunction, Serializable {
+ private static final long serialVersionUID = 4L;
+
+ /** Publish progress at least every this many input rows. */
+ private static final int METRICS_ROW_INTERVAL = 1000;
+
+ /** Publish progress at least every this many milliseconds. */
+ private static final long METRICS_TIME_INTERVAL_MS = 1000L;
+
+ private final List variableValues;
+ private final String metastoreJson;
+ private final String transformName;
+ private final String transformPluginId;
+ private final String transformMetaInterfaceXml;
+ private final String inputRowMetaJson;
+ private final String outputRowMetaJson;
+ private final boolean inputTransform;
+ private final List targetTransforms;
+
+ /** Names of info/side transforms (Stream Lookup "from", etc.). */
+ private final List infoTransforms;
+
+ /** Parallel to {@link #infoTransforms}: row meta JSON per info stream. */
+ private final List infoRowMetaJsons;
+
+ /** Parallel to {@link #infoTransforms}: broadcast Hop rows for each info stream. */
+ private final List>> infoBroadcasts;
+
+ private final SparkTransformMetricsAccumulator metricsAccumulator;
+ private final SparkExecutionDataAccumulator sampleDataAccumulator;
+ private final String runConfigName;
+ private final String parentLogChannelId;
+ private final String dataSamplersJson;
+
+ public HopMapPartitionsFn(
+ List variableValues,
+ String metastoreJson,
+ String transformName,
+ String transformPluginId,
+ String transformMetaInterfaceXml,
+ String inputRowMetaJson,
+ String outputRowMetaJson,
+ boolean inputTransform,
+ List targetTransforms) {
+ this(
+ variableValues,
+ metastoreJson,
+ transformName,
+ transformPluginId,
+ transformMetaInterfaceXml,
+ inputRowMetaJson,
+ outputRowMetaJson,
+ inputTransform,
+ targetTransforms,
+ null,
+ null,
+ null,
+ null,
+ null,
+ null,
+ null,
+ null);
+ }
+
+ public HopMapPartitionsFn(
+ List variableValues,
+ String metastoreJson,
+ String transformName,
+ String transformPluginId,
+ String transformMetaInterfaceXml,
+ String inputRowMetaJson,
+ String outputRowMetaJson,
+ boolean inputTransform,
+ List targetTransforms,
+ SparkTransformMetricsAccumulator metricsAccumulator) {
+ this(
+ variableValues,
+ metastoreJson,
+ transformName,
+ transformPluginId,
+ transformMetaInterfaceXml,
+ inputRowMetaJson,
+ outputRowMetaJson,
+ inputTransform,
+ targetTransforms,
+ null,
+ null,
+ null,
+ metricsAccumulator,
+ null,
+ null,
+ null,
+ null);
+ }
+
+ public HopMapPartitionsFn(
+ List variableValues,
+ String metastoreJson,
+ String transformName,
+ String transformPluginId,
+ String transformMetaInterfaceXml,
+ String inputRowMetaJson,
+ String outputRowMetaJson,
+ boolean inputTransform,
+ List targetTransforms,
+ SparkTransformMetricsAccumulator metricsAccumulator,
+ String runConfigName,
+ String parentLogChannelId,
+ String dataSamplersJson) {
+ this(
+ variableValues,
+ metastoreJson,
+ transformName,
+ transformPluginId,
+ transformMetaInterfaceXml,
+ inputRowMetaJson,
+ outputRowMetaJson,
+ inputTransform,
+ targetTransforms,
+ null,
+ null,
+ null,
+ metricsAccumulator,
+ null,
+ runConfigName,
+ parentLogChannelId,
+ dataSamplersJson);
+ }
+
+ public HopMapPartitionsFn(
+ List variableValues,
+ String metastoreJson,
+ String transformName,
+ String transformPluginId,
+ String transformMetaInterfaceXml,
+ String inputRowMetaJson,
+ String outputRowMetaJson,
+ boolean inputTransform,
+ List targetTransforms,
+ SparkTransformMetricsAccumulator metricsAccumulator,
+ SparkExecutionDataAccumulator sampleDataAccumulator,
+ String runConfigName,
+ String parentLogChannelId,
+ String dataSamplersJson) {
+ this(
+ variableValues,
+ metastoreJson,
+ transformName,
+ transformPluginId,
+ transformMetaInterfaceXml,
+ inputRowMetaJson,
+ outputRowMetaJson,
+ inputTransform,
+ targetTransforms,
+ null,
+ null,
+ null,
+ metricsAccumulator,
+ sampleDataAccumulator,
+ runConfigName,
+ parentLogChannelId,
+ dataSamplersJson);
+ }
+
+ public HopMapPartitionsFn(
+ List variableValues,
+ String metastoreJson,
+ String transformName,
+ String transformPluginId,
+ String transformMetaInterfaceXml,
+ String inputRowMetaJson,
+ String outputRowMetaJson,
+ boolean inputTransform,
+ List targetTransforms,
+ List infoTransforms,
+ List infoRowMetaJsons,
+ List>> infoBroadcasts,
+ SparkTransformMetricsAccumulator metricsAccumulator,
+ SparkExecutionDataAccumulator sampleDataAccumulator,
+ String runConfigName,
+ String parentLogChannelId,
+ String dataSamplersJson) {
+ this.variableValues = variableValues != null ? variableValues : List.of();
+ this.metastoreJson = metastoreJson;
+ this.transformName = transformName;
+ this.transformPluginId = transformPluginId;
+ this.transformMetaInterfaceXml = transformMetaInterfaceXml;
+ this.inputRowMetaJson = inputRowMetaJson;
+ this.outputRowMetaJson = outputRowMetaJson;
+ this.inputTransform = inputTransform;
+ this.targetTransforms = targetTransforms != null ? targetTransforms : List.of();
+ this.infoTransforms = infoTransforms != null ? infoTransforms : List.of();
+ this.infoRowMetaJsons = infoRowMetaJsons != null ? infoRowMetaJsons : List.of();
+ this.infoBroadcasts = infoBroadcasts != null ? infoBroadcasts : List.of();
+ this.metricsAccumulator = metricsAccumulator;
+ this.sampleDataAccumulator = sampleDataAccumulator;
+ this.runConfigName = runConfigName;
+ this.parentLogChannelId = parentLogChannelId;
+ this.dataSamplersJson = dataSamplersJson;
+ }
+
+ @Override
+ public Iterator call(Iterator input) throws Exception {
+ int copyNr = partitionId();
+ String host = localHost();
+ long partitionStartMs = System.currentTimeMillis();
+ ITransform mainTransform = null;
+ SparkTransformExecutionSampling sampling = null;
+ try {
+ SparkHop.init();
+
+ IHopMetadataProvider metadataProvider = new SerializableMetadataProvider(metastoreJson);
+ IVariables variables = new Variables();
+ for (SparkVariableValue variableValue : variableValues) {
+ if (StringUtils.isNotEmpty(variableValue.getVariable())) {
+ variables.setVariable(variableValue.getVariable(), variableValue.getValue());
+ }
+ }
+ // Spark project package: extract once per JVM into java.io.tmpdir and set PROJECT_HOME
+ // so Simple Mapping / Pipeline Executor path loads work on executors.
+ SparkProjectPackage.ensureMaterializedOnWorker(variables);
+ validateProjectHomeOnWorker(variables);
+ // Partition-scoped internal variables for classic I/O filenames (e.g. Text File Output
+ // with ${Internal.Transform.ID}). Readable stable ID = transform name + partition id.
+ applyPartitionInternalVariables(variables, transformName, copyNr);
+ // Nested Workflow/Pipeline Executor parent linkage for execution perspective drill-down.
+ applySparkTransformOwnerId(variables, parentLogChannelId, transformName, copyNr);
+
+ IRowMeta inputRowMeta =
+ StringUtils.isNotEmpty(inputRowMetaJson)
+ ? JsonRowMeta.fromJson(inputRowMetaJson)
+ : new org.apache.hop.core.row.RowMeta();
+ IRowMeta outputRowMeta = JsonRowMeta.fromJson(outputRowMetaJson);
+
+ PipelineMeta pipelineMeta = new PipelineMeta();
+ pipelineMeta.setName(transformName);
+ pipelineMeta.setPipelineType(PipelineMeta.PipelineType.SingleThreaded);
+ pipelineMeta.setMetadataProvider(metadataProvider);
+
+ if (infoTransforms.size() != infoRowMetaJsons.size()
+ || infoTransforms.size() != infoBroadcasts.size()) {
+ throw new HopException(
+ "Info stream configuration mismatch for transform '"
+ + transformName
+ + "': names="
+ + infoTransforms.size()
+ + " metas="
+ + infoRowMetaJsons.size()
+ + " broadcasts="
+ + infoBroadcasts.size());
+ }
+
+ List infoRowMetas = new ArrayList<>();
+ for (String infoRowMetaJson : infoRowMetaJsons) {
+ infoRowMetas.add(JsonRowMeta.fromJson(infoRowMetaJson));
+ }
+
+ TransformMeta mainInjectorTransformMeta = null;
+ if (!inputTransform) {
+ mainInjectorTransformMeta =
+ createInjectorTransform(
+ pipelineMeta, SparkConst.INJECTOR_TRANSFORM_NAME, inputRowMeta, 200, 200);
+ }
+
+ // Injectors for info/side streams (Stream Lookup, Validator, …) — Beam side-input pattern
+ List infoTransformMetas = new ArrayList<>();
+ for (int i = 0; i < infoTransforms.size(); i++) {
+ infoTransformMetas.add(
+ createInjectorTransform(
+ pipelineMeta, infoTransforms.get(i), infoRowMetas.get(i), 200, 350 + 150 * i));
+ }
+
+ int targetLocationY = 200;
+ List targetTransformMetas = new ArrayList<>();
+ for (String targetTransform : targetTransforms) {
+ DummyMeta dummyMeta = new DummyMeta();
+ TransformMeta targetTransformMeta = new TransformMeta(targetTransform, dummyMeta);
+ targetTransformMeta.setLocation(600, targetLocationY);
+ targetLocationY += 150;
+ targetTransformMetas.add(targetTransformMeta);
+ pipelineMeta.addTransform(targetTransformMeta);
+ }
+
+ PluginRegistry registry = PluginRegistry.getInstance();
+ ITransformMeta iTransformMeta =
+ registry.loadClass(TransformPluginType.class, transformPluginId, ITransformMeta.class);
+ if (iTransformMeta == null) {
+ throw new HopException(
+ "Unable to load transform plugin with ID "
+ + transformPluginId
+ + ", this plugin isn't in the plugin registry or classpath");
+ }
+
+ HopSparkUtil.loadTransformMetadataFromXml(
+ transformName, iTransformMeta, transformMetaInterfaceXml, metadataProvider);
+
+ // Resolve ${PROJECT_HOME}/… mapping/executor filenames on this JVM after package materialize
+ // so Simple Mapping / Pipeline Executor open the child .hpl from the local extract.
+ resolveNestedPipelineFilename(iTransformMeta, variables, transformName);
+
+ // Text File Input / Excel / etc. accept filenames from a *main* hop (not an INFO stream)
+ // and call findInputRowSet(accept_transform_name). The mini-pipeline only has the main
+ // injector named _INJECTOR_, so re-bind accept_transform_name to that injector. Filename
+ // rows must also be fully loaded before processRow drains the source (see below).
+ boolean acceptFilenamesFromMain = bindAcceptingFilenamesToMainInjector(iTransformMeta);
+
+ TransformMeta transformMeta = new TransformMeta(transformName, iTransformMeta);
+ transformMeta.setTransformPluginId(transformPluginId);
+ transformMeta.setLocation(400, 200);
+ pipelineMeta.addTransform(transformMeta);
+ // Give CurrentDirectoryResolver a folder under PROJECT_HOME when resolving relatives
+ String projectHome = variables.getVariable("PROJECT_HOME");
+ if (StringUtils.isNotEmpty(projectHome)) {
+ pipelineMeta.setFilename(
+ projectHome
+ + (projectHome.endsWith("/") || projectHome.endsWith("\\") ? "" : File.separator)
+ + "_spark_map_partitions.hpl");
+ }
+ if (!inputTransform) {
+ pipelineMeta.addPipelineHop(new PipelineHopMeta(mainInjectorTransformMeta, transformMeta));
+ }
+ for (TransformMeta infoTransformMeta : infoTransformMetas) {
+ pipelineMeta.addPipelineHop(new PipelineHopMeta(infoTransformMeta, transformMeta));
+ }
+ for (TransformMeta targetTransformMeta : targetTransformMetas) {
+ pipelineMeta.addPipelineHop(new PipelineHopMeta(transformMeta, targetTransformMeta));
+ }
+
+ // After injectors exist so Stream Lookup "from" can bind to the info injector by name
+ iTransformMeta.searchInfoAndTargetTransforms(pipelineMeta.getTransforms());
+
+ LocalPipelineEngine pipeline =
+ new LocalPipelineEngine(
+ pipelineMeta, variables, new LoggingObject("apache-spark-transform"));
+ pipeline.setMetadataProvider(metadataProvider);
+ pipeline
+ .getPipelineRunConfiguration()
+ .setName("spark-transform-local (" + transformName + ")");
+ try {
+ pipeline.prepareExecution();
+ } catch (Exception e) {
+ throw new HopException(
+ "Failed to prepare transform '"
+ + transformName
+ + "' (plugin id="
+ + transformPluginId
+ + ") on Spark partition "
+ + copyNr
+ + " host="
+ + host
+ + " PROJECT_HOME="
+ + variables.getVariable("PROJECT_HOME")
+ + " HOP_SPARK_PROJECT_PACKAGE="
+ + variables.getVariable(SparkProjectPackage.VAR_PACKAGE_URI),
+ e);
+ }
+
+ RowProducer rowProducer = null;
+ TransformMetaDataCombi mainInjectorCombi = null;
+ if (!inputTransform) {
+ rowProducer = pipeline.addRowProducer(SparkConst.INJECTOR_TRANSFORM_NAME, 0);
+ mainInjectorCombi = findCombi(pipeline, SparkConst.INJECTOR_TRANSFORM_NAME);
+ }
+ List infoRowProducers = new ArrayList<>();
+ for (String infoTransform : infoTransforms) {
+ infoRowProducers.add(pipeline.addRowProducer(infoTransform, 0));
+ }
+
+ // Non-blocking getRow/putRow (Beam pattern). Default handler busy-waits on empty hops and
+ // deadlocks Stream Lookup after readLookupValues when the main hop is empty in this thread.
+ for (TransformMetaDataCombi c : pipeline.getTransforms()) {
+ if (c.transform instanceof BaseTransform baseTransform) {
+ baseTransform.setRowHandler(new SparkRowHandler(baseTransform));
+ }
+ }
+
+ TransformMetaDataCombi transformCombi = findCombi(pipeline, transformName);
+ mainTransform = transformCombi.transform;
+ publishMetrics(mainTransform, copyNr, host, partitionStartMs, true, false);
+
+ // Execution data sampling → driver accumulator + optional local registerData
+ sampling =
+ new SparkTransformExecutionSampling(
+ transformName, parentLogChannelId, copyNr, sampleDataAccumulator);
+ try {
+ sampling.lookup(variables, metadataProvider, runConfigName, dataSamplersJson);
+ if (sampling.isActive()) {
+ sampling.registerExecutingTransform(pipeline);
+ sampling.attach(variables, pipeline, mainTransform, inputRowMeta, outputRowMeta);
+ }
+ } catch (Throwable sampleEx) {
+ // Non-fatal: pipeline must still process data. Catch Throwable so LinkageError
+ // (Jackson dual classloaders on local[*]) cannot abort the Spark stage.
+ LogChannel.GENERAL.logError(
+ "Execution data sampling disabled for transform '"
+ + transformName
+ + "' on partition "
+ + copyNr
+ + " (non-fatal): "
+ + sampleEx.getMessage(),
+ sampleEx instanceof Exception ex ? ex : new Exception(sampleEx));
+ sampling = null;
+ }
+
+ // Untagged main-path capture (no target streams) or per-target capture lists (Beam pattern)
+ List resultRows = new ArrayList<>();
+ List> targetResultRowsList = new ArrayList<>();
+ final boolean multiTarget = !targetTransforms.isEmpty();
+ if (!multiTarget) {
+ transformCombi.transform.addRowListener(
+ new RowAdapter() {
+ @Override
+ public void rowWrittenEvent(IRowMeta rowMeta, Object[] row)
+ throws HopTransformException {
+ resultRows.add(row);
+ }
+ });
+ } else {
+ for (String targetTransform : targetTransforms) {
+ List targetResultRows = new ArrayList<>();
+ targetResultRowsList.add(targetResultRows);
+ TransformMetaDataCombi targetCombi = findCombi(pipeline, targetTransform);
+ targetCombi.transform.addRowListener(
+ new RowAdapter() {
+ @Override
+ public void rowReadEvent(IRowMeta rowMeta, Object[] row)
+ throws HopTransformException {
+ targetResultRows.add(row);
+ }
+ });
+ }
+ }
+
+ SingleThreadedPipelineExecutor executor = new SingleThreadedPipelineExecutor(pipeline);
+ if (!executor.init()) {
+ throw new HopException(
+ "Error initializing single-threaded executor for transform '" + transformName + "'");
+ }
+ // Init calls setInternalVariables() with log-channel UUID / copy 0 — re-apply partition
+ // scope and beamContext so Text File Output auto-suffix and ${Internal.Transform.*} work.
+ applySparkParallelFileContext(pipeline, transformName, copyNr);
+
+ pipeline.startThreads();
+
+ // Load info streams once per partition (before main rows), same pattern as Beam TransformFn:
+ // put each side-input row through the info Injector via RowProducer, then processRow so the
+ // hop into Stream Lookup (etc.) is filled. finished() + one more processRow flags the
+ // injector done so readLookupValues / getRowFrom see isDone() and do not busy-wait.
+ for (int i = 0; i < infoTransforms.size(); i++) {
+ String infoName = infoTransforms.get(i);
+ IRowMeta infoRowMeta = infoRowMetas.get(i);
+ List infoData =
+ infoBroadcasts.get(i) != null ? infoBroadcasts.get(i).value() : Collections.emptyList();
+ RowProducer infoRowProducer = infoRowProducers.get(i);
+ TransformMetaDataCombi infoCombi = findCombi(pipeline, infoName);
+ for (Object[] infoRow : infoData) {
+ infoRowProducer.putRow(infoRowMeta, infoRow);
+ infoCombi.transform.processRow();
+ }
+ infoRowProducer.finished();
+ infoCombi.transform.processRow();
+ }
+
+ List output = new ArrayList<>();
+ MetricsThrottle throttle = new MetricsThrottle();
+ final SparkTransformExecutionSampling samplingRef = sampling;
+
+ if (inputTransform) {
+ // Source transform: drive until finished with no external input
+ driveUntilDone(
+ pipeline,
+ executor,
+ output,
+ outputRowMeta,
+ multiTarget,
+ resultRows,
+ targetTransforms,
+ targetResultRowsList,
+ throttle,
+ mainTransform,
+ copyNr,
+ host,
+ partitionStartMs,
+ samplingRef);
+ } else if (acceptFilenamesFromMain) {
+ // Accept-filenames (Text File Input, Excel, …):
+ // 1) Pre-load every partition row (filenames) into the main injector and mark finished.
+ // 2) Drive the file reader with processRow() directly until it returns false.
+ //
+ // We cannot use SingleThreadedPipelineExecutor.oneIteration() after filenames are
+ // drained: with a non-empty input rowset list it only calls processRow once per
+ // remaining hop size. After the first processRow consumes all filenames, size is 0
+ // and further oneIteration() calls never advance the reader (infinite stall after
+ // openNextFile/createReader — the log line users see just before hang).
+ long rowsSeen = 0;
+ while (input.hasNext()) {
+ Row sparkRow = input.next();
+ Object[] hopRow = HopSparkRowConverter.toHopRow(inputRowMeta, sparkRow);
+ rowProducer.putRow(inputRowMeta, hopRow, false);
+ if (mainInjectorCombi != null) {
+ mainInjectorCombi.transform.processRow();
+ }
+ rowsSeen++;
+ }
+ if (rowProducer != null) {
+ rowProducer.finished();
+ if (mainInjectorCombi != null) {
+ mainInjectorCombi.transform.processRow();
+ }
+ }
+ if (rowsSeen > 0 || throttle.shouldPublish(0)) {
+ publishMetrics(mainTransform, copyNr, host, partitionStartMs, true, false);
+ }
+ driveAcceptFilenamesUntilDone(
+ pipeline,
+ mainTransform,
+ output,
+ outputRowMeta,
+ multiTarget,
+ resultRows,
+ targetTransforms,
+ targetResultRowsList,
+ throttle,
+ copyNr,
+ host,
+ partitionStartMs,
+ samplingRef);
+ } else {
+ long rowsSeen = 0;
+ while (input.hasNext()) {
+ Row sparkRow = input.next();
+ Object[] hopRow = HopSparkRowConverter.toHopRow(inputRowMeta, sparkRow);
+ clearCapture(resultRows, targetResultRowsList);
+ rowProducer.putRow(inputRowMeta, hopRow, false);
+ // Forward the main row onto the hop before Stream Lookup's info-first processRow
+ // calls getRow() for the main stream (same timing Beam gets from topo order +
+ // non-blocking handler).
+ if (mainInjectorCombi != null) {
+ mainInjectorCombi.transform.processRow();
+ }
+ executor.oneIteration();
+ appendCapturedRows(
+ output,
+ outputRowMeta,
+ multiTarget,
+ resultRows,
+ targetTransforms,
+ targetResultRowsList);
+ rowsSeen++;
+ if (throttle.shouldPublish(1) || rowsSeen % METRICS_ROW_INTERVAL == 0) {
+ publishMetrics(mainTransform, copyNr, host, partitionStartMs, true, false);
+ flushSamplesQuietly(samplingRef, false);
+ }
+ }
+ if (rowProducer != null) {
+ rowProducer.finished();
+ if (mainInjectorCombi != null) {
+ mainInjectorCombi.transform.processRow();
+ }
+ clearCapture(resultRows, targetResultRowsList);
+ executor.oneIteration();
+ appendCapturedRows(
+ output,
+ outputRowMeta,
+ multiTarget,
+ resultRows,
+ targetTransforms,
+ targetResultRowsList);
+ }
+ }
+
+ if (pipeline.getErrors() > 0) {
+ publishMetrics(mainTransform, copyNr, host, partitionStartMs, false, true);
+ throw new HopException(
+ "Errors detected while executing transform '"
+ + transformName
+ + "' on a Spark partition");
+ }
+
+ executor.dispose();
+ publishMetrics(mainTransform, copyNr, host, partitionStartMs, false, true);
+ if (sampling != null) {
+ sampling.close();
+ sampling = null;
+ }
+ return output.iterator();
+ } catch (Exception e) {
+ if (mainTransform != null) {
+ try {
+ publishMetrics(mainTransform, copyNr, host, partitionStartMs, false, true);
+ } catch (Exception ignored) {
+ // best-effort metrics on failure
+ }
+ } else {
+ publishErrorSlice(copyNr, host, partitionStartMs);
+ }
+ if (sampling != null) {
+ try {
+ sampling.close();
+ } catch (Exception ignored) {
+ // best-effort
+ }
+ }
+ throw new HopRuntimeException(
+ "Error executing Hop transform '" + transformName + "' in Spark mapPartitions", e);
+ }
+ }
+
+ private static void flushSamplesQuietly(
+ SparkTransformExecutionSampling sampling, boolean finished) {
+ if (sampling == null || !sampling.isActive()) {
+ return;
+ }
+ try {
+ sampling.sendSamplesToLocation(finished);
+ } catch (Exception e) {
+ LogChannel.GENERAL.logError(
+ "Error flushing transform samples to execution location (non-fatal)", e);
+ }
+ }
+
+ private void publishMetrics(
+ ITransform transform,
+ int copyNr,
+ String host,
+ long startTimeMs,
+ boolean running,
+ boolean finished) {
+ if (metricsAccumulator == null || transform == null) {
+ return;
+ }
+ long endTimeMs = finished ? System.currentTimeMillis() : 0L;
+ metricsAccumulator.add(
+ new SparkTransformMetricSlice(
+ transformName,
+ copyNr,
+ host,
+ transform.getLinesRead(),
+ transform.getLinesWritten(),
+ transform.getLinesInput(),
+ transform.getLinesOutput(),
+ transform.getErrors(),
+ running,
+ finished,
+ startTimeMs,
+ endTimeMs));
+ }
+
+ private void publishErrorSlice(int copyNr, String host, long startTimeMs) {
+ if (metricsAccumulator == null) {
+ return;
+ }
+ long now = System.currentTimeMillis();
+ long start = startTimeMs > 0 ? startTimeMs : now;
+ metricsAccumulator.add(
+ new SparkTransformMetricSlice(
+ transformName, copyNr, host, 0, 0, 0, 0, 1, false, true, start, now));
+ }
+
+ private static int partitionId() {
+ TaskContext ctx = TaskContext.get();
+ return ctx != null ? ctx.partitionId() : 0;
+ }
+
+ private static String localHost() {
+ try {
+ return InetAddress.getLocalHost().getHostName();
+ } catch (Exception e) {
+ return null;
+ }
+ }
+
+ private static TransformMeta createInjectorTransform(
+ PipelineMeta pipelineMeta,
+ String injectorTransformName,
+ IRowMeta injectorRowMeta,
+ int x,
+ int y) {
+ InjectorMeta injectorMeta = new InjectorMeta();
+ for (IValueMeta valueMeta : injectorRowMeta.getValueMetaList()) {
+ injectorMeta
+ .getInjectorFields()
+ .add(
+ new InjectorField(
+ valueMeta.getName(),
+ valueMeta.getTypeDesc(),
+ Integer.toString(valueMeta.getLength()),
+ Integer.toString(valueMeta.getPrecision())));
+ }
+ TransformMeta injectorTransformMeta = new TransformMeta(injectorTransformName, injectorMeta);
+ injectorTransformMeta.setLocation(x, y);
+ pipelineMeta.addTransform(injectorTransformMeta);
+ return injectorTransformMeta;
+ }
+
+ private static TransformMetaDataCombi findCombi(LocalPipelineEngine pipeline, String name) {
+ for (TransformMetaDataCombi combi : pipeline.getTransforms()) {
+ if (combi.transformName.equals(name)) {
+ return combi;
+ }
+ }
+ throw new HopRuntimeException(
+ "Configuration error, transform '" + name + "' not found in mini-pipeline");
+ }
+
+ /**
+ * Sets partition-scoped {@code Internal.Transform.*} variables on a variable space (readable ID =
+ * {@code transformName-partitionId}).
+ */
+ static void applyPartitionInternalVariables(
+ IVariables variables, String transformName, int partitionId) {
+ if (variables == null) {
+ return;
+ }
+ String partitionIdStr = Integer.toString(partitionId);
+ String instanceId = transformName + "-" + partitionIdStr;
+ variables.setVariable(Const.INTERNAL_VARIABLE_TRANSFORM_NAME, transformName);
+ variables.setVariable(Const.INTERNAL_VARIABLE_TRANSFORM_COPYNR, partitionIdStr);
+ variables.setVariable(Const.INTERNAL_VARIABLE_TRANSFORM_ID, instanceId);
+ variables.setVariable(Const.INTERNAL_VARIABLE_TRANSFORM_BUNDLE_NR, partitionIdStr);
+ }
+
+ /**
+ * Publish the same synthetic owner id used by {@link
+ * org.apache.hop.spark.execution.SparkTransformExecutionSampling} so nested engines can parent
+ * their execution-info rows for GUI drill-down.
+ */
+ static void applySparkTransformOwnerId(
+ IVariables variables, String parentLogChannelId, String transformName, int partitionId) {
+ if (variables == null || StringUtils.isEmpty(transformName)) {
+ return;
+ }
+ String parent =
+ StringUtils.isNotEmpty(parentLogChannelId) ? parentLogChannelId : "spark-pipeline";
+ String ownerId = parent + "|" + transformName + "|" + partitionId;
+ variables.setVariable(SparkConst.VAR_TRANSFORM_OWNER_ID, ownerId);
+ }
+
+ /**
+ * After package materialize, resolve nested pipeline paths (Simple Mapping, Pipeline Executor, …)
+ * to absolute files on this JVM and verify they exist.
+ */
+ static void resolveNestedPipelineFilename(
+ ITransformMeta iTransformMeta, IVariables variables, String transformName)
+ throws HopException {
+ if (!(iTransformMeta instanceof TransformWithMappingMeta mappingMeta)) {
+ return;
+ }
+ String filename = mappingMeta.getFilename();
+ if (StringUtils.isEmpty(filename)) {
+ throw new HopException(
+ "Transform '"
+ + transformName
+ + "' is a nested pipeline transform but has no filename configured");
+ }
+ String resolved = variables != null ? variables.resolve(filename) : filename;
+ if (StringUtils.isEmpty(resolved) || resolved.contains("${") || resolved.contains("%")) {
+ throw new HopException(
+ "Could not resolve nested pipeline filename for transform '"
+ + transformName
+ + "': '"
+ + filename
+ + "' → '"
+ + resolved
+ + "' (PROJECT_HOME="
+ + (variables != null ? variables.getVariable("PROJECT_HOME") : null)
+ + ", package="
+ + (variables != null
+ ? variables.getVariable(SparkProjectPackage.VAR_PACKAGE_URI)
+ : null)
+ + "). Deploy with Upload Spark project package so PROJECT_HOME is set on every"
+ + " executor.");
+ }
+ File f = new File(resolved);
+ boolean exists = f.isFile();
+ if (!exists) {
+ try {
+ exists = HopVfs.fileExists(resolved);
+ } catch (Exception e) {
+ throw new HopException(
+ "Nested pipeline file not found for transform '"
+ + transformName
+ + "': "
+ + resolved
+ + " (from '"
+ + filename
+ + "')",
+ e);
+ }
+ }
+ if (!exists) {
+ throw new HopException(
+ "Nested pipeline file not found for transform '"
+ + transformName
+ + "': "
+ + resolved
+ + " (from '"
+ + filename
+ + "'). Check that hop-spark-package.zip contains this path under project/ "
+ + "(e.g. project/mappings/add-uuid.hpl) and re-run Deploy & run with package"
+ + " upload.");
+ }
+ mappingMeta.setFilename(resolved);
+ }
+
+ /**
+ * After package materialize, ensure PROJECT_HOME is a real directory on this JVM. Detects the
+ * common failure where only the driver extract path was broadcast and workers never re-extracted
+ * the package (nested Simple Mapping / Pipeline Executor then fail to open {@code
+ * ${PROJECT_HOME}/…}).
+ */
+ static void validateProjectHomeOnWorker(IVariables variables) throws HopException {
+ if (variables == null) {
+ return;
+ }
+ String packageUri = variables.getVariable(SparkProjectPackage.VAR_PACKAGE_URI);
+ String sparkFile = variables.getVariable(SparkProjectPackage.VAR_PACKAGE_SPARK_FILE);
+ String projectHome = variables.getVariable("PROJECT_HOME");
+ if (StringUtils.isEmpty(packageUri)
+ && StringUtils.isEmpty(sparkFile)
+ && StringUtils.isEmpty(projectHome)) {
+ return;
+ }
+ if (StringUtils.isNotEmpty(projectHome)) {
+ File home = new File(projectHome);
+ if (home.isDirectory()) {
+ return;
+ }
+ throw new HopException(
+ "PROJECT_HOME is not a directory on this Spark executor: "
+ + projectHome
+ + ". Package URI="
+ + packageUri
+ + ", SparkFiles basename="
+ + sparkFile
+ + ". The driver extract under /tmp is not visible on workers — re-run with a fat jar"
+ + " that stages the project package via SparkFiles (local copy + addFile) and/or a"
+ + " UC Volume path every node can open. Nested definition files (mappings,"
+ + " sub-pipelines) live under project/ inside the package zip.");
+ }
+ if (StringUtils.isNotEmpty(packageUri) || StringUtils.isNotEmpty(sparkFile)) {
+ throw new HopException(
+ "HOP_SPARK_PROJECT_PACKAGE is set ("
+ + packageUri
+ + ", sparkFile="
+ + sparkFile
+ + ") but PROJECT_HOME was not set after package materialization on this executor.");
+ }
+ }
+
+ /**
+ * Beam-parity parallel file context for classic I/O in mapPartitions: enable {@code beamContext}
+ * (Text File Output auto-appends {@code _transformId_bundleNr}) and re-apply partition-scoped
+ * internal variables after transform init overwrote them with log-channel UUIDs.
+ */
+ static void applySparkParallelFileContext(
+ LocalPipelineEngine pipeline, String primaryTransformName, int partitionId) {
+ if (pipeline == null) {
+ return;
+ }
+ String partitionIdStr = Integer.toString(partitionId);
+ String ownerId = pipeline.getVariable(SparkConst.VAR_TRANSFORM_OWNER_ID);
+ for (TransformMetaDataCombi combi : pipeline.getTransforms()) {
+ if (combi.data != null) {
+ combi.data.setBeamContext(true);
+ combi.data.setBeamBundleNr(partitionId);
+ }
+ if (combi.transform instanceof BaseTransform baseTransform) {
+ String name =
+ StringUtils.isNotEmpty(combi.transformName)
+ ? combi.transformName
+ : primaryTransformName;
+ // Prefer the Hop transform name for the main transform (filename templates)
+ if (primaryTransformName != null && primaryTransformName.equals(combi.transformName)) {
+ name = primaryTransformName;
+ }
+ String instanceId = name + "-" + partitionIdStr;
+ baseTransform.setVariable(Const.INTERNAL_VARIABLE_TRANSFORM_NAME, name);
+ baseTransform.setVariable(Const.INTERNAL_VARIABLE_TRANSFORM_COPYNR, partitionIdStr);
+ baseTransform.setVariable(Const.INTERNAL_VARIABLE_TRANSFORM_ID, instanceId);
+ baseTransform.setVariable(Const.INTERNAL_VARIABLE_TRANSFORM_BUNDLE_NR, partitionIdStr);
+ if (StringUtils.isNotEmpty(ownerId)) {
+ baseTransform.setVariable(SparkConst.VAR_TRANSFORM_OWNER_ID, ownerId);
+ }
+ }
+ }
+ }
+
+ /**
+ * When a file input transform accepts filenames from a previous hop, re-point {@code
+ * accept_transform_name} at the mini-pipeline main injector so {@code findInputRowSet} succeeds.
+ * Covers {@link BaseFileInputMeta} (Text File Input, JSON, VCard, …) and Excel-style metas with
+ * {@code isAcceptingFilenames}/{@code setAcceptingTransformName}.
+ *
+ * @return true when the transform consumes filename rows from the main injector this way
+ */
+ static boolean bindAcceptingFilenamesToMainInjector(ITransformMeta meta) {
+ if (meta == null) {
+ return false;
+ }
+ if (meta instanceof BaseFileInputMeta baseFile) {
+ if (baseFile.isAcceptingFilenames()) {
+ baseFile.setAcceptingTransformName(SparkConst.INJECTOR_TRANSFORM_NAME);
+ return true;
+ }
+ return false;
+ }
+ // ExcelInputMeta (and similar) are not BaseFileInputMeta but use the same contract
+ try {
+ Object accepting = meta.getClass().getMethod("isAcceptingFilenames").invoke(meta);
+ if (Boolean.TRUE.equals(accepting)) {
+ meta.getClass()
+ .getMethod("setAcceptingTransformName", String.class)
+ .invoke(meta, SparkConst.INJECTOR_TRANSFORM_NAME);
+ return true;
+ }
+ } catch (ReflectiveOperationException ignored) {
+ // not an accept-filenames transform
+ }
+ return false;
+ }
+
+ /**
+ * Drive the single-threaded mini-pipeline until it finishes (source transforms with no main input
+ * hop — Row Generator, Get File Names, etc.).
+ */
+ private void driveUntilDone(
+ LocalPipelineEngine pipeline,
+ SingleThreadedPipelineExecutor executor,
+ List output,
+ IRowMeta outputRowMeta,
+ boolean multiTarget,
+ List resultRows,
+ List targetTransforms,
+ List> targetResultRowsList,
+ MetricsThrottle throttle,
+ ITransform mainTransform,
+ int copyNr,
+ String host,
+ long partitionStartMs,
+ SparkTransformExecutionSampling samplingRef)
+ throws HopException {
+ boolean more = true;
+ while (more && !pipeline.isFinished() && pipeline.getErrors() == 0) {
+ clearCapture(resultRows, targetResultRowsList);
+ more = executor.oneIteration();
+ appendCapturedRows(
+ output, outputRowMeta, multiTarget, resultRows, targetTransforms, targetResultRowsList);
+ if (throttle.shouldPublish(resultRows.size())) {
+ publishMetrics(mainTransform, copyNr, host, partitionStartMs, true, false);
+ flushSamplesQuietly(samplingRef, false);
+ }
+ }
+ }
+
+ /**
+ * After filename rows are pre-loaded onto the main injector, call {@code processRow()} on the
+ * file-input transform until it finishes reading content.
+ *
+ *
{@link SingleThreadedPipelineExecutor#oneIteration()} only schedules {@code processRow} once
+ * per remaining input-rowset size when the transform has input hops. After accept-filenames
+ * drains those rows, size is 0 and further iterations never advance the reader.
+ */
+ private void driveAcceptFilenamesUntilDone(
+ LocalPipelineEngine pipeline,
+ ITransform mainTransform,
+ List output,
+ IRowMeta outputRowMeta,
+ boolean multiTarget,
+ List resultRows,
+ List targetTransforms,
+ List> targetResultRowsList,
+ MetricsThrottle throttle,
+ int copyNr,
+ String host,
+ long partitionStartMs,
+ SparkTransformExecutionSampling samplingRef)
+ throws HopException {
+ boolean more = true;
+ long contentRows = 0;
+ while (more && !pipeline.isFinished() && pipeline.getErrors() == 0) {
+ clearCapture(resultRows, targetResultRowsList);
+ more = mainTransform.processRow();
+ appendCapturedRows(
+ output, outputRowMeta, multiTarget, resultRows, targetTransforms, targetResultRowsList);
+ contentRows += resultRows.size();
+ if (throttle.shouldPublish(resultRows.size()) || contentRows % METRICS_ROW_INTERVAL == 0) {
+ publishMetrics(mainTransform, copyNr, host, partitionStartMs, true, false);
+ flushSamplesQuietly(samplingRef, false);
+ }
+ }
+ }
+
+ private static void clearCapture(
+ List resultRows, List> targetResultRowsList) {
+ resultRows.clear();
+ for (List list : targetResultRowsList) {
+ list.clear();
+ }
+ }
+
+ private static void appendCapturedRows(
+ List output,
+ IRowMeta outputRowMeta,
+ boolean multiTarget,
+ List resultRows,
+ List targetTransforms,
+ List> targetResultRowsList)
+ throws HopException {
+ if (!multiTarget) {
+ for (Object[] hopRow : resultRows) {
+ output.add(HopSparkRowConverter.toSparkRow(outputRowMeta, hopRow));
+ }
+ return;
+ }
+ for (int t = 0; t < targetTransforms.size(); t++) {
+ String tag = targetTransforms.get(t);
+ for (Object[] hopRow : targetResultRowsList.get(t)) {
+ output.add(HopSparkRowConverter.toTaggedSparkRow(tag, outputRowMeta, hopRow));
+ }
+ }
+ }
+
+ /** Throttles metric publishes by wall-clock time (row interval handled by caller). */
+ private static final class MetricsThrottle implements Serializable {
+ private static final long serialVersionUID = 1L;
+ private long lastPublishMs = 0L;
+
+ boolean shouldPublish(int rowsThisBatch) {
+ long now = System.currentTimeMillis();
+ if (lastPublishMs == 0L || now - lastPublishMs >= METRICS_TIME_INTERVAL_MS) {
+ lastPublishMs = now;
+ return true;
+ }
+ return false;
+ }
+ }
+}
diff --git a/plugins/engines/spark/src/main/java/org/apache/hop/spark/core/HopSparkRowConverter.java b/plugins/engines/spark/src/main/java/org/apache/hop/spark/core/HopSparkRowConverter.java
new file mode 100644
index 00000000000..55a0a462b4c
--- /dev/null
+++ b/plugins/engines/spark/src/main/java/org/apache/hop/spark/core/HopSparkRowConverter.java
@@ -0,0 +1,225 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hop.spark.core;
+
+import java.math.BigDecimal;
+import java.nio.ByteBuffer;
+import java.sql.Timestamp;
+import java.util.ArrayList;
+import java.util.Date;
+import java.util.List;
+import org.apache.hop.core.exception.HopException;
+import org.apache.hop.core.row.IRowMeta;
+import org.apache.hop.core.row.IValueMeta;
+import org.apache.spark.sql.Row;
+import org.apache.spark.sql.RowFactory;
+import org.apache.spark.sql.types.DataType;
+import org.apache.spark.sql.types.DataTypes;
+import org.apache.spark.sql.types.DecimalType;
+import org.apache.spark.sql.types.Metadata;
+import org.apache.spark.sql.types.StructField;
+import org.apache.spark.sql.types.StructType;
+
+/**
+ * Bidirectional conversion between Hop {@link IRowMeta}/{@code Object[]} and Spark {@link
+ * StructType}/{@link Row}.
+ */
+public final class HopSparkRowConverter {
+
+ private HopSparkRowConverter() {}
+
+ public static StructType toStructType(IRowMeta rowMeta) throws HopException {
+ if (rowMeta == null) {
+ return new StructType();
+ }
+ List fields = new ArrayList<>(rowMeta.size());
+ for (int i = 0; i < rowMeta.size(); i++) {
+ IValueMeta valueMeta = rowMeta.getValueMeta(i);
+ fields.add(
+ new StructField(valueMeta.getName(), toDataType(valueMeta), true, Metadata.empty()));
+ }
+ return new StructType(fields.toArray(new StructField[0]));
+ }
+
+ public static DataType toDataType(IValueMeta valueMeta) throws HopException {
+ return switch (valueMeta.getType()) {
+ case IValueMeta.TYPE_STRING, IValueMeta.TYPE_INET -> DataTypes.StringType;
+ case IValueMeta.TYPE_INTEGER -> DataTypes.LongType;
+ case IValueMeta.TYPE_NUMBER -> DataTypes.DoubleType;
+ case IValueMeta.TYPE_BIGNUMBER -> DataTypes.createDecimalType(38, 18);
+ case IValueMeta.TYPE_BOOLEAN -> DataTypes.BooleanType;
+ case IValueMeta.TYPE_DATE -> DataTypes.TimestampType;
+ case IValueMeta.TYPE_TIMESTAMP -> DataTypes.TimestampType;
+ case IValueMeta.TYPE_BINARY -> DataTypes.BinaryType;
+ case IValueMeta.TYPE_NONE, IValueMeta.TYPE_SERIALIZABLE -> DataTypes.StringType;
+ default ->
+ throw new HopException(
+ "Unsupported Hop type for Spark conversion: "
+ + valueMeta.getTypeDesc()
+ + " ("
+ + valueMeta.getName()
+ + ")");
+ };
+ }
+
+ public static Row toSparkRow(IRowMeta rowMeta, Object[] hopRow) throws HopException {
+ Object[] values = new Object[rowMeta.size()];
+ for (int i = 0; i < rowMeta.size(); i++) {
+ Object hopValue = hopRow == null || i >= hopRow.length ? null : hopRow[i];
+ values[i] = toSparkValue(rowMeta.getValueMeta(i), hopValue);
+ }
+ return RowFactory.create(values);
+ }
+
+ /**
+ * Spark row with a leading target-branch tag column ({@link HopSparkUtil#TARGET_TAG_COLUMN}) for
+ * multi-output transforms (Filter Rows, Switch/Case).
+ */
+ public static Row toTaggedSparkRow(String targetTag, IRowMeta rowMeta, Object[] hopRow)
+ throws HopException {
+ Object[] values = new Object[rowMeta.size() + 1];
+ values[0] = targetTag != null ? targetTag : HopSparkUtil.MAIN_TARGET_TAG;
+ for (int i = 0; i < rowMeta.size(); i++) {
+ Object hopValue = hopRow == null || i >= hopRow.length ? null : hopRow[i];
+ values[i + 1] = toSparkValue(rowMeta.getValueMeta(i), hopValue);
+ }
+ return RowFactory.create(values);
+ }
+
+ /** Schema for {@link #toTaggedSparkRow}: tag string + payload fields. */
+ public static StructType toTaggedStructType(IRowMeta rowMeta) throws HopException {
+ StructType payload = toStructType(rowMeta);
+ StructField tagField =
+ new StructField(
+ HopSparkUtil.TARGET_TAG_COLUMN, DataTypes.StringType, false, Metadata.empty());
+ StructField[] fields = new StructField[payload.size() + 1];
+ fields[0] = tagField;
+ for (int i = 0; i < payload.size(); i++) {
+ fields[i + 1] = payload.fields()[i];
+ }
+ return new StructType(fields);
+ }
+
+ public static Object[] toHopRow(IRowMeta rowMeta, Row sparkRow) throws HopException {
+ Object[] hopRow = new Object[rowMeta.size()];
+ if (sparkRow == null) {
+ return hopRow;
+ }
+ // Tolerate short Spark rows (schema drift) — pad remaining Hop fields with null.
+ int available = Math.min(rowMeta.size(), sparkRow.length());
+ for (int i = 0; i < available; i++) {
+ hopRow[i] =
+ toHopValue(rowMeta.getValueMeta(i), sparkRow.isNullAt(i) ? null : sparkRow.get(i));
+ }
+ return hopRow;
+ }
+
+ static Object toSparkValue(IValueMeta valueMeta, Object hopValue) throws HopException {
+ if (hopValue == null) {
+ return null;
+ }
+ try {
+ return switch (valueMeta.getType()) {
+ case IValueMeta.TYPE_STRING, IValueMeta.TYPE_INET -> valueMeta.getString(hopValue);
+ case IValueMeta.TYPE_INTEGER -> valueMeta.getInteger(hopValue);
+ case IValueMeta.TYPE_NUMBER -> valueMeta.getNumber(hopValue);
+ case IValueMeta.TYPE_BIGNUMBER -> valueMeta.getBigNumber(hopValue);
+ case IValueMeta.TYPE_BOOLEAN -> valueMeta.getBoolean(hopValue);
+ case IValueMeta.TYPE_DATE, IValueMeta.TYPE_TIMESTAMP -> {
+ Date date = valueMeta.getDate(hopValue);
+ yield date == null ? null : new Timestamp(date.getTime());
+ }
+ case IValueMeta.TYPE_BINARY -> valueMeta.getBinary(hopValue);
+ default -> valueMeta.getString(hopValue);
+ };
+ } catch (Exception e) {
+ throw new HopException(
+ "Error converting Hop value to Spark for field '" + valueMeta.getName() + "'", e);
+ }
+ }
+
+ static Object toHopValue(IValueMeta valueMeta, Object sparkValue) throws HopException {
+ if (sparkValue == null) {
+ return null;
+ }
+ try {
+ return switch (valueMeta.getType()) {
+ case IValueMeta.TYPE_STRING, IValueMeta.TYPE_INET -> sparkValue.toString();
+ case IValueMeta.TYPE_INTEGER -> {
+ if (sparkValue instanceof Number number) {
+ yield number.longValue();
+ }
+ yield Long.parseLong(sparkValue.toString());
+ }
+ case IValueMeta.TYPE_NUMBER -> {
+ if (sparkValue instanceof Number number) {
+ yield number.doubleValue();
+ }
+ yield Double.parseDouble(sparkValue.toString());
+ }
+ case IValueMeta.TYPE_BIGNUMBER -> {
+ if (sparkValue instanceof BigDecimal bigDecimal) {
+ yield bigDecimal;
+ }
+ if (sparkValue instanceof Number number) {
+ yield BigDecimal.valueOf(number.doubleValue());
+ }
+ yield new BigDecimal(sparkValue.toString());
+ }
+ case IValueMeta.TYPE_BOOLEAN -> {
+ if (sparkValue instanceof Boolean bool) {
+ yield bool;
+ }
+ yield Boolean.parseBoolean(sparkValue.toString());
+ }
+ case IValueMeta.TYPE_DATE, IValueMeta.TYPE_TIMESTAMP -> {
+ if (sparkValue instanceof Timestamp timestamp) {
+ yield new Date(timestamp.getTime());
+ }
+ if (sparkValue instanceof Date date) {
+ yield date;
+ }
+ if (sparkValue instanceof java.time.Instant instant) {
+ yield Date.from(instant);
+ }
+ yield sparkValue;
+ }
+ case IValueMeta.TYPE_BINARY -> {
+ if (sparkValue instanceof byte[] bytes) {
+ yield bytes;
+ }
+ if (sparkValue instanceof ByteBuffer buffer) {
+ byte[] bytes = new byte[buffer.remaining()];
+ buffer.get(bytes);
+ yield bytes;
+ }
+ yield sparkValue;
+ }
+ default -> sparkValue.toString();
+ };
+ } catch (Exception e) {
+ throw new HopException(
+ "Error converting Spark value to Hop for field '" + valueMeta.getName() + "'", e);
+ }
+ }
+
+ /** Whether a Spark DataType is a decimal (for tests). */
+ public static boolean isDecimal(DataType dataType) {
+ return dataType instanceof DecimalType;
+ }
+}
diff --git a/plugins/engines/spark/src/main/java/org/apache/hop/spark/core/HopSparkUtil.java b/plugins/engines/spark/src/main/java/org/apache/hop/spark/core/HopSparkUtil.java
new file mode 100644
index 00000000000..c4d425a6f06
--- /dev/null
+++ b/plugins/engines/spark/src/main/java/org/apache/hop/spark/core/HopSparkUtil.java
@@ -0,0 +1,81 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hop.spark.core;
+
+import org.apache.hop.core.exception.HopException;
+import org.apache.hop.core.xml.XmlHandler;
+import org.apache.hop.core.xml.XmlHandlerCache;
+import org.apache.hop.metadata.api.IHopMetadataProvider;
+import org.apache.hop.pipeline.transform.ITransformMeta;
+import org.apache.hop.pipeline.transform.TransformMeta;
+import org.w3c.dom.Document;
+import org.w3c.dom.Node;
+
+public final class HopSparkUtil {
+
+ private static final Object LOAD_LOCK = new Object();
+
+ /**
+ * Marker value for the main (non-targeted) output branch in tagged mapPartitions rows. Must not
+ * collide with a real transform name.
+ */
+ public static final String MAIN_TARGET_TAG = "__main__";
+
+ /** Temporary column used to split multi-target mapPartitions output into Datasets. */
+ public static final String TARGET_TAG_COLUMN = "_hop_target";
+
+ private HopSparkUtil() {}
+
+ /**
+ * Dataset-map key for a named target stream (matches Beam {@code HopBeamUtil.createTargetTupleId}
+ * string format; no dependency on the Beam plugin).
+ */
+ public static String createTargetTupleId(String transformName, String targetTransformName) {
+ return transformName + " - TARGET - " + targetTransformName;
+ }
+
+ public static void loadTransformMetadataFromXml(
+ String transformName,
+ ITransformMeta iTransformMeta,
+ String transformXml,
+ IHopMetadataProvider metadataProvider)
+ throws HopException {
+ synchronized (LOAD_LOCK) {
+ Document transformDocument = XmlHandler.loadXmlString(transformXml);
+ if (transformDocument == null) {
+ throw new HopException("Unable to load transform XML document from : " + transformXml);
+ }
+ Node transformNode = XmlHandler.getSubNode(transformDocument, TransformMeta.XML_TAG);
+ if (transformNode == null) {
+ throw new HopException(
+ "Unable to find XML tag " + TransformMeta.XML_TAG + " from : " + transformXml);
+ }
+ try {
+ iTransformMeta.loadXml(transformNode, metadataProvider);
+ } catch (Exception e) {
+ throw new HopException(
+ "There was an error loading transform metadata information (loadXml) for transform '"
+ + transformName
+ + "'",
+ e);
+ } finally {
+ XmlHandlerCache.getInstance().clear();
+ }
+ }
+ }
+}
diff --git a/plugins/engines/spark/src/main/java/org/apache/hop/spark/core/SparkExecutionDataAccumulator.java b/plugins/engines/spark/src/main/java/org/apache/hop/spark/core/SparkExecutionDataAccumulator.java
new file mode 100644
index 00000000000..8b279dbc5d5
--- /dev/null
+++ b/plugins/engines/spark/src/main/java/org/apache/hop/spark/core/SparkExecutionDataAccumulator.java
@@ -0,0 +1,132 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hop.spark.core;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+import org.apache.spark.util.AccumulatorV2;
+
+/**
+ * Collects JSON-serialized {@code ExecutionData} samples from Spark executors and merges them on
+ * the driver. Keyed by owner id (transform copy log channel id) so the latest flush per copy wins.
+ *
+ *
Caching execution info locations keep a single parent {@code CacheEntry} in the driver
+ * process; executor-side {@code registerData} cannot reliably attach samples to that entry.
+ * Shipping samples through this accumulator lets the driver register them where the parent entry
+ * already exists.
+ */
+public class SparkExecutionDataAccumulator extends AccumulatorV2> {
+
+ private static final long serialVersionUID = 1L;
+
+ /** Delimiter between ownerId and JSON payload in accumulator elements. */
+ private static final char SEP = '\u0001';
+
+ private final Map byOwnerId;
+
+ public SparkExecutionDataAccumulator() {
+ this.byOwnerId = new HashMap<>();
+ }
+
+ private SparkExecutionDataAccumulator(Map byOwnerId) {
+ this.byOwnerId = byOwnerId;
+ }
+
+ /**
+ * Pack owner id + JSON for {@link #add(String)}. Prefer {@link #addSample(String, String)} from
+ * Hop code.
+ */
+ public static String pack(String ownerId, String executionDataJson) {
+ if (ownerId == null) {
+ ownerId = "";
+ }
+ if (executionDataJson == null) {
+ executionDataJson = "";
+ }
+ return ownerId + SEP + executionDataJson;
+ }
+
+ public void addSample(String ownerId, String executionDataJson) {
+ add(pack(ownerId, executionDataJson));
+ }
+
+ @Override
+ public boolean isZero() {
+ synchronized (byOwnerId) {
+ return byOwnerId.isEmpty();
+ }
+ }
+
+ @Override
+ public AccumulatorV2> copy() {
+ synchronized (byOwnerId) {
+ return new SparkExecutionDataAccumulator(new HashMap<>(byOwnerId));
+ }
+ }
+
+ @Override
+ public void reset() {
+ synchronized (byOwnerId) {
+ byOwnerId.clear();
+ }
+ }
+
+ @Override
+ public void add(String v) {
+ if (v == null || v.isEmpty()) {
+ return;
+ }
+ int idx = v.indexOf(SEP);
+ if (idx < 0) {
+ return;
+ }
+ String ownerId = v.substring(0, idx);
+ String json = v.substring(idx + 1);
+ if (ownerId.isEmpty() || json.isEmpty()) {
+ return;
+ }
+ synchronized (byOwnerId) {
+ byOwnerId.put(ownerId, json);
+ }
+ }
+
+ @Override
+ public void merge(AccumulatorV2> other) {
+ if (other == null) {
+ return;
+ }
+ Map otherMap = other.value();
+ if (otherMap == null || otherMap.isEmpty()) {
+ return;
+ }
+ synchronized (byOwnerId) {
+ byOwnerId.putAll(otherMap);
+ }
+ }
+
+ @Override
+ public Map value() {
+ synchronized (byOwnerId) {
+ if (byOwnerId.isEmpty()) {
+ return Collections.emptyMap();
+ }
+ return Collections.unmodifiableMap(new HashMap<>(byOwnerId));
+ }
+ }
+}
diff --git a/plugins/engines/spark/src/main/java/org/apache/hop/spark/core/SparkHop.java b/plugins/engines/spark/src/main/java/org/apache/hop/spark/core/SparkHop.java
new file mode 100644
index 00000000000..61d852b3739
--- /dev/null
+++ b/plugins/engines/spark/src/main/java/org/apache/hop/spark/core/SparkHop.java
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hop.spark.core;
+
+import org.apache.hop.core.Const;
+import org.apache.hop.core.HopEnvironment;
+import org.apache.hop.core.exception.HopException;
+import org.apache.hop.core.plugins.PluginRegistry;
+import org.apache.hop.core.xml.XmlHandlerCache;
+
+/** Hop initialization helpers for Spark executors (mirrors BeamHop). */
+public final class SparkHop {
+
+ private SparkHop() {}
+
+ public static boolean isInitialized() {
+ return HopEnvironment.isInitialized();
+ }
+
+ public static void init() throws HopException {
+ synchronized (PluginRegistry.getInstance()) {
+ // Don't create hop config files on every executor JVM
+ System.setProperty(Const.HOP_AUTO_CREATE_CONFIG, "N");
+ HopEnvironment.init();
+ XmlHandlerCache.getInstance();
+ }
+ }
+}
diff --git a/plugins/engines/spark/src/main/java/org/apache/hop/spark/core/SparkInfoStreamSupport.java b/plugins/engines/spark/src/main/java/org/apache/hop/spark/core/SparkInfoStreamSupport.java
new file mode 100644
index 00000000000..17802e0e5ab
--- /dev/null
+++ b/plugins/engines/spark/src/main/java/org/apache/hop/spark/core/SparkInfoStreamSupport.java
@@ -0,0 +1,99 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hop.spark.core;
+
+import java.util.ArrayList;
+import java.util.List;
+import org.apache.hop.core.exception.HopException;
+import org.apache.hop.core.row.IRowMeta;
+import org.apache.spark.api.java.JavaSparkContext;
+import org.apache.spark.broadcast.Broadcast;
+import org.apache.spark.sql.Dataset;
+import org.apache.spark.sql.Row;
+import org.apache.spark.sql.SparkSession;
+
+/**
+ * Collects an info/side Dataset into Hop rows and broadcasts it to executors (Beam {@code
+ * View.asList} analogue).
+ *
+ *
Info streams must fit in memory on the driver and on each executor that receives the
+ * broadcast.
+ */
+public final class SparkInfoStreamSupport {
+
+ /** Soft guard: fail fast if an info stream is unexpectedly huge. 0 = unlimited. */
+ public static final int DEFAULT_MAX_INFO_ROWS = 5_000_000;
+
+ private SparkInfoStreamSupport() {}
+
+ /**
+ * Collect {@code infoDataset} as Hop {@code Object[]} rows and broadcast them.
+ *
+ * @param spark active session
+ * @param infoDataset source Dataset (already computed in the transform graph)
+ * @param rowMeta Hop row metadata matching the Dataset columns
+ * @param transformName name of the info source (for error messages)
+ * @param maxRows maximum rows to collect; use {@link #DEFAULT_MAX_INFO_ROWS} or {@literal <=0}
+ * for unlimited
+ * @return broadcast of Hop rows (empty list if Dataset has no rows)
+ */
+ public static Broadcast> broadcastInfoRows(
+ SparkSession spark,
+ Dataset infoDataset,
+ IRowMeta rowMeta,
+ String transformName,
+ int maxRows)
+ throws HopException {
+ if (spark == null || infoDataset == null) {
+ throw new HopException(
+ "Cannot broadcast info stream for transform '"
+ + transformName
+ + "': null session/dataset");
+ }
+ if (rowMeta == null) {
+ throw new HopException(
+ "Cannot broadcast info stream for transform '" + transformName + "': null row meta");
+ }
+
+ List sparkRows = infoDataset.collectAsList();
+ if (maxRows > 0 && sparkRows.size() > maxRows) {
+ throw new HopException(
+ "Info stream from transform '"
+ + transformName
+ + "' has "
+ + sparkRows.size()
+ + " rows which exceeds the maximum allowed ("
+ + maxRows
+ + "). Reduce the lookup size or raise the limit.");
+ }
+
+ List hopRows = new ArrayList<>(sparkRows.size());
+ for (Row sparkRow : sparkRows) {
+ hopRows.add(HopSparkRowConverter.toHopRow(rowMeta, sparkRow));
+ }
+
+ JavaSparkContext jsc = JavaSparkContext.fromSparkContext(spark.sparkContext());
+ return jsc.broadcast(hopRows);
+ }
+
+ public static Broadcast> broadcastInfoRows(
+ SparkSession spark, Dataset infoDataset, IRowMeta rowMeta, String transformName)
+ throws HopException {
+ return broadcastInfoRows(spark, infoDataset, rowMeta, transformName, DEFAULT_MAX_INFO_ROWS);
+ }
+}
diff --git a/plugins/engines/spark/src/main/java/org/apache/hop/spark/core/SparkNativeMetrics.java b/plugins/engines/spark/src/main/java/org/apache/hop/spark/core/SparkNativeMetrics.java
new file mode 100644
index 00000000000..0c19f06a8fa
--- /dev/null
+++ b/plugins/engines/spark/src/main/java/org/apache/hop/spark/core/SparkNativeMetrics.java
@@ -0,0 +1,210 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hop.spark.core;
+
+import java.io.Serializable;
+import java.net.InetAddress;
+import java.util.Iterator;
+import java.util.Objects;
+import org.apache.spark.TaskContext;
+import org.apache.spark.api.java.JavaRDD;
+import org.apache.spark.sql.Dataset;
+import org.apache.spark.sql.Row;
+import org.apache.spark.sql.types.StructType;
+
+/**
+ * Instruments native Spark {@link Dataset} stages so row flow is visible in Hop {@code
+ * EngineMetrics}. Inserts a pass-through {@code mapPartitions} that reports absolute counters into
+ * the same {@link SparkTransformMetricsAccumulator} used by {@link HopMapPartitionsFn}.
+ *
+ *
Counters are only updated when the lineage is materialized (an action). Per-partition {@code
+ * copyNr} matches {@link TaskContext#partitionId()}.
+ */
+public final class SparkNativeMetrics {
+
+ /** How counters are attributed for a native Dataset stage. */
+ public enum Role {
+ /** Source / file input: physical input + rows produced. */
+ INPUT,
+ /** Sink / file output: rows consumed + physical output. */
+ OUTPUT,
+ /** Intermediate Dataset op: rows in and out of the stage (output cardinality). */
+ TRANSFORM
+ }
+
+ private static final int ROW_INTERVAL = 1000;
+ private static final long TIME_INTERVAL_MS = 1000L;
+
+ private SparkNativeMetrics() {}
+
+ /**
+ * Wrap {@code dataset} so that when it is computed, each partition reports metrics for {@code
+ * transformName}. Returns {@code dataset} unchanged when accumulator or name is null.
+ */
+ public static Dataset track(
+ Dataset dataset,
+ String transformName,
+ SparkTransformMetricsAccumulator accumulator,
+ Role role) {
+ if (dataset == null
+ || accumulator == null
+ || transformName == null
+ || transformName.isEmpty()) {
+ return dataset;
+ }
+ Role effectiveRole = role != null ? role : Role.TRANSFORM;
+ StructType schema = dataset.schema();
+ JavaRDD tracked =
+ dataset
+ .toJavaRDD()
+ .mapPartitions(
+ iterator ->
+ new CountingIterator(iterator, transformName, accumulator, effectiveRole),
+ true);
+ return dataset.sparkSession().createDataFrame(tracked, schema);
+ }
+
+ static final class CountingIterator implements Iterator, Serializable {
+ private static final long serialVersionUID = 1L;
+
+ private final Iterator source;
+ private final String transformName;
+ private final SparkTransformMetricsAccumulator accumulator;
+ private final Role role;
+ private final int copyNr;
+ private final String host;
+
+ private long count;
+ private long partitionStartMs;
+ private long lastPublishMs;
+ private boolean finishedPublished;
+ private boolean startedPublished;
+
+ CountingIterator(
+ Iterator source,
+ String transformName,
+ SparkTransformMetricsAccumulator accumulator,
+ Role role) {
+ this.source = Objects.requireNonNull(source, "source");
+ this.transformName = transformName;
+ this.accumulator = accumulator;
+ this.role = role;
+ this.copyNr = partitionId();
+ this.host = localHost();
+ }
+
+ @Override
+ public boolean hasNext() {
+ ensureStarted();
+ if (source.hasNext()) {
+ return true;
+ }
+ publishFinished();
+ return false;
+ }
+
+ @Override
+ public Row next() {
+ ensureStarted();
+ Row row = source.next();
+ count++;
+ if (shouldPublishProgress()) {
+ publish(true, false);
+ }
+ return row;
+ }
+
+ private void ensureStarted() {
+ if (!startedPublished) {
+ startedPublished = true;
+ partitionStartMs = System.currentTimeMillis();
+ lastPublishMs = partitionStartMs;
+ publish(true, false);
+ }
+ }
+
+ private boolean shouldPublishProgress() {
+ if (count > 0 && count % ROW_INTERVAL == 0) {
+ return true;
+ }
+ long now = System.currentTimeMillis();
+ if (now - lastPublishMs >= TIME_INTERVAL_MS) {
+ lastPublishMs = now;
+ return true;
+ }
+ return false;
+ }
+
+ private void publishFinished() {
+ if (!finishedPublished) {
+ finishedPublished = true;
+ publish(false, true);
+ }
+ }
+
+ private void publish(boolean running, boolean finished) {
+ long read = 0;
+ long written = 0;
+ long input = 0;
+ long output = 0;
+ switch (role) {
+ case INPUT:
+ input = count;
+ written = count;
+ break;
+ case OUTPUT:
+ read = count;
+ output = count;
+ break;
+ case TRANSFORM:
+ default:
+ read = count;
+ written = count;
+ break;
+ }
+ long endTimeMs = finished ? System.currentTimeMillis() : 0L;
+ accumulator.add(
+ new SparkTransformMetricSlice(
+ transformName,
+ copyNr,
+ host,
+ read,
+ written,
+ input,
+ output,
+ 0,
+ running,
+ finished,
+ partitionStartMs,
+ endTimeMs));
+ }
+
+ private static int partitionId() {
+ TaskContext ctx = TaskContext.get();
+ return ctx != null ? ctx.partitionId() : 0;
+ }
+
+ private static String localHost() {
+ try {
+ return InetAddress.getLocalHost().getHostName();
+ } catch (Exception e) {
+ return null;
+ }
+ }
+ }
+}
diff --git a/plugins/engines/spark/src/main/java/org/apache/hop/spark/core/SparkRowHandler.java b/plugins/engines/spark/src/main/java/org/apache/hop/spark/core/SparkRowHandler.java
new file mode 100644
index 00000000000..dff5a7a3932
--- /dev/null
+++ b/plugins/engines/spark/src/main/java/org/apache/hop/spark/core/SparkRowHandler.java
@@ -0,0 +1,128 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hop.spark.core;
+
+import java.util.List;
+import org.apache.hop.core.IRowSet;
+import org.apache.hop.core.exception.HopException;
+import org.apache.hop.core.exception.HopTransformException;
+import org.apache.hop.core.row.IRowMeta;
+import org.apache.hop.pipeline.transform.BaseTransform;
+import org.apache.hop.pipeline.transform.IRowHandler;
+import org.apache.hop.pipeline.transform.IRowListener;
+
+/**
+ * Non-blocking row I/O for single-threaded mini-pipelines on Spark executors (same idea as Beam's
+ * {@code BeamRowHandler}).
+ *
+ *
The default {@link BaseTransform} row handler busy-waits when an input hop is empty and not
+ * yet done. That deadlocks Stream Lookup: after {@code readLookupValues()} it calls {@code
+ * getRow()} for the main stream while the single-threaded executor has not yet (or cannot) produce
+ * the next main row on another thread.
+ */
+public class SparkRowHandler implements IRowHandler {
+
+ private final BaseTransform transform;
+
+ public SparkRowHandler(BaseTransform transform) {
+ this.transform = transform;
+ }
+
+ /**
+ * Prefer live rowsets from the transform (not a snapshot at construction): hop wiring and {@code
+ * addRowProducer} can still change after the handler is installed.
+ */
+ private IRowSet firstInput() {
+ List inputRowSets = transform.getInputRowSets();
+ return inputRowSets == null || inputRowSets.isEmpty() ? null : inputRowSets.get(0);
+ }
+
+ private IRowSet firstOutput() {
+ List outputRowSets = transform.getOutputRowSets();
+ return outputRowSets == null || outputRowSets.isEmpty() ? null : outputRowSets.get(0);
+ }
+
+ @Override
+ public Object[] getRow() throws HopException {
+ IRowSet inputRowSet = firstInput();
+ if (inputRowSet == null) {
+ return null;
+ }
+ Object[] row = inputRowSet.getRow();
+ if (row != null) {
+ transform.incrementLinesRead();
+ transform.setInputRowMeta(inputRowSet.getRowMeta());
+ List rowListeners = transform.getRowListeners();
+ for (IRowListener rowListener : rowListeners) {
+ rowListener.rowReadEvent(inputRowSet.getRowMeta(), row);
+ }
+ }
+ return row;
+ }
+
+ @Override
+ public void putRow(IRowMeta rowMeta, Object[] row) throws HopTransformException {
+ List rowListeners = transform.getRowListeners();
+ for (IRowListener rowListener : rowListeners) {
+ rowListener.rowWrittenEvent(rowMeta, row);
+ }
+ IRowSet outputRowSet = firstOutput();
+ if (outputRowSet != null) {
+ outputRowSet.putRow(rowMeta, row);
+ }
+ // Always count written (including leaf transforms with no output hop)
+ transform.incrementLinesWritten();
+ }
+
+ @Override
+ public void putError(
+ IRowMeta rowMeta,
+ Object[] row,
+ long nrErrors,
+ String errorDescriptions,
+ String fieldNames,
+ String errorCodes)
+ throws HopTransformException {
+ transform.handlePutError(
+ transform, rowMeta, row, nrErrors, errorDescriptions, fieldNames, errorCodes);
+ }
+
+ @Override
+ public void putRowTo(IRowMeta rowMeta, Object[] row, IRowSet rowSet)
+ throws HopTransformException {
+ List rowListeners = transform.getRowListeners();
+ for (IRowListener listener : rowListeners) {
+ listener.rowWrittenEvent(rowMeta, row);
+ }
+ rowSet.putRow(rowMeta, row);
+ transform.incrementLinesWritten();
+ }
+
+ @Override
+ public Object[] getRowFrom(IRowSet rowSet) throws HopTransformException {
+ Object[] row = rowSet.getRow();
+ if (row != null) {
+ transform.incrementLinesRead();
+ List rowListeners = transform.getRowListeners();
+ for (IRowListener listener : rowListeners) {
+ listener.rowReadEvent(rowSet.getRowMeta(), row);
+ }
+ }
+ return row;
+ }
+}
diff --git a/plugins/engines/spark/src/main/java/org/apache/hop/spark/core/SparkTransformMetricSlice.java b/plugins/engines/spark/src/main/java/org/apache/hop/spark/core/SparkTransformMetricSlice.java
new file mode 100644
index 00000000000..d62f734dc7e
--- /dev/null
+++ b/plugins/engines/spark/src/main/java/org/apache/hop/spark/core/SparkTransformMetricSlice.java
@@ -0,0 +1,232 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hop.spark.core;
+
+import java.io.Serializable;
+import java.util.Objects;
+
+/**
+ * Snapshot of Hop transform counters for one Spark partition (engine component copy).
+ *
+ *
Counters are absolute values from the mini-pipeline / native tracker on that partition.
+ * Merging multiple snapshots for the same key takes the maximum of each counter (progress updates),
+ * not a sum. Timing uses earliest start and latest end (epoch millis).
+ */
+public class SparkTransformMetricSlice implements Serializable {
+ private static final long serialVersionUID = 2L;
+
+ private final String transformName;
+ private final int copyNr;
+ private final String host;
+ private final long linesRead;
+ private final long linesWritten;
+ private final long linesInput;
+ private final long linesOutput;
+ private final long errors;
+ private final boolean running;
+ private final boolean finished;
+
+ /** Wall-clock start of this partition's work (epoch ms), or 0 if unknown. */
+ private final long startTimeMs;
+
+ /**
+ * Wall-clock end of this partition's work (epoch ms). 0 while still running so the GUI can treat
+ * duration as "live" until finish.
+ */
+ private final long endTimeMs;
+
+ public SparkTransformMetricSlice(
+ String transformName,
+ int copyNr,
+ String host,
+ long linesRead,
+ long linesWritten,
+ long linesInput,
+ long linesOutput,
+ long errors,
+ boolean running,
+ boolean finished,
+ long startTimeMs,
+ long endTimeMs) {
+ this.transformName = transformName;
+ this.copyNr = copyNr;
+ this.host = host;
+ this.linesRead = linesRead;
+ this.linesWritten = linesWritten;
+ this.linesInput = linesInput;
+ this.linesOutput = linesOutput;
+ this.errors = errors;
+ this.running = running;
+ this.finished = finished;
+ this.startTimeMs = startTimeMs;
+ this.endTimeMs = endTimeMs;
+ }
+
+ /** Map key for merge-by-partition. */
+ public String key() {
+ return transformName + '\0' + copyNr;
+ }
+
+ /**
+ * Merge two absolute snapshots for the same partition: max counters; min start; max end when
+ * finished; finished wins over running.
+ */
+ public SparkTransformMetricSlice mergeWith(SparkTransformMetricSlice other) {
+ if (other == null) {
+ return this;
+ }
+ boolean mergedFinished = this.finished || other.finished;
+ boolean mergedRunning = !mergedFinished && (this.running || other.running);
+ String mergedHost = this.host != null ? this.host : other.host;
+ long mergedStart = minPositive(this.startTimeMs, other.startTimeMs);
+ long mergedEnd;
+ if (mergedFinished) {
+ mergedEnd = Math.max(this.endTimeMs, other.endTimeMs);
+ // If finish was reported without a clock stamp, fall back to the later of known ends/starts
+ if (mergedEnd <= 0) {
+ mergedEnd = Math.max(this.endTimeMs, other.endTimeMs);
+ }
+ } else {
+ // Still running: keep end unset so UI duration keeps ticking
+ mergedEnd = 0L;
+ }
+ return new SparkTransformMetricSlice(
+ this.transformName,
+ this.copyNr,
+ mergedHost,
+ Math.max(this.linesRead, other.linesRead),
+ Math.max(this.linesWritten, other.linesWritten),
+ Math.max(this.linesInput, other.linesInput),
+ Math.max(this.linesOutput, other.linesOutput),
+ Math.max(this.errors, other.errors),
+ mergedRunning,
+ mergedFinished,
+ mergedStart,
+ mergedEnd);
+ }
+
+ private static long minPositive(long a, long b) {
+ if (a <= 0) {
+ return b;
+ }
+ if (b <= 0) {
+ return a;
+ }
+ return Math.min(a, b);
+ }
+
+ /**
+ * Duration in milliseconds for this slice. While running (no end), uses wall clock from start to
+ * now. Returns 0 when start is unknown.
+ */
+ public long durationMs() {
+ if (startTimeMs <= 0) {
+ return 0L;
+ }
+ long end = endTimeMs > 0 ? endTimeMs : System.currentTimeMillis();
+ return Math.max(0L, end - startTimeMs);
+ }
+
+ public String getTransformName() {
+ return transformName;
+ }
+
+ public int getCopyNr() {
+ return copyNr;
+ }
+
+ public String getHost() {
+ return host;
+ }
+
+ public long getLinesRead() {
+ return linesRead;
+ }
+
+ public long getLinesWritten() {
+ return linesWritten;
+ }
+
+ public long getLinesInput() {
+ return linesInput;
+ }
+
+ public long getLinesOutput() {
+ return linesOutput;
+ }
+
+ public long getErrors() {
+ return errors;
+ }
+
+ public boolean isRunning() {
+ return running;
+ }
+
+ public boolean isFinished() {
+ return finished;
+ }
+
+ public long getStartTimeMs() {
+ return startTimeMs;
+ }
+
+ public long getEndTimeMs() {
+ return endTimeMs;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (o == null || getClass() != o.getClass()) {
+ return false;
+ }
+ SparkTransformMetricSlice that = (SparkTransformMetricSlice) o;
+ return copyNr == that.copyNr
+ && linesRead == that.linesRead
+ && linesWritten == that.linesWritten
+ && linesInput == that.linesInput
+ && linesOutput == that.linesOutput
+ && errors == that.errors
+ && running == that.running
+ && finished == that.finished
+ && startTimeMs == that.startTimeMs
+ && endTimeMs == that.endTimeMs
+ && Objects.equals(transformName, that.transformName)
+ && Objects.equals(host, that.host);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(
+ transformName,
+ copyNr,
+ host,
+ linesRead,
+ linesWritten,
+ linesInput,
+ linesOutput,
+ errors,
+ running,
+ finished,
+ startTimeMs,
+ endTimeMs);
+ }
+}
diff --git a/plugins/engines/spark/src/main/java/org/apache/hop/spark/core/SparkTransformMetricsAccumulator.java b/plugins/engines/spark/src/main/java/org/apache/hop/spark/core/SparkTransformMetricsAccumulator.java
new file mode 100644
index 00000000000..9a340925322
--- /dev/null
+++ b/plugins/engines/spark/src/main/java/org/apache/hop/spark/core/SparkTransformMetricsAccumulator.java
@@ -0,0 +1,106 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hop.spark.core;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+import org.apache.spark.util.AccumulatorV2;
+
+/**
+ * Merging Spark accumulator of per-partition Hop transform metric slices. Registered on the driver
+ * and referenced from {@link HopMapPartitionsFn} closures so executors can report progress.
+ */
+public class SparkTransformMetricsAccumulator
+ extends AccumulatorV2> {
+
+ private static final long serialVersionUID = 1L;
+
+ private final Map map;
+
+ public SparkTransformMetricsAccumulator() {
+ this.map = new HashMap<>();
+ }
+
+ private SparkTransformMetricsAccumulator(Map map) {
+ this.map = map;
+ }
+
+ @Override
+ public boolean isZero() {
+ synchronized (map) {
+ return map.isEmpty();
+ }
+ }
+
+ @Override
+ public AccumulatorV2> copy() {
+ synchronized (map) {
+ return new SparkTransformMetricsAccumulator(new HashMap<>(map));
+ }
+ }
+
+ @Override
+ public void reset() {
+ synchronized (map) {
+ map.clear();
+ }
+ }
+
+ @Override
+ public void add(SparkTransformMetricSlice v) {
+ if (v == null || v.getTransformName() == null) {
+ return;
+ }
+ synchronized (map) {
+ String key = v.key();
+ SparkTransformMetricSlice existing = map.get(key);
+ map.put(key, existing == null ? v : existing.mergeWith(v));
+ }
+ }
+
+ @Override
+ public void merge(
+ AccumulatorV2> other) {
+ if (other == null) {
+ return;
+ }
+ Map otherMap = other.value();
+ if (otherMap == null || otherMap.isEmpty()) {
+ return;
+ }
+ synchronized (map) {
+ for (Map.Entry entry : otherMap.entrySet()) {
+ SparkTransformMetricSlice existing = map.get(entry.getKey());
+ map.put(
+ entry.getKey(),
+ existing == null ? entry.getValue() : existing.mergeWith(entry.getValue()));
+ }
+ }
+ }
+
+ @Override
+ public Map value() {
+ synchronized (map) {
+ if (map.isEmpty()) {
+ return Collections.emptyMap();
+ }
+ return Collections.unmodifiableMap(new HashMap<>(map));
+ }
+ }
+}
diff --git a/plugins/engines/spark/src/main/java/org/apache/hop/spark/core/SparkVariableValue.java b/plugins/engines/spark/src/main/java/org/apache/hop/spark/core/SparkVariableValue.java
new file mode 100644
index 00000000000..c34c93b1fa7
--- /dev/null
+++ b/plugins/engines/spark/src/main/java/org/apache/hop/spark/core/SparkVariableValue.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hop.spark.core;
+
+import java.io.Serializable;
+
+/** Serializable variable name/value pair for Spark executors. */
+public class SparkVariableValue implements Serializable {
+ private static final long serialVersionUID = 1L;
+
+ private final String variable;
+ private final String value;
+
+ public SparkVariableValue(String variable, String value) {
+ this.variable = variable;
+ this.value = value;
+ }
+
+ public String getVariable() {
+ return variable;
+ }
+
+ public String getValue() {
+ return value;
+ }
+}
diff --git a/plugins/engines/spark/src/main/java/org/apache/hop/spark/engines/ISparkPipelineEngineRunConfiguration.java b/plugins/engines/spark/src/main/java/org/apache/hop/spark/engines/ISparkPipelineEngineRunConfiguration.java
new file mode 100644
index 00000000000..19c9a241cb1
--- /dev/null
+++ b/plugins/engines/spark/src/main/java/org/apache/hop/spark/engines/ISparkPipelineEngineRunConfiguration.java
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hop.spark.engines;
+
+import org.apache.hop.pipeline.config.IPipelineEngineRunConfiguration;
+
+public interface ISparkPipelineEngineRunConfiguration extends IPipelineEngineRunConfiguration {
+
+ String getSparkMaster();
+
+ String getSparkAppName();
+
+ String getFatJar();
+
+ String getSparkConfigs();
+
+ String getDriverMemory();
+
+ String getExecutorMemory();
+
+ String getExecutorCores();
+
+ String getTempLocation();
+
+ String getPluginsToStage();
+
+ /**
+ * Optional multi-line path scheme map for Spark Dataset / Lake PATH I/O ({@code from=to} per
+ * line, e.g. {@code s3=s3a}). Rewrites only the URI scheme before {@code load}/{@code save};
+ * classic Hop VFS paths are unaffected.
+ */
+ String getPathSchemeMap();
+
+ /**
+ * Default execution mode for generic (mapPartitions) Hop transforms: {@code DISTRIBUTED} or
+ * {@code DRIVER_ONLY}. Per-transform overrides can force either mode via transform attributes.
+ */
+ String getGenericTransformRunMode();
+}
diff --git a/plugins/engines/spark/src/main/java/org/apache/hop/spark/engines/SparkPipelineEngine.java b/plugins/engines/spark/src/main/java/org/apache/hop/spark/engines/SparkPipelineEngine.java
new file mode 100644
index 00000000000..2b4c56ef52e
--- /dev/null
+++ b/plugins/engines/spark/src/main/java/org/apache/hop/spark/engines/SparkPipelineEngine.java
@@ -0,0 +1,1841 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hop.spark.engines;
+
+import java.util.ArrayList;
+import java.util.Calendar;
+import java.util.Collections;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.Timer;
+import java.util.TimerTask;
+import java.util.concurrent.ConcurrentHashMap;
+import lombok.Getter;
+import lombok.Setter;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.hop.core.Const;
+import org.apache.hop.core.IRowSet;
+import org.apache.hop.core.Result;
+import org.apache.hop.core.exception.HopException;
+import org.apache.hop.core.extension.ExtensionPointHandler;
+import org.apache.hop.core.extension.HopExtensionPoint;
+import org.apache.hop.core.json.HopJson;
+import org.apache.hop.core.logging.ILogChannel;
+import org.apache.hop.core.logging.ILoggingObject;
+import org.apache.hop.core.logging.LogChannel;
+import org.apache.hop.core.logging.LogLevel;
+import org.apache.hop.core.logging.LoggingObjectType;
+import org.apache.hop.core.parameters.DuplicateParamException;
+import org.apache.hop.core.parameters.INamedParameterDefinitions;
+import org.apache.hop.core.parameters.INamedParameters;
+import org.apache.hop.core.parameters.NamedParameters;
+import org.apache.hop.core.parameters.UnknownParamException;
+import org.apache.hop.core.plugins.EngineCompatibility;
+import org.apache.hop.core.plugins.IPlugin;
+import org.apache.hop.core.util.ExecutorUtil;
+import org.apache.hop.core.variables.IVariables;
+import org.apache.hop.core.variables.Variables;
+import org.apache.hop.execution.Execution;
+import org.apache.hop.execution.ExecutionBuilder;
+import org.apache.hop.execution.ExecutionData;
+import org.apache.hop.execution.ExecutionDataSetMeta;
+import org.apache.hop.execution.ExecutionInfoLocation;
+import org.apache.hop.execution.ExecutionState;
+import org.apache.hop.execution.ExecutionStateBuilder;
+import org.apache.hop.execution.ExecutionType;
+import org.apache.hop.execution.IExecutionInfoLocation;
+import org.apache.hop.execution.sampler.IExecutionDataSampler;
+import org.apache.hop.execution.sampler.IExecutionDataSamplerStore;
+import org.apache.hop.metadata.api.IHopMetadataProvider;
+import org.apache.hop.pipeline.IExecutionFinishedListener;
+import org.apache.hop.pipeline.IExecutionStartedListener;
+import org.apache.hop.pipeline.IExecutionStoppedListener;
+import org.apache.hop.pipeline.Pipeline;
+import org.apache.hop.pipeline.PipelineMeta;
+import org.apache.hop.pipeline.config.IPipelineEngineRunConfiguration;
+import org.apache.hop.pipeline.config.PipelineRunConfiguration;
+import org.apache.hop.pipeline.engine.EngineComponent;
+import org.apache.hop.pipeline.engine.EngineComponent.ComponentExecutionStatus;
+import org.apache.hop.pipeline.engine.EngineMetrics;
+import org.apache.hop.pipeline.engine.IEngineComponent;
+import org.apache.hop.pipeline.engine.IEngineMetric;
+import org.apache.hop.pipeline.engine.IPipelineComponentRowsReceived;
+import org.apache.hop.pipeline.engine.IPipelineEngine;
+import org.apache.hop.pipeline.engine.PipelineEngineCapabilities;
+import org.apache.hop.pipeline.engine.PipelineEnginePlugin;
+import org.apache.hop.pipeline.transform.TransformMeta;
+import org.apache.hop.spark.core.SparkExecutionDataAccumulator;
+import org.apache.hop.spark.core.SparkTransformMetricSlice;
+import org.apache.hop.spark.core.SparkTransformMetricsAccumulator;
+import org.apache.hop.spark.execution.SparkTransformExecutionSampling;
+import org.apache.hop.spark.pipeline.HopPipelineMetaToSparkConverter;
+import org.apache.hop.spark.pkg.SparkProjectPackage;
+import org.apache.hop.spark.table.LakeSessionPlan;
+import org.apache.hop.spark.util.SparkConst;
+import org.apache.hop.workflow.WorkflowMeta;
+import org.apache.hop.workflow.engine.IWorkflowEngine;
+import org.apache.spark.SparkConf;
+import org.apache.spark.sql.Dataset;
+import org.apache.spark.sql.Row;
+import org.apache.spark.sql.SparkSession;
+import scala.Option;
+
+/**
+ * Native Apache Spark 4.x pipeline engine. Converts Hop pipelines to Spark Dataset graphs and
+ * executes Hop transforms via mapPartitions mini-pipelines.
+ *
+ * @see issue #7486
+ */
+@PipelineEnginePlugin(
+ id = SparkConst.PLUGIN_ID,
+ name = SparkConst.PLUGIN_NAME,
+ description =
+ "Executes batch Hop pipelines natively on Apache Spark 4.x (mapPartitions + Dataset API)")
+@Getter
+@Setter
+public class SparkPipelineEngine extends Variables implements IPipelineEngine {
+
+ private final PipelineEngineCapabilities engineCapabilities =
+ new SparkPipelineEngineCapabilities();
+
+ private PipelineMeta pipelineMeta;
+ private String pluginId = SparkConst.PLUGIN_ID;
+ private PipelineRunConfiguration pipelineRunConfiguration;
+ private boolean preparing;
+ private boolean readyToStart;
+ private boolean running;
+ private boolean finished;
+ private boolean stopped;
+ private boolean paused;
+ private boolean hasHaltedComponents;
+ private boolean preview;
+ private int errors;
+ private IHopMetadataProvider metadataProvider;
+ private ILogChannel logChannel = LogChannel.GENERAL;
+ private String containerId;
+ private EngineMetrics engineMetrics = new EngineMetrics();
+ private Result previousResult;
+
+ private ILoggingObject parent;
+ private IPipelineEngine parentPipeline;
+ private IWorkflowEngine parentWorkflow;
+ private LogLevel logLevel;
+
+ private Date executionStartDate;
+ private Date executionEndDate;
+
+ private final List>>
+ executionStartedListeners = Collections.synchronizedList(new ArrayList<>());
+ private final List>>
+ executionFinishedListeners = Collections.synchronizedList(new ArrayList<>());
+ private final List>>
+ executionStoppedListeners = Collections.synchronizedList(new ArrayList<>());
+
+ private final Map activeSubPipelines = new HashMap<>();
+ private final Map> activeSubWorkflows = new HashMap<>();
+ private final Map extensionDataMap = Collections.synchronizedMap(new HashMap<>());
+
+ private final INamedParameters namedParams = new NamedParameters();
+
+ private String statusDescription = Pipeline.STRING_WAITING;
+ private ComponentExecutionStatus status = ComponentExecutionStatus.STATUS_EMPTY;
+
+ private HopPipelineMetaToSparkConverter converter;
+ private ISparkPipelineEngineRunConfiguration sparkEngineRunConfiguration;
+ private SparkSession sparkSession;
+
+ /**
+ * True when this engine created the {@link SparkSession} (and may stop it). False when reusing an
+ * active/default session or nesting under a parent Spark engine — never stop/cancel the shared
+ * context in that case.
+ */
+ private boolean sessionOwnedByThisEngine;
+
+ private Dataset resultDataset;
+ private Thread sparkThread;
+ private SparkTransformMetricsAccumulator metricsAccumulator;
+ private SparkExecutionDataAccumulator sampleDataAccumulator;
+ private Timer metricsRefreshTimer;
+
+ /** Execution information location from the run configuration (optional). */
+ private ExecutionInfoLocation executionInfoLocation;
+
+ private Timer executionInfoTimer;
+ private volatile boolean executionInfoClosed;
+
+ /** Stable log channel IDs per transform name + copy for execution info / GUI. */
+ private final Map componentLogChannelIds = new ConcurrentHashMap<>();
+
+ private final List> dataSamplers =
+ Collections.synchronizedList(new ArrayList<>());
+
+ public SparkPipelineEngine() {
+ super();
+ }
+
+ public SparkPipelineEngine(
+ PipelineMeta pipelineMeta, ILoggingObject parent, IVariables variables) {
+ this();
+ this.pipelineMeta = pipelineMeta;
+ setParent(parent);
+ initializeFrom(variables);
+ copyParametersFromDefinitions(pipelineMeta);
+ activateParameters(this);
+ }
+
+ @Override
+ public IPipelineEngineRunConfiguration createDefaultPipelineEngineRunConfiguration() {
+ return new SparkPipelineRunConfiguration();
+ }
+
+ public void validatePipelineRunConfigurationClass(
+ IPipelineEngineRunConfiguration engineRunConfiguration) throws HopException {
+ if (!(engineRunConfiguration instanceof SparkPipelineRunConfiguration)) {
+ throw new HopException(
+ "A native Spark pipeline engine needs a Spark run configuration, not of class "
+ + engineRunConfiguration.getClass().getName());
+ }
+ }
+
+ @Override
+ public void prepareExecution() throws HopException {
+ try {
+ executionStartDate = new Date();
+ status = ComponentExecutionStatus.STATUS_INIT;
+ statusDescription = Pipeline.STRING_INITIALIZING;
+ setPreparing(true);
+
+ IPipelineEngineRunConfiguration engineRunConfiguration =
+ pipelineRunConfiguration.getEngineRunConfiguration();
+ validatePipelineRunConfigurationClass(engineRunConfiguration);
+ sparkEngineRunConfiguration = (ISparkPipelineEngineRunConfiguration) engineRunConfiguration;
+
+ if (metadataProvider == null) {
+ throw new HopException("The Spark pipeline engine did not receive a metadata provider");
+ }
+
+ // Force a new log channel ID on every prepare/run (same as LocalPipelineEngine). Nested
+ // Pipeline Executor runs of the same child HPL would otherwise reuse one LoggingRegistry
+ // entry (same filename + parent transform) and overwrite a single execution-info file.
+ logChannel = new LogChannel(this, parent, false, true);
+ if (logLevel != null) {
+ logChannel.setLogLevel(logLevel);
+ }
+
+ // Execution info location (optional): load + initialize for later register/update/close
+ lookupExecutionInformationLocation();
+
+ converter =
+ new HopPipelineMetaToSparkConverter(
+ this,
+ pipelineMeta,
+ metadataProvider,
+ pipelineRunConfiguration.getName(),
+ sparkEngineRunConfiguration);
+ converter.validatePipeline();
+
+ logChannel.logBasic(
+ "Prepared native Spark pipeline engine with run configuration '"
+ + pipelineRunConfiguration.getName()
+ + "' (master="
+ + resolve(sparkEngineRunConfiguration.getSparkMaster())
+ + ")");
+
+ setRunning(false);
+ setReadyToStart(true);
+ } catch (Exception e) {
+ setRunning(false);
+ setReadyToStart(false);
+ setStopped(true);
+ setErrors(getErrors() + 1);
+ setPaused(false);
+ setPreparing(false);
+ throw new HopException("Error preparing Spark pipeline", e);
+ } finally {
+ setPreparing(false);
+ }
+ }
+
+ @Override
+ public void startThreads() throws HopException {
+ ClassLoader pluginCl = getClass().getClassLoader();
+ ClassLoader previousCl = Thread.currentThread().getContextClassLoader();
+ try {
+ // Spark 4 resolves classic.SparkSession via the context classloader / plugin CL
+ Thread.currentThread().setContextClassLoader(pluginCl);
+
+ setRunning(true);
+ setReadyToStart(false);
+ status = ComponentExecutionStatus.STATUS_RUNNING;
+ statusDescription = Pipeline.STRING_RUNNING;
+
+ sparkSession = createSparkSession();
+ requireClassicSparkContext(sparkSession);
+
+ // Register metrics + sample-data accumulators for mapPartitions transforms
+ metricsAccumulator = new SparkTransformMetricsAccumulator();
+ sparkSession.sparkContext().register(metricsAccumulator, "hop-transform-metrics");
+ converter.setMetricsAccumulator(metricsAccumulator);
+ sampleDataAccumulator = new SparkExecutionDataAccumulator();
+ sparkSession.sparkContext().register(sampleDataAccumulator, "hop-execution-sample-data");
+ converter.setSampleDataAccumulator(sampleDataAccumulator);
+ try {
+ converter.setExecutionSamplingContext(
+ getLogChannelId(), SparkTransformExecutionSampling.serializeDataSamplers(dataSamplers));
+ } catch (HopException e) {
+ logChannel.logError("Error serializing execution data samplers (non-fatal)", e);
+ converter.setExecutionSamplingContext(getLogChannelId(), "[]");
+ }
+ seedEngineMetricsComponents();
+
+ // Register at execution info location (if configured) and keep state updated
+ try {
+ registerPipelineExecutionInformation();
+ startExecutionInfoTimer();
+ } catch (HopException e) {
+ logChannel.logError("Error starting execution information tracking (non-fatal)", e);
+ }
+
+ fireExecutionStartedListeners();
+ ExtensionPointHandler.callExtensionPoint(
+ logChannel, this, HopExtensionPoint.PipelineStart.id, this);
+
+ // Build + materialize Dataset graph on a worker thread so waitUntilFinished can poll
+ sparkThread =
+ new Thread(
+ () -> {
+ ClassLoader prev = Thread.currentThread().getContextClassLoader();
+ try {
+ Thread.currentThread().setContextClassLoader(pluginCl);
+ resultDataset = converter.createDataset(sparkSession);
+ // Materialize: count is a safe action when no dedicated sink handler wrote data
+ long rows = resultDataset.count();
+ logChannel.logBasic("Spark pipeline finished, result row count=" + rows);
+ setErrors(0);
+ } catch (Throwable e) {
+ logChannel.logError("Error executing Spark pipeline", e);
+ setErrors(getErrors() + 1);
+ setStopped(true);
+ } finally {
+ Thread.currentThread().setContextClassLoader(prev);
+ setRunning(false);
+ setFinished(true);
+ executionEndDate = new Date();
+ if (getErrors() == 0) {
+ status = ComponentExecutionStatus.STATUS_FINISHED;
+ statusDescription = Pipeline.STRING_FINISHED;
+ } else {
+ status = ComponentExecutionStatus.STATUS_STOPPED;
+ statusDescription = Pipeline.STRING_STOPPED;
+ }
+ ExecutorUtil.cleanup(metricsRefreshTimer);
+ try {
+ populateEngineMetrics();
+ } catch (Exception emEx) {
+ logChannel.logError("Error populating final engine metrics", emEx);
+ }
+ try {
+ stopExecutionInfoTimer();
+ } catch (Exception eiEx) {
+ logChannel.logError("Error finalizing execution information (non-fatal)", eiEx);
+ }
+ try {
+ fireExecutionFinishedListeners();
+ } catch (HopException ex) {
+ logChannel.logError("Error firing finished listeners", ex);
+ }
+ }
+ },
+ "hop-spark-pipeline");
+ sparkThread.setContextClassLoader(pluginCl);
+ sparkThread.start();
+
+ // Live progress for the Hop Metrics grid (same cadence as Beam)
+ metricsRefreshTimer = new Timer("hop-spark-metrics-refresh", true);
+ metricsRefreshTimer.schedule(
+ new TimerTask() {
+ @Override
+ public void run() {
+ try {
+ if (isFinished() || isStopped()) {
+ ExecutorUtil.cleanup(metricsRefreshTimer);
+ return;
+ }
+ populateEngineMetrics();
+ } catch (Throwable e) {
+ if (logChannel != null) {
+ logChannel.logError("Error refreshing Spark engine metrics", e);
+ }
+ }
+ }
+ },
+ 0L,
+ 1000L);
+ } catch (Throwable e) {
+ setRunning(false);
+ setStopped(true);
+ setErrors(getErrors() + 1);
+ ExecutorUtil.cleanup(metricsRefreshTimer);
+ throw new HopException("Unexpected error starting Spark pipeline", e);
+ } finally {
+ Thread.currentThread().setContextClassLoader(previousCl);
+ }
+ }
+
+ /**
+ * Seed the metrics grid with one placeholder component per transform so the UI lists them before
+ * executor slices arrive.
+ */
+ private void seedEngineMetricsComponents() {
+ EngineMetrics em = new EngineMetrics();
+ em.setStartDate(executionStartDate);
+ if (pipelineMeta != null) {
+ for (TransformMeta transformMeta : pipelineMeta.getTransforms()) {
+ EngineComponent component = new EngineComponent(transformMeta.getName(), 0);
+ component.setRunning(true);
+ component.setStatus(ComponentExecutionStatus.STATUS_RUNNING);
+ component.setExecutionStartDate(executionStartDate);
+ assignComponentIdentity(component);
+ em.addComponent(component);
+ em.setComponentRunning(component, true);
+ em.setComponentStatus(component, ComponentExecutionStatus.STATUS_RUNNING.getDescription());
+ }
+ }
+ synchronized (this) {
+ engineMetrics = em;
+ }
+ }
+
+ /** Stable log-channel id per transform copy for execution info + GUI. */
+ private String componentLogChannelId(String transformName, int copyNr) {
+ String key = transformName + '\0' + copyNr;
+ return componentLogChannelIds.computeIfAbsent(
+ key,
+ k -> {
+ String parent = getLogChannelId();
+ if (StringUtils.isEmpty(parent)) {
+ parent = "spark";
+ }
+ return parent + "|" + transformName + "|" + copyNr;
+ });
+ }
+
+ private void assignComponentIdentity(EngineComponent component) {
+ if (component == null) {
+ return;
+ }
+ component.setLogChannelId(componentLogChannelId(component.getName(), component.getCopyNr()));
+ }
+
+ /**
+ * Build {@link EngineMetrics} from Spark transform metric slices (mapPartitions) plus seeded
+ * placeholders for transforms that only have native Dataset handlers.
+ */
+ protected synchronized void populateEngineMetrics() {
+ EngineMetrics em = new EngineMetrics();
+ em.setStartDate(getExecutionStartDate());
+ em.setEndDate(getExecutionEndDate());
+
+ Set transformsWithSlices = new HashSet<>();
+ Map slices =
+ metricsAccumulator != null ? metricsAccumulator.value() : Collections.emptyMap();
+
+ for (SparkTransformMetricSlice slice : slices.values()) {
+ transformsWithSlices.add(slice.getTransformName());
+ EngineComponent component = new EngineComponent(slice.getTransformName(), slice.getCopyNr());
+ component.setLinesRead(slice.getLinesRead());
+ component.setLinesWritten(slice.getLinesWritten());
+ component.setLinesInput(slice.getLinesInput());
+ component.setLinesOutput(slice.getLinesOutput());
+ component.setErrors(slice.getErrors());
+ applySliceTiming(component, slice);
+ if (StringUtils.isNotEmpty(slice.getHost())) {
+ component.setLogText("host=" + slice.getHost());
+ }
+
+ ComponentExecutionStatus componentStatus = resolveComponentStatus(slice);
+ component.setStatus(componentStatus);
+ component.setRunning(componentStatus == ComponentExecutionStatus.STATUS_RUNNING);
+ component.setStopped(
+ componentStatus == ComponentExecutionStatus.STATUS_STOPPED
+ || componentStatus == ComponentExecutionStatus.STATUS_HALTING);
+ assignComponentIdentity(component);
+
+ em.addComponent(component);
+ em.setComponentMetric(component, Pipeline.METRIC_READ, slice.getLinesRead());
+ em.setComponentMetric(component, Pipeline.METRIC_WRITTEN, slice.getLinesWritten());
+ em.setComponentMetric(component, Pipeline.METRIC_INPUT, slice.getLinesInput());
+ em.setComponentMetric(component, Pipeline.METRIC_OUTPUT, slice.getLinesOutput());
+ em.setComponentMetric(component, Pipeline.METRIC_ERROR, slice.getErrors());
+ em.setComponentStatus(component, componentStatus.getDescription());
+ em.setComponentRunning(component, component.isRunning());
+ long speedRows =
+ Math.max(
+ Math.max(slice.getLinesRead(), slice.getLinesWritten()),
+ Math.max(slice.getLinesInput(), slice.getLinesOutput()));
+ if (speedRows > 0) {
+ long durationMs = Math.max(1L, component.getExecutionDuration());
+ double speed = (speedRows * 1000.0d) / durationMs;
+ em.setComponentSpeed(component, String.format("%.1f", speed));
+ }
+ }
+
+ // Transforms with no metric slices yet (not scheduled / no accumulator data)
+ if (pipelineMeta != null) {
+ ComponentExecutionStatus pipelineStatus = status;
+ for (TransformMeta transformMeta : pipelineMeta.getTransforms()) {
+ if (transformsWithSlices.contains(transformMeta.getName())) {
+ continue;
+ }
+ EngineComponent component = new EngineComponent(transformMeta.getName(), 0);
+ component.setExecutionStartDate(getExecutionStartDate());
+ component.setExecutionEndDate(getExecutionEndDate());
+ component.setExecutionDuration(
+ calculateDuration(getExecutionStartDate(), getExecutionEndDate()));
+ component.setStatus(pipelineStatus);
+ component.setRunning(pipelineStatus == ComponentExecutionStatus.STATUS_RUNNING);
+ component.setStopped(
+ pipelineStatus == ComponentExecutionStatus.STATUS_STOPPED || isStopped());
+ assignComponentIdentity(component);
+ em.addComponent(component);
+ em.setComponentStatus(component, pipelineStatus.getDescription());
+ em.setComponentRunning(component, component.isRunning());
+ }
+ }
+
+ engineMetrics = em;
+ }
+
+ // --- Execution information location (Beam/Local pattern, driver-side) ---
+
+ /**
+ * Load and initialize the execution information location named on the pipeline run configuration
+ * (if any).
+ */
+ public void lookupExecutionInformationLocation() throws HopException {
+ if (pipelineRunConfiguration == null || metadataProvider == null) {
+ return;
+ }
+ String locationName = resolve(pipelineRunConfiguration.getExecutionInfoLocationName());
+ if (StringUtils.isEmpty(locationName)) {
+ return;
+ }
+ ExecutionInfoLocation location =
+ metadataProvider.getSerializer(ExecutionInfoLocation.class).load(locationName);
+ if (location == null) {
+ logChannel.logError(
+ "Execution information location '"
+ + locationName
+ + "' could not be found in the metadata");
+ return;
+ }
+ // Clone so nested engines do not share timer/rootFolder state on the same instance
+ executionInfoLocation = location.clone();
+ executionInfoClosed = false;
+ IExecutionInfoLocation iLocation = executionInfoLocation.getExecutionInfoLocation();
+ iLocation.initialize(this, metadataProvider);
+ logChannel.logBasic(
+ "Using execution information location '"
+ + locationName
+ + "' (logChannelId="
+ + getLogChannelId()
+ + ")");
+ }
+
+ /**
+ * Register this pipeline execution at the configured location (metadata, variables, parameters).
+ */
+ public void registerPipelineExecutionInformation() throws HopException {
+ if (executionInfoLocation == null) {
+ return;
+ }
+ executionInfoLocation
+ .getExecutionInfoLocation()
+ .registerExecution(ExecutionBuilder.fromExecutor(this).build());
+ }
+
+ /** Periodically push pipeline + transform state to the execution information location. */
+ public void startExecutionInfoTimer() {
+ if (executionInfoLocation == null) {
+ return;
+ }
+ long delay = Const.toLong(resolve(executionInfoLocation.getDataLoggingDelay()), 2000L);
+ long interval = Const.toLong(resolve(executionInfoLocation.getDataLoggingInterval()), 5000L);
+ final IExecutionInfoLocation iLocation = executionInfoLocation.getExecutionInfoLocation();
+
+ executionInfoTimer = new Timer("hop-spark-execution-info", true);
+ executionInfoTimer.schedule(
+ new TimerTask() {
+ @Override
+ public void run() {
+ try {
+ if (isFinished() || isStopped() || executionInfoClosed) {
+ return;
+ }
+ // Refresh metrics so component state includes latest Spark counters
+ populateEngineMetrics();
+ updatePipelineState(iLocation);
+ } catch (Exception e) {
+ if (logChannel != null) {
+ logChannel.logBasic(
+ "Warning: unable to register execution info at location "
+ + executionInfoLocation.getName()
+ + " (non-fatal): "
+ + e.getMessage());
+ }
+ }
+ }
+ },
+ delay,
+ interval);
+ }
+
+ protected void updatePipelineState(IExecutionInfoLocation iLocation) throws HopException {
+ // Register sample rows collected on executors before updating parent/transform state
+ registerSampleDataFromExecutors(iLocation);
+
+ ExecutionState executionState =
+ ExecutionStateBuilder.fromExecutor(SparkPipelineEngine.this, -1).build();
+ iLocation.updateExecutionState(executionState);
+
+ // Transform Execution + state nodes under the parent pipeline (Beam does the same from workers;
+ // we do it on the driver so caching locations keep hierarchy in one CacheEntry).
+ for (IEngineComponent component : getComponents()) {
+ registerTransformExecution(iLocation, component);
+ ExecutionState transformState =
+ ExecutionStateBuilder.fromTransform(SparkPipelineEngine.this, component).build();
+ iLocation.updateExecutionState(transformState);
+ }
+ }
+
+ /**
+ * Register a transform {@link Execution} child of this pipeline so the GUI can resolve parent →
+ * transform hierarchy (see {@code PipelineExecutionViewer.loadSelectedTransformData}).
+ */
+ protected void registerTransformExecution(
+ IExecutionInfoLocation iLocation, IEngineComponent component) throws HopException {
+ if (iLocation == null || component == null) {
+ return;
+ }
+ String id = component.getLogChannelId();
+ if (StringUtils.isEmpty(id)) {
+ id = componentLogChannelId(component.getName(), component.getCopyNr());
+ }
+ Execution execution =
+ ExecutionBuilder.of()
+ .withId(id)
+ .withParentId(getLogChannelId())
+ .withName(component.getName())
+ .withCopyNr(Integer.toString(component.getCopyNr()))
+ .withExecutorType(ExecutionType.Transform)
+ .withExecutionStartDate(
+ component.getExecutionStartDate() != null
+ ? component.getExecutionStartDate()
+ : getExecutionStartDate())
+ .build();
+ iLocation.registerExecution(execution);
+ }
+
+ /**
+ * Pull JSON {@link ExecutionData} samples from the Spark accumulator and register them on the
+ * driver-side execution info location (where the parent pipeline CacheEntry lives).
+ */
+ protected void registerSampleDataFromExecutors(IExecutionInfoLocation iLocation) {
+ if (iLocation == null || sampleDataAccumulator == null || sampleDataAccumulator.isZero()) {
+ return;
+ }
+ try {
+ Map samples = sampleDataAccumulator.value();
+ if (samples == null || samples.isEmpty()) {
+ return;
+ }
+ var mapper = HopJson.newMapper();
+ int registered = 0;
+ for (Map.Entry entry : samples.entrySet()) {
+ try {
+ ExecutionData data = mapper.readValue(entry.getValue(), ExecutionData.class);
+ if (data != null) {
+ // Ensure parent id points at this pipeline execution
+ if (StringUtils.isEmpty(data.getParentId())) {
+ data.setParentId(getLogChannelId());
+ }
+ // Ensure transform Execution child exists (ownerId is the child log channel id)
+ registerTransformExecutionFromSample(iLocation, data);
+ iLocation.registerData(data);
+ registered++;
+ }
+ } catch (Exception e) {
+ if (logChannel != null) {
+ logChannel.logError(
+ "Error registering executor sample data for owner '"
+ + entry.getKey()
+ + "' (non-fatal)",
+ e);
+ }
+ }
+ }
+ if (registered > 0 && logChannel != null) {
+ logChannel.logBasic(
+ "Registered " + registered + " transform sample set(s) from Spark executors");
+ }
+ } catch (Exception e) {
+ if (logChannel != null) {
+ logChannel.logError("Error reading sample data accumulator (non-fatal)", e);
+ }
+ }
+ }
+
+ /**
+ * Beam {@code TransformBaseFn.registerExecutingTransform}: child Execution id = sample owner id,
+ * parent = pipeline log channel id.
+ */
+ private void registerTransformExecutionFromSample(
+ IExecutionInfoLocation iLocation, ExecutionData data) throws HopException {
+ if (StringUtils.isEmpty(data.getOwnerId())) {
+ return;
+ }
+ String transformName = null;
+ String copyNr = "0";
+ if (data.getSetMetaData() != null) {
+ for (ExecutionDataSetMeta setMeta : data.getSetMetaData().values()) {
+ if (setMeta != null && StringUtils.isNotEmpty(setMeta.getName())) {
+ transformName = setMeta.getName();
+ if (StringUtils.isNotEmpty(setMeta.getCopyNr())) {
+ copyNr = setMeta.getCopyNr();
+ }
+ break;
+ }
+ }
+ }
+ if (StringUtils.isEmpty(transformName)) {
+ // ownerId format: parent|transformName|copyNr
+ String ownerId = data.getOwnerId();
+ int first = ownerId.indexOf('|');
+ int last = ownerId.lastIndexOf('|');
+ if (first > 0 && last > first) {
+ transformName = ownerId.substring(first + 1, last);
+ copyNr = ownerId.substring(last + 1);
+ } else {
+ transformName = ownerId;
+ }
+ }
+ Execution execution =
+ ExecutionBuilder.of()
+ .withId(data.getOwnerId())
+ .withParentId(
+ StringUtils.isNotEmpty(data.getParentId()) ? data.getParentId() : getLogChannelId())
+ .withName(transformName)
+ .withCopyNr(copyNr)
+ .withExecutorType(ExecutionType.Transform)
+ .withExecutionStartDate(getExecutionStartDate())
+ .build();
+ iLocation.registerExecution(execution);
+ }
+
+ /** Final pipeline/transform state update and close the location. Safe to call multiple times. */
+ public void stopExecutionInfoTimer() {
+ if (executionInfoLocation == null || executionInfoClosed) {
+ ExecutorUtil.cleanup(executionInfoTimer);
+ return;
+ }
+ try {
+ ExecutorUtil.cleanup(executionInfoTimer);
+ executionInfoTimer = null;
+
+ try {
+ populateEngineMetrics();
+ } catch (Exception e) {
+ // ignore — still try to write state
+ }
+
+ IExecutionInfoLocation iLocation = executionInfoLocation.getExecutionInfoLocation();
+ // Final sample flush from executors (after jobs complete, accumulator is fully merged)
+ registerSampleDataFromExecutors(iLocation);
+
+ ExecutionState executionState =
+ ExecutionStateBuilder.fromExecutor(SparkPipelineEngine.this, -1).build();
+ iLocation.updateExecutionState(executionState);
+
+ for (IEngineComponent component : getComponents()) {
+ registerTransformExecution(iLocation, component);
+ ExecutionState transformState =
+ ExecutionStateBuilder.fromTransform(SparkPipelineEngine.this, component).build();
+ iLocation.updateExecutionState(transformState);
+ }
+
+ iLocation.close();
+ } catch (Throwable e) {
+ if (logChannel != null) {
+ logChannel.logError(
+ "Error writing final execution state to location (non-fatal): "
+ + (executionInfoLocation != null ? executionInfoLocation.getName() : "?"),
+ e);
+ }
+ } finally {
+ executionInfoClosed = true;
+ }
+ }
+
+ /**
+ * Map partition wall-clock times onto {@link EngineComponent} fields the GUI Metrics grid uses
+ * for Duration ({@code firstRowReadDate} / {@code lastRowWrittenDate}).
+ */
+ private void applySliceTiming(EngineComponent component, SparkTransformMetricSlice slice) {
+ long startMs = slice.getStartTimeMs();
+ long endMs = slice.getEndTimeMs();
+ if (startMs > 0) {
+ Date start = new Date(startMs);
+ component.setInitStartDate(start);
+ component.setExecutionStartDate(start);
+ component.setFirstRowReadDate(start);
+ } else if (getExecutionStartDate() != null) {
+ // Fallback for slices without timing
+ component.setExecutionStartDate(getExecutionStartDate());
+ component.setFirstRowReadDate(getExecutionStartDate());
+ }
+
+ if (slice.isFinished() && endMs > 0) {
+ Date end = new Date(endMs);
+ component.setExecutionEndDate(end);
+ component.setLastRowWrittenDate(end);
+ } else if (slice.isFinished() && getExecutionEndDate() != null) {
+ component.setExecutionEndDate(getExecutionEndDate());
+ component.setLastRowWrittenDate(getExecutionEndDate());
+ }
+ // While running, leave lastRowWrittenDate null so the grid ticks duration live
+
+ long durationMs = slice.durationMs();
+ if (durationMs <= 0 && startMs > 0) {
+ durationMs =
+ calculateDuration(component.getFirstRowReadDate(), component.getLastRowWrittenDate());
+ }
+ if (durationMs <= 0) {
+ durationMs = calculateDuration(getExecutionStartDate(), getExecutionEndDate());
+ }
+ component.setExecutionDuration(durationMs);
+ }
+
+ private ComponentExecutionStatus resolveComponentStatus(SparkTransformMetricSlice slice) {
+ if (isStopped() || getErrors() > 0) {
+ if (slice.isFinished() || slice.getErrors() > 0) {
+ return ComponentExecutionStatus.STATUS_STOPPED;
+ }
+ }
+ if (slice.getErrors() > 0) {
+ return ComponentExecutionStatus.STATUS_STOPPED;
+ }
+ if (slice.isFinished() || isFinished()) {
+ return ComponentExecutionStatus.STATUS_FINISHED;
+ }
+ if (slice.isRunning() || isRunning()) {
+ return ComponentExecutionStatus.STATUS_RUNNING;
+ }
+ return status != null ? status : ComponentExecutionStatus.STATUS_EMPTY;
+ }
+
+ protected long calculateDuration(Date startTime, Date stopTime) {
+ if (startTime != null && stopTime == null) {
+ return Calendar.getInstance().getTimeInMillis() - startTime.getTime();
+ }
+ if (startTime != null) {
+ return stopTime.getTime() - startTime.getTime();
+ }
+ return 0L;
+ }
+
+ protected SparkSession createSparkSession() throws HopException {
+ ClassLoader pluginCl = getClass().getClassLoader();
+ ClassLoader previousCl = Thread.currentThread().getContextClassLoader();
+ try {
+ Thread.currentThread().setContextClassLoader(pluginCl);
+ SparkSession session = buildSparkSession();
+ // Ship project package zip to every executor (SparkFiles) when HOP_SPARK_PROJECT_PACKAGE set
+ distributeProjectPackageIfConfigured(session);
+ return session;
+ } catch (IllegalStateException e) {
+ // Surface a actionable message when classic SparkSession cannot be loaded
+ throw new IllegalStateException(
+ e.getMessage()
+ + " Ensure the native Spark engine plugin ships spark-sql, hadoop-client-api/runtime,"
+ + " and log4j under plugins/engines/spark/lib (Spark 4 loads"
+ + " org.apache.spark.sql.classic.SparkSession reflectively).",
+ e);
+ } finally {
+ Thread.currentThread().setContextClassLoader(previousCl);
+ }
+ }
+
+ /**
+ * Native Spark uses RDDs, accumulators, and {@code SparkContext} APIs. Databricks Spark Connect
+ * sessions (cluster access mode Standard / shared, or Serverless) do not expose
+ * a real SparkContext. Fail early with an actionable message.
+ */
+ static void requireClassicSparkContext(SparkSession session) throws HopException {
+ if (session == null) {
+ throw new HopException("SparkSession is null");
+ }
+ String sessionClass = session.getClass().getName();
+ try {
+ // Touches the real context; Connect client throws SparkUnsupportedOperationException.
+ session.sparkContext().appName();
+ } catch (Throwable t) {
+ String msg = t.getMessage() != null ? t.getMessage() : t.getClass().getSimpleName();
+ if (isSparkConnectContextError(sessionClass, msg, t)) {
+ throw new HopException(
+ "Apache Hop Native Spark requires a classic SparkContext (RDD / accumulator APIs). "
+ + "This session is Spark Connect ("
+ + sessionClass
+ + ") which does not expose SparkContext on Databricks "
+ + "Standard access mode or Serverless compute. "
+ + "Use a classic job cluster (Deploy & run with cluster field = new_cluster) or an "
+ + "all-purpose cluster with access mode Dedicated (single user), not Standard/shared "
+ + "or Serverless. Underlying error: "
+ + msg,
+ t);
+ }
+ throw new HopException("Unable to access SparkContext from SparkSession: " + msg, t);
+ }
+ }
+
+ static boolean isSparkConnectContextError(String sessionClass, String message, Throwable t) {
+ if (sessionClass != null && sessionClass.contains("connect")) {
+ return true;
+ }
+ if (message != null) {
+ String m = message.toLowerCase();
+ if (m.contains("spark connect")
+ || m.contains("unsupported_connect")
+ || m.contains("session_spark_context")
+ || m.contains("standard access mode")
+ || m.contains("serverless")) {
+ return true;
+ }
+ }
+ String tn = t != null ? t.getClass().getName() : "";
+ return tn.contains("SparkUnsupportedOperationException") || tn.contains("connect");
+ }
+
+ /**
+ * When a Native Spark project package is configured, make it available on executors (shared path
+ * or {@code SparkContext.addFile}) and re-materialize {@code PROJECT_HOME} on the driver.
+ */
+ private void distributeProjectPackageIfConfigured(SparkSession session) throws HopException {
+ String packageUri = getVariable(SparkProjectPackage.VAR_PACKAGE_URI);
+ if (StringUtils.isEmpty(packageUri)) {
+ return;
+ }
+ SparkProjectPackage.distributeToCluster(session, this);
+ SparkProjectPackage.ensureMaterializedOnWorker(this);
+ if (logChannel != null) {
+ String sparkFile = getVariable(SparkProjectPackage.VAR_PACKAGE_SPARK_FILE);
+ String source = getVariable(SparkProjectPackage.VAR_PACKAGE_URI);
+ if (StringUtils.isNotEmpty(sparkFile)) {
+ logChannel.logBasic(
+ "Distributed Spark project package for executors (SparkFiles="
+ + sparkFile
+ + ", source="
+ + source
+ + "); PROJECT_HOME="
+ + getVariable("PROJECT_HOME"));
+ } else {
+ logChannel.logBasic(
+ "Using Spark project package URI only (no SparkFiles basename): "
+ + source
+ + "; PROJECT_HOME="
+ + getVariable("PROJECT_HOME")
+ + " — workers must open this path directly");
+ }
+ }
+ }
+
+ private SparkSession buildSparkSession() throws HopException {
+ LakeSessionPlan lakePlan = LakeSessionPlan.from(pipelineMeta, metadataProvider, this);
+ if (!lakePlan.isEmpty()) {
+ lakePlan.verifyClasspath(getClass().getClassLoader());
+ logChannel.logBasic(
+ "Lakehouse session plan formats="
+ + lakePlan.getFormatsNeeded()
+ + " catalogs="
+ + lakePlan.getCatalogsByMetaName().keySet());
+ }
+
+ // Nested under another Native Spark pipeline: always reuse the parent's session when present.
+ if (parentPipeline instanceof SparkPipelineEngine parentSpark
+ && parentSpark.sparkSession != null) {
+ sessionOwnedByThisEngine = false;
+ SparkSession session = parentSpark.sparkSession;
+ logChannel.logBasic(
+ "Nested Native Spark pipeline reusing parent SparkSession (parent='"
+ + (parentSpark.pipelineMeta != null ? parentSpark.pipelineMeta.getName() : "?")
+ + "', version="
+ + session.version()
+ + ")");
+ if (!lakePlan.isEmpty()) {
+ lakePlan.verifyActiveSession(session, logChannel, this);
+ }
+ return session;
+ }
+
+ // spark-submit / existing driver: reuse the active session if present
+ Option active = SparkSession.getActiveSession();
+ if (active.isDefined()) {
+ sessionOwnedByThisEngine = false;
+ SparkSession session = active.get();
+ logChannel.logBasic(
+ "Reusing active SparkSession (version="
+ + session.version()
+ + ", spark-submit or existing driver context)");
+ if (!lakePlan.isEmpty()) {
+ lakePlan.verifyActiveSession(session, logChannel, this);
+ }
+ return session;
+ }
+ Option def = SparkSession.getDefaultSession();
+ if (def.isDefined()) {
+ sessionOwnedByThisEngine = false;
+ SparkSession session = def.get();
+ logChannel.logBasic("Reusing default SparkSession (version=" + session.version() + ")");
+ if (!lakePlan.isEmpty()) {
+ lakePlan.verifyActiveSession(session, logChannel, this);
+ }
+ return session;
+ }
+
+ boolean underSubmit = isRunningUnderSparkSubmit();
+ String master = resolve(sparkEngineRunConfiguration.getSparkMaster());
+ if (StringUtils.isEmpty(master)) {
+ // Prefer master from spark-submit / SparkConf when run config leaves it blank
+ String confMaster = System.getProperty("spark.master");
+ if (StringUtils.isEmpty(confMaster)) {
+ try {
+ SparkConf conf = new SparkConf(true);
+ if (conf.contains("spark.master")) {
+ confMaster = conf.get("spark.master");
+ }
+ } catch (Exception e) {
+ // ignore — fall back to local
+ }
+ }
+ master = StringUtils.isNotEmpty(confMaster) ? confMaster : "local[*]";
+ }
+ String appName = resolve(sparkEngineRunConfiguration.getSparkAppName());
+ if (StringUtils.isEmpty(appName)) {
+ appName = "Apache Hop - " + pipelineMeta.getName();
+ }
+
+ SparkSession.Builder builder = SparkSession.builder().appName(appName);
+
+ // Under spark-submit, master is already in the conf; re-setting can still work but prefer
+ // not fighting the launcher when blank was intended to inherit.
+ if (!underSubmit
+ || StringUtils.isNotEmpty(resolve(sparkEngineRunConfiguration.getSparkMaster()))) {
+ builder = builder.master(master);
+ } else if (StringUtils.isNotEmpty(master)) {
+ builder = builder.master(master);
+ }
+
+ applySparkConfig(builder, "spark.driver.memory", sparkEngineRunConfiguration.getDriverMemory());
+ applySparkConfig(
+ builder, "spark.executor.memory", sparkEngineRunConfiguration.getExecutorMemory());
+ applySparkConfig(
+ builder, "spark.executor.cores", sparkEngineRunConfiguration.getExecutorCores());
+
+ // Do not set spark.jars under spark-submit — the application jar is already the fat jar.
+ // Setting it again can confuse class loading.
+ if (!underSubmit) {
+ String fatJar = resolve(sparkEngineRunConfiguration.getFatJar());
+ if (StringUtils.isNotEmpty(fatJar)) {
+ builder.config("spark.jars", fatJar);
+ }
+ }
+
+ String extra = sparkEngineRunConfiguration.getSparkConfigs();
+ if (StringUtils.isNotEmpty(extra)) {
+ for (String line : extra.split("\\r?\\n")) {
+ String trimmed = line.trim();
+ if (trimmed.isEmpty() || trimmed.startsWith("#")) {
+ continue;
+ }
+ int eq = trimmed.indexOf('=');
+ if (eq > 0) {
+ builder.config(
+ resolve(trimmed.substring(0, eq).trim()), resolve(trimmed.substring(eq + 1).trim()));
+ }
+ }
+ }
+
+ // Lakehouse: Delta/Iceberg extensions, hop_iceberg PATH catalog, SparkCatalog metadata.
+ // Applied after run-config sparkConfigs so lake defaults fill gaps; explicit run-config
+ // keys already set above win if the user overrode them (except catalog apply overwrites).
+ if (!lakePlan.isEmpty()) {
+ lakePlan.applyToBuilder(builder, this);
+ }
+
+ // Defaults for local/dev only when not already set by submit or run config extras
+ if (!underSubmit && System.getProperty("spark.ui.enabled") == null) {
+ builder.config("spark.ui.enabled", "false");
+ }
+ if (!underSubmit && System.getProperty("spark.sql.shuffle.partitions") == null) {
+ builder.config("spark.sql.shuffle.partitions", "4");
+ }
+
+ SparkSession session = builder.getOrCreate();
+ // getOrCreate may still return a shared session under spark-submit; only claim ownership
+ // when we are not nested and no prior active session existed (we already returned above).
+ sessionOwnedByThisEngine = true;
+ logChannel.logBasic(
+ "Created SparkSession version=" + session.version() + " master=" + master + " (owned)");
+ return session;
+ }
+
+ /**
+ * Heuristic: spark-submit sets deploy mode / app id properties and loads SPARK_ENV / conf before
+ * main().
+ */
+ private static boolean isRunningUnderSparkSubmit() {
+ if (System.getenv("SPARK_ENV_LOADED") != null) {
+ return true;
+ }
+ if (System.getProperty("spark.submit.deployMode") != null) {
+ return true;
+ }
+ if (System.getProperty("spark.app.id") != null) {
+ return true;
+ }
+ // spark-submit always sets spark.master before invoking main
+ String master = System.getProperty("spark.master");
+ return StringUtils.isNotEmpty(master) && !master.startsWith("local");
+ }
+
+ private void applySparkConfig(SparkSession.Builder builder, String key, String value) {
+ String resolved = resolve(value);
+ if (StringUtils.isNotEmpty(resolved)) {
+ builder.config(key, resolved);
+ }
+ }
+
+ @Override
+ public void execute() throws HopException {
+ prepareExecution();
+ startThreads();
+ }
+
+ @Override
+ public void waitUntilFinished() {
+ while ((running || paused || readyToStart) && !(stopped || finished)) {
+ try {
+ Thread.sleep(100);
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ break;
+ }
+ }
+ if (sparkThread != null) {
+ try {
+ sparkThread.join();
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ }
+ }
+ }
+
+ @Override
+ public void stopAll() {
+ setStopped(true);
+ setRunning(false);
+ ExecutorUtil.cleanup(metricsRefreshTimer);
+ // Nested / reused sessions share the parent's SparkContext — never cancelAllJobs there.
+ if (sparkSession != null && sessionOwnedByThisEngine) {
+ try {
+ sparkSession.sparkContext().cancelAllJobs();
+ } catch (Exception e) {
+ if (logChannel != null) {
+ logChannel.logError("Error cancelling Spark jobs", e);
+ }
+ }
+ } else if (sparkSession != null && logChannel != null) {
+ logChannel.logBasic(
+ "Skipping Spark job cancel for nested/reused session (pipeline='"
+ + (pipelineMeta != null ? pipelineMeta.getName() : "?")
+ + "')");
+ }
+ try {
+ populateEngineMetrics();
+ } catch (Exception e) {
+ if (logChannel != null) {
+ logChannel.logError("Error populating metrics after stop", e);
+ }
+ }
+ stopExecutionInfoTimer();
+ try {
+ fireExecutionStoppedListeners();
+ } catch (HopException e) {
+ if (logChannel != null) {
+ logChannel.logError("Error firing stopped listeners", e);
+ }
+ }
+ }
+
+ @Override
+ public void cleanup() {
+ ExecutorUtil.cleanup(metricsRefreshTimer);
+ ExecutorUtil.cleanup(executionInfoTimer);
+ if (sparkSession != null) {
+ try {
+ // Only stop sessions this engine created. Nested Pipeline Executor children and
+ // spark-submit drivers must leave the shared SparkSession alone.
+ if (sessionOwnedByThisEngine) {
+ String master =
+ sparkEngineRunConfiguration != null
+ ? resolve(sparkEngineRunConfiguration.getSparkMaster())
+ : "";
+ if (master != null && master.startsWith("local")) {
+ logChannel.logBasic(
+ "Stopping owned local SparkSession for pipeline '"
+ + (pipelineMeta != null ? pipelineMeta.getName() : "?")
+ + "'");
+ sparkSession.stop();
+ }
+ } else if (logChannel != null) {
+ logChannel.logDetailed(
+ "Leaving shared SparkSession running after nested/reused pipeline '"
+ + (pipelineMeta != null ? pipelineMeta.getName() : "?")
+ + "'");
+ }
+ } catch (Exception e) {
+ if (logChannel != null) {
+ logChannel.logError("Error stopping SparkSession", e);
+ }
+ } finally {
+ sparkSession = null;
+ }
+ }
+ }
+
+ /** Whether this engine owns (created) the SparkSession and may stop it. */
+ boolean isSessionOwnedByThisEngine() {
+ return sessionOwnedByThisEngine;
+ }
+
+ @Override
+ public EngineCompatibility supports(IPlugin transformPlugin) {
+ if (transformPlugin == null) {
+ return EngineCompatibility.unknown();
+ }
+ String[] ids = transformPlugin.getIds();
+ if (ids != null) {
+ for (String id : ids) {
+ String ban = HopPipelineMetaToSparkConverter.HARD_BANNED_PLUGIN_IDS.get(id);
+ if (ban != null) {
+ return EngineCompatibility.unsupported(ban);
+ }
+ if (HopPipelineMetaToSparkConverter.EXPLICIT_HANDLER_PLUGIN_IDS.contains(id)) {
+ return EngineCompatibility.supported();
+ }
+ }
+ }
+ // No opinion for ordinary transforms — generic mapPartitions may still work
+ return EngineCompatibility.unknown();
+ }
+
+ @Override
+ public EngineMetrics getEngineMetrics() {
+ return engineMetrics;
+ }
+
+ @Override
+ public EngineMetrics getEngineMetrics(String componentName, int copyNr) {
+ EngineMetrics em = new EngineMetrics();
+ em.setStartDate(engineMetrics.getStartDate());
+ em.setEndDate(engineMetrics.getEndDate());
+ for (IEngineComponent component : engineMetrics.getComponents()) {
+ if (componentName != null && !component.getName().equalsIgnoreCase(componentName)) {
+ continue;
+ }
+ if (copyNr >= 0 && component.getCopyNr() != copyNr) {
+ continue;
+ }
+ em.addComponent(component);
+ for (IEngineMetric metric : engineMetrics.getMetricsList()) {
+ Long value = engineMetrics.getComponentMetric(component, metric);
+ if (value != null) {
+ em.setComponentMetric(component, metric, value);
+ }
+ }
+ Boolean running = engineMetrics.getComponentRunningMap().get(component);
+ if (running != null) {
+ em.setComponentRunning(component, running);
+ }
+ String componentStatus = engineMetrics.getComponentStatusMap().get(component);
+ if (componentStatus != null) {
+ em.setComponentStatus(component, componentStatus);
+ }
+ String speed = engineMetrics.getComponentSpeedMap().get(component);
+ if (speed != null) {
+ em.setComponentSpeed(component, speed);
+ }
+ }
+ return em;
+ }
+
+ @Override
+ public Result getResult() {
+ Result result = new Result();
+ result.setNrErrors(errors);
+ result.setResult(errors == 0);
+ result.setStopped(isStopped());
+ result.setLogChannelId(getLogChannelId());
+ // Roll up lines from engine metrics when available
+ long linesRead = 0;
+ long linesWritten = 0;
+ long linesInput = 0;
+ long linesOutput = 0;
+ for (IEngineComponent component : engineMetrics.getComponents()) {
+ linesRead += nullToZero(engineMetrics.getComponentMetric(component, Pipeline.METRIC_READ));
+ linesWritten +=
+ nullToZero(engineMetrics.getComponentMetric(component, Pipeline.METRIC_WRITTEN));
+ linesInput += nullToZero(engineMetrics.getComponentMetric(component, Pipeline.METRIC_INPUT));
+ linesOutput +=
+ nullToZero(engineMetrics.getComponentMetric(component, Pipeline.METRIC_OUTPUT));
+ }
+ result.setNrLinesRead(linesRead);
+ result.setNrLinesWritten(linesWritten);
+ result.setNrLinesInput(linesInput);
+ result.setNrLinesOutput(linesOutput);
+ return result;
+ }
+
+ private static long nullToZero(Long value) {
+ return value == null ? 0L : value;
+ }
+
+ @Override
+ public void pauseExecution() {
+ // Not supported
+ }
+
+ @Override
+ public void resumeExecution() {
+ // Not supported
+ }
+
+ @Override
+ public boolean hasHaltedComponents() {
+ return hasHaltedComponents;
+ }
+
+ @Override
+ public String getComponentLogText(String componentName, int copyNr) {
+ return "";
+ }
+
+ @Override
+ public List getComponents() {
+ return engineMetrics.getComponents();
+ }
+
+ @Override
+ public List getComponentCopies(String name) {
+ List copies = new ArrayList<>();
+ for (IEngineComponent component : engineMetrics.getComponents()) {
+ if (component.getName().equalsIgnoreCase(name)) {
+ copies.add(component);
+ }
+ }
+ return copies;
+ }
+
+ @Override
+ public IEngineComponent findComponent(String name, int copyNr) {
+ for (IEngineComponent component : engineMetrics.getComponents()) {
+ if (component.getName().equalsIgnoreCase(name) && component.getCopyNr() == copyNr) {
+ return component;
+ }
+ }
+ return null;
+ }
+
+ @Override
+ public void retrieveComponentOutput(
+ IVariables variables,
+ String componentName,
+ int copyNr,
+ int nrRows,
+ IPipelineComponentRowsReceived rowsReceived)
+ throws HopException {
+ throw new HopException(
+ "Retrieving component output is not supported by the native Spark pipeline engine");
+ }
+
+ @Override
+ public boolean isSafeModeEnabled() {
+ return false;
+ }
+
+ @Override
+ public IRowSet findRowSet(
+ String fromTransformName,
+ int fromTransformCopy,
+ String toTransformName,
+ int toTransformCopy) {
+ return null;
+ }
+
+ @Override
+ public void pipelineCompleted() throws HopException {
+ stopExecutionInfoTimer();
+ cleanup();
+ }
+
+ // --- listeners ---
+
+ @Override
+ public void addExecutionStartedListener(
+ IExecutionStartedListener> listener) {
+ executionStartedListeners.add(listener);
+ }
+
+ @Override
+ public void removeExecutionStartedListener(
+ IExecutionStartedListener> listener) {
+ executionStartedListeners.remove(listener);
+ }
+
+ @Override
+ public void firePipelineExecutionStartedListeners() throws HopException {
+ fireExecutionStartedListeners();
+ }
+
+ @Override
+ public void fireExecutionStartedListeners() throws HopException {
+ synchronized (executionStartedListeners) {
+ for (IExecutionStartedListener> listener :
+ executionStartedListeners) {
+ listener.started(this);
+ }
+ }
+ }
+
+ @Override
+ public void addExecutionFinishedListener(
+ IExecutionFinishedListener> listener) {
+ executionFinishedListeners.add(listener);
+ }
+
+ @Override
+ public void removeExecutionFinishedListener(
+ IExecutionFinishedListener> listener) {
+ executionFinishedListeners.remove(listener);
+ }
+
+ @Override
+ public void firePipelineExecutionFinishedListeners() throws HopException {
+ fireExecutionFinishedListeners();
+ }
+
+ @Override
+ public void fireExecutionFinishedListeners() throws HopException {
+ synchronized (executionFinishedListeners) {
+ for (IExecutionFinishedListener> listener :
+ executionFinishedListeners) {
+ listener.finished(this);
+ }
+ }
+ }
+
+ @Override
+ public void addExecutionStoppedListener(
+ IExecutionStoppedListener> listener) {
+ executionStoppedListeners.add(listener);
+ }
+
+ @Override
+ public void removeExecutionStoppedListener(
+ IExecutionStoppedListener> listener) {
+ executionStoppedListeners.remove(listener);
+ }
+
+ @Override
+ public void firePipelineExecutionStoppedListeners() throws HopException {
+ fireExecutionStoppedListeners();
+ }
+
+ @Override
+ public void fireExecutionStoppedListeners() throws HopException {
+ synchronized (executionStoppedListeners) {
+ for (IExecutionStoppedListener> listener :
+ executionStoppedListeners) {
+ listener.stopped(this);
+ }
+ }
+ }
+
+ @Override
+ public >
+ void addExecutionDataSampler(Sampler sampler) {
+ dataSamplers.add(sampler);
+ }
+
+ // --- logging object ---
+
+ @Override
+ public String getObjectName() {
+ return pipelineMeta != null ? pipelineMeta.getName() : SparkConst.PLUGIN_NAME;
+ }
+
+ @Override
+ public String getFilename() {
+ return pipelineMeta != null ? pipelineMeta.getFilename() : null;
+ }
+
+ @Override
+ public LoggingObjectType getObjectType() {
+ return LoggingObjectType.PIPELINE;
+ }
+
+ @Override
+ public String getObjectCopy() {
+ return null;
+ }
+
+ @Override
+ public Date getRegistrationDate() {
+ return null;
+ }
+
+ @Override
+ public boolean isGatheringMetrics() {
+ return logChannel != null && logChannel.isGatheringMetrics();
+ }
+
+ @Override
+ public void setGatheringMetrics(boolean gatheringMetrics) {
+ if (logChannel != null) {
+ logChannel.setGatheringMetrics(gatheringMetrics);
+ }
+ }
+
+ @Override
+ public void setForcingSeparateLogging(boolean forcingSeparateLogging) {
+ if (logChannel != null) {
+ logChannel.setForcingSeparateLogging(forcingSeparateLogging);
+ }
+ }
+
+ @Override
+ public boolean isForcingSeparateLogging() {
+ return logChannel != null && logChannel.isForcingSeparateLogging();
+ }
+
+ @Override
+ public String getLogChannelId() {
+ return logChannel != null ? logChannel.getLogChannelId() : null;
+ }
+
+ @Override
+ public ILogChannel getLogChannel() {
+ return logChannel;
+ }
+
+ @Override
+ public void setLogChannel(ILogChannel log) {
+ this.logChannel = log;
+ }
+
+ @Override
+ public void setParent(ILoggingObject parent) {
+ this.parent = parent;
+ }
+
+ @Override
+ public ILoggingObject getParent() {
+ return parent;
+ }
+
+ // --- parameters ---
+
+ @Override
+ public void addParameterDefinition(String key, String defValue, String description)
+ throws DuplicateParamException {
+ namedParams.addParameterDefinition(key, defValue, description);
+ }
+
+ @Override
+ public String getParameterDescription(String key) throws UnknownParamException {
+ return namedParams.getParameterDescription(key);
+ }
+
+ @Override
+ public String getParameterDefault(String key) throws UnknownParamException {
+ return namedParams.getParameterDefault(key);
+ }
+
+ @Override
+ public String getParameterValue(String key) throws UnknownParamException {
+ return namedParams.getParameterValue(key);
+ }
+
+ @Override
+ public String[] listParameters() {
+ return namedParams.listParameters();
+ }
+
+ @Override
+ public void setParameterValue(String key, String value) throws UnknownParamException {
+ namedParams.setParameterValue(key, value);
+ }
+
+ @Override
+ public void removeAllParameters() {
+ namedParams.removeAllParameters();
+ }
+
+ @Override
+ public void clearParameterValues() {
+ namedParams.clearParameterValues();
+ }
+
+ @Override
+ public void copyParametersFromDefinitions(INamedParameterDefinitions definitions) {
+ namedParams.copyParametersFromDefinitions(definitions);
+ }
+
+ @Override
+ public void activateParameters(IVariables variables) {
+ namedParams.activateParameters(variables);
+ }
+
+ // --- getters / setters ---
+
+ @Override
+ public PipelineMeta getPipelineMeta() {
+ return pipelineMeta;
+ }
+
+ @Override
+ public void setPipelineMeta(PipelineMeta pipelineMeta) {
+ this.pipelineMeta = pipelineMeta;
+ }
+
+ @Override
+ public String getPluginId() {
+ return pluginId;
+ }
+
+ @Override
+ public void setPluginId(String pluginId) {
+ this.pluginId = pluginId;
+ }
+
+ @Override
+ public void setPipelineRunConfiguration(PipelineRunConfiguration pipelineRunConfiguration) {
+ this.pipelineRunConfiguration = pipelineRunConfiguration;
+ }
+
+ @Override
+ public PipelineRunConfiguration getPipelineRunConfiguration() {
+ return pipelineRunConfiguration;
+ }
+
+ @Override
+ public PipelineEngineCapabilities getEngineCapabilities() {
+ return engineCapabilities;
+ }
+
+ @Override
+ public boolean isPreparing() {
+ return preparing;
+ }
+
+ @Override
+ public boolean isReadyToStart() {
+ return readyToStart;
+ }
+
+ @Override
+ public boolean isRunning() {
+ return running;
+ }
+
+ @Override
+ public boolean isFinished() {
+ return finished;
+ }
+
+ @Override
+ public boolean isStopped() {
+ return stopped;
+ }
+
+ @Override
+ public boolean isPaused() {
+ return paused;
+ }
+
+ @Override
+ public int getErrors() {
+ return errors;
+ }
+
+ @Override
+ public void setMetadataProvider(IHopMetadataProvider metadataProvider) {
+ this.metadataProvider = metadataProvider;
+ }
+
+ @Override
+ public IHopMetadataProvider getMetadataProvider() {
+ return metadataProvider;
+ }
+
+ @Override
+ public void setLogLevel(LogLevel logLevel) {
+ this.logLevel = logLevel;
+ if (logChannel != null) {
+ logChannel.setLogLevel(logLevel);
+ }
+ }
+
+ @Override
+ public LogLevel getLogLevel() {
+ return logLevel;
+ }
+
+ @Override
+ public void setPreview(boolean preview) {
+ this.preview = preview;
+ }
+
+ @Override
+ public boolean isPreview() {
+ return preview;
+ }
+
+ @Override
+ public void setPreviousResult(Result previousResult) {
+ this.previousResult = previousResult;
+ }
+
+ @Override
+ public Result getPreviousResult() {
+ return previousResult;
+ }
+
+ @Override
+ public Date getExecutionStartDate() {
+ return executionStartDate;
+ }
+
+ @Override
+ public Date getExecutionEndDate() {
+ return executionEndDate;
+ }
+
+ @Override
+ public void addActiveSubPipeline(String transformName, IPipelineEngine executorPipeline) {
+ activeSubPipelines.put(transformName, executorPipeline);
+ }
+
+ @Override
+ public IPipelineEngine getActiveSubPipeline(String subPipelineName) {
+ return activeSubPipelines.get(subPipelineName);
+ }
+
+ @Override
+ public void addActiveSubWorkflow(
+ String subWorkflowName, IWorkflowEngine subWorkflow) {
+ activeSubWorkflows.put(subWorkflowName, subWorkflow);
+ }
+
+ @Override
+ public IWorkflowEngine getActiveSubWorkflow(String subWorkflowName) {
+ return activeSubWorkflows.get(subWorkflowName);
+ }
+
+ @Override
+ public void setInternalHopVariables(IVariables var) {
+ var.setVariable(
+ Const.INTERNAL_VARIABLE_PIPELINE_NAME,
+ Const.NVL(pipelineMeta != null ? pipelineMeta.getName() : "", ""));
+ var.setVariable(
+ Const.INTERNAL_VARIABLE_PIPELINE_ID,
+ logChannel != null ? logChannel.getLogChannelId() : "");
+ }
+
+ @Override
+ public void setContainerId(String containerId) {
+ this.containerId = containerId;
+ }
+
+ @Override
+ public String getContainerId() {
+ return containerId;
+ }
+
+ @Override
+ public String getStatusDescription() {
+ return statusDescription;
+ }
+
+ @Override
+ public Map getExtensionDataMap() {
+ return extensionDataMap;
+ }
+
+ @Override
+ public IPipelineEngine getParentPipeline() {
+ return parentPipeline;
+ }
+
+ @Override
+ public void setParentPipeline(IPipelineEngine parentPipeline) {
+ this.parentPipeline = parentPipeline;
+ }
+
+ @Override
+ public IWorkflowEngine getParentWorkflow() {
+ return parentWorkflow;
+ }
+
+ @Override
+ public void setParentWorkflow(IWorkflowEngine parentWorkflow) {
+ this.parentWorkflow = parentWorkflow;
+ }
+
+ @Override
+ public boolean isFeedbackShown() {
+ return false;
+ }
+
+ @Override
+ public int getFeedbackSize() {
+ return 0;
+ }
+
+ public ComponentExecutionStatus getStatus() {
+ return status;
+ }
+
+ /** Exposed for tests. */
+ public Dataset getResultDataset() {
+ return resultDataset;
+ }
+}
diff --git a/plugins/engines/spark/src/main/java/org/apache/hop/spark/engines/SparkPipelineEngineCapabilities.java b/plugins/engines/spark/src/main/java/org/apache/hop/spark/engines/SparkPipelineEngineCapabilities.java
new file mode 100644
index 00000000000..d800f98f4ae
--- /dev/null
+++ b/plugins/engines/spark/src/main/java/org/apache/hop/spark/engines/SparkPipelineEngineCapabilities.java
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hop.spark.engines;
+
+import org.apache.hop.pipeline.engine.PipelineEngineCapabilities;
+
+/** Native Spark engine: no preview/debug/sniff/pause in v1. */
+public class SparkPipelineEngineCapabilities extends PipelineEngineCapabilities {
+
+ public SparkPipelineEngineCapabilities() {
+ super(false, false, false, false);
+ }
+}
diff --git a/plugins/engines/spark/src/main/java/org/apache/hop/spark/engines/SparkPipelineRunConfiguration.java b/plugins/engines/spark/src/main/java/org/apache/hop/spark/engines/SparkPipelineRunConfiguration.java
new file mode 100644
index 00000000000..e3d478990cf
--- /dev/null
+++ b/plugins/engines/spark/src/main/java/org/apache/hop/spark/engines/SparkPipelineRunConfiguration.java
@@ -0,0 +1,236 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hop.spark.engines;
+
+import java.util.List;
+import lombok.Getter;
+import lombok.Setter;
+import org.apache.hop.core.gui.plugin.GuiElementType;
+import org.apache.hop.core.gui.plugin.GuiPlugin;
+import org.apache.hop.core.gui.plugin.GuiWidgetElement;
+import org.apache.hop.core.logging.ILogChannel;
+import org.apache.hop.i18n.BaseMessages;
+import org.apache.hop.metadata.api.HopMetadataProperty;
+import org.apache.hop.metadata.api.IHopMetadataProvider;
+import org.apache.hop.pipeline.config.PipelineRunConfiguration;
+import org.apache.hop.pipeline.engines.EmptyPipelineRunConfiguration;
+import org.apache.hop.spark.engines.template.SparkRunConfigTemplate;
+import org.apache.hop.spark.util.SparkConst;
+import org.apache.hop.spark.util.SparkRunMode;
+import org.apache.hop.ui.core.dialog.EnterSelectionDialog;
+import org.apache.hop.ui.hopgui.HopGui;
+import org.eclipse.swt.SWT;
+import org.eclipse.swt.widgets.MessageBox;
+import org.eclipse.swt.widgets.Shell;
+
+@GuiPlugin
+@Getter
+@Setter
+public class SparkPipelineRunConfiguration extends EmptyPipelineRunConfiguration
+ implements ISparkPipelineEngineRunConfiguration, Cloneable {
+
+ private static final Class PKG = SparkPipelineRunConfiguration.class;
+
+ /**
+ * Fill the form from a named deployment template (local, standalone, submit, Databricks, …).
+ * Mutates {@code object} (the live editor model); widgets are refreshed by GuiCompositeWidgets.
+ */
+ @GuiWidgetElement(
+ id = "load-spark-config-template",
+ order = "19990-spark-options",
+ parentId = PipelineRunConfiguration.GUI_PLUGIN_ELEMENT_PARENT_ID,
+ type = GuiElementType.BUTTON,
+ label = "i18n::SparkEngine.LoadTemplate.Label",
+ toolTip = "i18n::SparkEngine.LoadTemplate.ToolTip")
+ public void loadConfigurationTemplate(Object object) {
+ if (!(object instanceof SparkPipelineRunConfiguration config)) {
+ return;
+ }
+ Shell shell = HopGui.getInstance().getShell();
+ String[] labels = SparkRunConfigTemplate.displayNames();
+ EnterSelectionDialog dialog =
+ new EnterSelectionDialog(
+ shell,
+ labels,
+ BaseMessages.getString(PKG, "SparkEngine.LoadTemplate.Dialog.Title"),
+ BaseMessages.getString(PKG, "SparkEngine.LoadTemplate.Dialog.Message"));
+ dialog.setAvoidQuickSearch();
+ String choice = dialog.open();
+ if (choice == null) {
+ return;
+ }
+ SparkRunConfigTemplate template = SparkRunConfigTemplate.fromDisplayName(choice);
+ if (template == null) {
+ return;
+ }
+ if (SparkRunConfigTemplate.looksCustomized(config)) {
+ MessageBox box = new MessageBox(shell, SWT.YES | SWT.NO | SWT.ICON_QUESTION);
+ box.setText(BaseMessages.getString(PKG, "SparkEngine.LoadTemplate.Confirm.Title"));
+ box.setMessage(
+ BaseMessages.getString(
+ PKG, "SparkEngine.LoadTemplate.Confirm.Message", template.getDisplayName()));
+ if (box.open() != SWT.YES) {
+ return;
+ }
+ }
+ template.applyTo(config);
+ }
+
+ @GuiWidgetElement(
+ order = "20000-spark-options",
+ parentId = PipelineRunConfiguration.GUI_PLUGIN_ELEMENT_PARENT_ID,
+ type = GuiElementType.TEXT,
+ label = "i18n::SparkEngine.OptionsMaster.Label",
+ toolTip = "i18n::SparkEngine.OptionsMaster.ToolTip")
+ @HopMetadataProperty
+ private String sparkMaster;
+
+ @GuiWidgetElement(
+ order = "20010-spark-options",
+ parentId = PipelineRunConfiguration.GUI_PLUGIN_ELEMENT_PARENT_ID,
+ type = GuiElementType.TEXT,
+ label = "i18n::SparkEngine.OptionsAppName.Label",
+ toolTip = "i18n::SparkEngine.OptionsAppName.ToolTip")
+ @HopMetadataProperty
+ private String sparkAppName;
+
+ @GuiWidgetElement(
+ order = "20020-spark-options",
+ parentId = PipelineRunConfiguration.GUI_PLUGIN_ELEMENT_PARENT_ID,
+ type = GuiElementType.FILENAME,
+ label = "i18n::SparkEngine.OptionsFatJar.Label",
+ toolTip = "i18n::SparkEngine.OptionsFatJar.ToolTip")
+ @HopMetadataProperty
+ private String fatJar;
+
+ @GuiWidgetElement(
+ order = "20030-spark-options",
+ parentId = PipelineRunConfiguration.GUI_PLUGIN_ELEMENT_PARENT_ID,
+ type = GuiElementType.MULTI_LINE_TEXT,
+ multiLineTextHeight = 5,
+ label = "i18n::SparkEngine.OptionsConfigs.Label",
+ toolTip = "i18n::SparkEngine.OptionsConfigs.ToolTip")
+ @HopMetadataProperty
+ private String sparkConfigs;
+
+ @GuiWidgetElement(
+ order = "20040-spark-options",
+ parentId = PipelineRunConfiguration.GUI_PLUGIN_ELEMENT_PARENT_ID,
+ type = GuiElementType.TEXT,
+ label = "i18n::SparkEngine.OptionsDriverMemory.Label",
+ toolTip = "i18n::SparkEngine.OptionsDriverMemory.ToolTip")
+ @HopMetadataProperty
+ private String driverMemory;
+
+ @GuiWidgetElement(
+ order = "20050-spark-options",
+ parentId = PipelineRunConfiguration.GUI_PLUGIN_ELEMENT_PARENT_ID,
+ type = GuiElementType.TEXT,
+ label = "i18n::SparkEngine.OptionsExecutorMemory.Label",
+ toolTip = "i18n::SparkEngine.OptionsExecutorMemory.ToolTip")
+ @HopMetadataProperty
+ private String executorMemory;
+
+ @GuiWidgetElement(
+ order = "20060-spark-options",
+ parentId = PipelineRunConfiguration.GUI_PLUGIN_ELEMENT_PARENT_ID,
+ type = GuiElementType.TEXT,
+ label = "i18n::SparkEngine.OptionsExecutorCores.Label",
+ toolTip = "i18n::SparkEngine.OptionsExecutorCores.ToolTip")
+ @HopMetadataProperty
+ private String executorCores;
+
+ @GuiWidgetElement(
+ order = "20070-spark-options",
+ parentId = PipelineRunConfiguration.GUI_PLUGIN_ELEMENT_PARENT_ID,
+ type = GuiElementType.FOLDER,
+ label = "i18n::SparkEngine.OptionsTempLocation.Label",
+ toolTip = "i18n::SparkEngine.OptionsTempLocation.ToolTip")
+ @HopMetadataProperty
+ private String tempLocation;
+
+ @GuiWidgetElement(
+ order = "20080-spark-options",
+ parentId = PipelineRunConfiguration.GUI_PLUGIN_ELEMENT_PARENT_ID,
+ type = GuiElementType.TEXT,
+ label = "i18n::SparkEngine.OptionsPluginsToStage.Label",
+ toolTip = "i18n::SparkEngine.OptionsPluginsToStage.ToolTip")
+ @HopMetadataProperty
+ private String pluginsToStage;
+
+ @GuiWidgetElement(
+ order = "20090-spark-options",
+ parentId = PipelineRunConfiguration.GUI_PLUGIN_ELEMENT_PARENT_ID,
+ type = GuiElementType.MULTI_LINE_TEXT,
+ multiLineTextHeight = 4,
+ label = "i18n::SparkEngine.OptionsPathSchemeMap.Label",
+ toolTip = "i18n::SparkEngine.OptionsPathSchemeMap.ToolTip")
+ @HopMetadataProperty
+ private String pathSchemeMap;
+
+ /**
+ * Default for generic mapPartitions transforms ({@link SparkRunMode#DISTRIBUTED} or {@link
+ * SparkRunMode#DRIVER_ONLY}). Overridable per transform via the "Spark Run Mode" context action.
+ */
+ @GuiWidgetElement(
+ id = "genericTransformRunMode",
+ order = "20100-spark-options",
+ parentId = PipelineRunConfiguration.GUI_PLUGIN_ELEMENT_PARENT_ID,
+ type = GuiElementType.COMBO,
+ label = "i18n::SparkEngine.OptionsGenericRunMode.Label",
+ toolTip = "i18n::SparkEngine.OptionsGenericRunMode.ToolTip",
+ comboValuesMethod = "getGenericTransformRunModes")
+ @HopMetadataProperty(key = "generic_transform_run_mode")
+ private String genericTransformRunMode;
+
+ public SparkPipelineRunConfiguration() {
+ super();
+ this.sparkMaster = "local[*]";
+ this.sparkAppName = "Apache Hop";
+ this.tempLocation = System.getProperty("java.io.tmpdir");
+ this.genericTransformRunMode = SparkRunMode.DISTRIBUTED.name();
+ setEnginePluginId(SparkConst.PLUGIN_ID);
+ setEnginePluginName(SparkConst.PLUGIN_NAME);
+ }
+
+ public SparkPipelineRunConfiguration(SparkPipelineRunConfiguration config) {
+ super(config);
+ this.sparkMaster = config.sparkMaster;
+ this.sparkAppName = config.sparkAppName;
+ this.fatJar = config.fatJar;
+ this.sparkConfigs = config.sparkConfigs;
+ this.driverMemory = config.driverMemory;
+ this.executorMemory = config.executorMemory;
+ this.executorCores = config.executorCores;
+ this.tempLocation = config.tempLocation;
+ this.pluginsToStage = config.pluginsToStage;
+ this.pathSchemeMap = config.pathSchemeMap;
+ this.genericTransformRunMode = config.genericTransformRunMode;
+ }
+
+ @Override
+ public SparkPipelineRunConfiguration clone() {
+ return new SparkPipelineRunConfiguration(this);
+ }
+
+ /** Combo values for {@link #genericTransformRunMode}. */
+ public List getGenericTransformRunModes(
+ ILogChannel log, IHopMetadataProvider metadataProvider) {
+ return SparkRunMode.configDisplayValues();
+ }
+}
diff --git a/plugins/engines/spark/src/main/java/org/apache/hop/spark/engines/template/SparkRunConfigTemplate.java b/plugins/engines/spark/src/main/java/org/apache/hop/spark/engines/template/SparkRunConfigTemplate.java
new file mode 100644
index 00000000000..f6bb7cfb1ec
--- /dev/null
+++ b/plugins/engines/spark/src/main/java/org/apache/hop/spark/engines/template/SparkRunConfigTemplate.java
@@ -0,0 +1,215 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hop.spark.engines.template;
+
+import java.util.Arrays;
+import java.util.Objects;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.hop.spark.engines.SparkPipelineRunConfiguration;
+
+/**
+ * Named presets for {@link SparkPipelineRunConfiguration} fields. Pure data — no SWT. Apply via
+ * {@link #applyTo(SparkPipelineRunConfiguration)}.
+ */
+public enum SparkRunConfigTemplate {
+ LOCAL_EXECUTION(
+ "Local execution",
+ "Embedded Spark in the Hop process using all local cores (local[*]).",
+ "local[*]",
+ "Apache Hop",
+ "",
+ "spark.ui.enabled=false",
+ "",
+ "",
+ ""),
+ LOCAL_CAPPED(
+ "Local (capped cores)",
+ "Embedded Spark limited to 4 local cores (local[4]).",
+ "local[4]",
+ "Apache Hop",
+ "",
+ "spark.ui.enabled=false",
+ "",
+ "",
+ ""),
+ SPARK_STANDALONE(
+ "Spark standalone server",
+ "Connect Hop as a Spark client to a standalone master (replace host).",
+ "spark://host:7077",
+ "Apache Hop",
+ "",
+ "spark.ui.enabled=false",
+ "2g",
+ "2g",
+ "2"),
+ SPARK_SUBMIT(
+ "spark-submit (cluster)",
+ "Leave master empty so spark-submit --master wins; use a native-provided fat jar.",
+ "",
+ "Apache Hop",
+ "",
+ "spark.ui.enabled=false",
+ "",
+ "",
+ ""),
+ DATABRICKS(
+ "Databricks",
+ "Cluster / submit oriented placeholders. Master empty for submit-style runs; set paths for your workspace.",
+ "",
+ "Apache Hop",
+ "",
+ """
+spark.ui.enabled=false
+spark.executor.cores=3
+spark.driver.cores=1
+spark.default.parallelism=12
+spark.sql.shuffle.partitions=12
+ """,
+ "",
+ "",
+ ""),
+ YARN_CLIENT(
+ "YARN client",
+ "Submit to YARN in client deploy mode (cluster provides Spark).",
+ "yarn",
+ "Apache Hop",
+ "",
+ "spark.ui.enabled=false\nspark.submit.deployMode=client",
+ "2g",
+ "2g",
+ "2"),
+ LAKEHOUSE_LOCAL(
+ "Lakehouse local",
+ "Local[*] for Delta/Iceberg pipelines (connectors on plugin classpath; session plan applies extensions).",
+ "local[*]",
+ "Apache Hop",
+ "",
+ "spark.ui.enabled=false",
+ "",
+ "",
+ "");
+
+ private final String displayName;
+ private final String description;
+ private final String sparkMaster;
+ private final String sparkAppName;
+ private final String fatJar;
+ private final String sparkConfigs;
+ private final String driverMemory;
+ private final String executorMemory;
+ private final String executorCores;
+
+ SparkRunConfigTemplate(
+ String displayName,
+ String description,
+ String sparkMaster,
+ String sparkAppName,
+ String fatJar,
+ String sparkConfigs,
+ String driverMemory,
+ String executorMemory,
+ String executorCores) {
+ this.displayName = displayName;
+ this.description = description;
+ this.sparkMaster = sparkMaster;
+ this.sparkAppName = sparkAppName;
+ this.fatJar = fatJar;
+ this.sparkConfigs = sparkConfigs;
+ this.driverMemory = driverMemory;
+ this.executorMemory = executorMemory;
+ this.executorCores = executorCores;
+ }
+
+ public String getDisplayName() {
+ return displayName;
+ }
+
+ public String getDescription() {
+ return description;
+ }
+
+ /** Labels for selection dialogs (same order as {@link #values()}). */
+ public static String[] displayNames() {
+ return Arrays.stream(values())
+ .map(SparkRunConfigTemplate::getDisplayName)
+ .toArray(String[]::new);
+ }
+
+ public static SparkRunConfigTemplate fromDisplayName(String name) {
+ if (name == null) {
+ return null;
+ }
+ for (SparkRunConfigTemplate t : values()) {
+ if (t.displayName.equals(name)) {
+ return t;
+ }
+ }
+ return null;
+ }
+
+ /**
+ * Apply this template's field values onto {@code config}. Does not change temp location or
+ * plugins to stage (keeps operator environment defaults).
+ */
+ public void applyTo(SparkPipelineRunConfiguration config) {
+ Objects.requireNonNull(config, "config");
+ config.setSparkMaster(sparkMaster);
+ config.setSparkAppName(sparkAppName);
+ config.setFatJar(fatJar);
+ config.setSparkConfigs(sparkConfigs);
+ config.setDriverMemory(driverMemory);
+ config.setExecutorMemory(executorMemory);
+ config.setExecutorCores(executorCores);
+ }
+
+ /**
+ * True if the configuration looks customized relative to a fresh {@link
+ * SparkPipelineRunConfiguration} default (used for overwrite confirmation).
+ */
+ public static boolean looksCustomized(SparkPipelineRunConfiguration config) {
+ if (config == null) {
+ return false;
+ }
+ SparkPipelineRunConfiguration def = new SparkPipelineRunConfiguration();
+ if (!Objects.equals(nz(config.getSparkMaster()), nz(def.getSparkMaster()))) {
+ return true;
+ }
+ if (!Objects.equals(nz(config.getSparkAppName()), nz(def.getSparkAppName()))) {
+ return true;
+ }
+ if (StringUtils.isNotEmpty(config.getFatJar())) {
+ return true;
+ }
+ if (StringUtils.isNotEmpty(config.getSparkConfigs())) {
+ return true;
+ }
+ if (StringUtils.isNotEmpty(config.getDriverMemory())
+ || StringUtils.isNotEmpty(config.getExecutorMemory())
+ || StringUtils.isNotEmpty(config.getExecutorCores())) {
+ return true;
+ }
+ if (StringUtils.isNotEmpty(config.getPathSchemeMap())) {
+ return true;
+ }
+ return StringUtils.isNotEmpty(config.getPluginsToStage());
+ }
+
+ private static String nz(String s) {
+ return s == null ? "" : s;
+ }
+}
diff --git a/plugins/engines/spark/src/main/java/org/apache/hop/spark/execution/SparkTransformExecutionSampling.java b/plugins/engines/spark/src/main/java/org/apache/hop/spark/execution/SparkTransformExecutionSampling.java
new file mode 100644
index 00000000000..3c6a2c717c2
--- /dev/null
+++ b/plugins/engines/spark/src/main/java/org/apache/hop/spark/execution/SparkTransformExecutionSampling.java
@@ -0,0 +1,466 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hop.spark.execution;
+
+import com.fasterxml.jackson.core.JsonProcessingException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Date;
+import java.util.List;
+import java.util.Timer;
+import java.util.TimerTask;
+import lombok.Getter;
+import lombok.Setter;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.hop.core.Const;
+import org.apache.hop.core.exception.HopException;
+import org.apache.hop.core.exception.HopRuntimeException;
+import org.apache.hop.core.exception.HopTransformException;
+import org.apache.hop.core.json.HopJson;
+import org.apache.hop.core.logging.LogChannel;
+import org.apache.hop.core.row.IRowMeta;
+import org.apache.hop.core.row.RowBuffer;
+import org.apache.hop.core.row.RowMetaBuilder;
+import org.apache.hop.core.variables.IVariables;
+import org.apache.hop.execution.Execution;
+import org.apache.hop.execution.ExecutionBuilder;
+import org.apache.hop.execution.ExecutionDataBuilder;
+import org.apache.hop.execution.ExecutionDataSetMeta;
+import org.apache.hop.execution.ExecutionInfoLocation;
+import org.apache.hop.execution.ExecutionState;
+import org.apache.hop.execution.ExecutionStateBuilder;
+import org.apache.hop.execution.ExecutionType;
+import org.apache.hop.execution.IExecutionInfoLocation;
+import org.apache.hop.execution.profiling.ExecutionDataProfile;
+import org.apache.hop.execution.sampler.ExecutionDataSamplerMeta;
+import org.apache.hop.execution.sampler.IExecutionDataSampler;
+import org.apache.hop.execution.sampler.IExecutionDataSamplerStore;
+import org.apache.hop.metadata.api.IHopMetadataProvider;
+import org.apache.hop.pipeline.Pipeline;
+import org.apache.hop.pipeline.config.PipelineRunConfiguration;
+import org.apache.hop.pipeline.engine.IEngineComponent;
+import org.apache.hop.pipeline.engines.local.LocalPipelineEngine;
+import org.apache.hop.pipeline.transform.ITransform;
+import org.apache.hop.pipeline.transform.RowAdapter;
+import org.apache.hop.pipeline.transform.stream.IStream;
+import org.apache.hop.spark.core.SparkExecutionDataAccumulator;
+
+/**
+ * Executor-side sampling for Hop transforms running in Spark mapPartitions, ported from Beam {@code
+ * TransformBaseFn}.
+ *
+ *
Samples are shipped to the driver via {@link SparkExecutionDataAccumulator} so the driver's
+ * execution info location (which owns the parent pipeline {@code CacheEntry}) can call {@code
+ * registerData}. Executor-local {@code registerData} is best-effort only (file locations).
+ *
+ *
Not serializable — construct after Hop/plugin init on the executor.
+ */
+@Getter
+@Setter
+public class SparkTransformExecutionSampling {
+
+ private final String transformName;
+ private final String parentLogChannelId;
+ private final int copyNr;
+ private final String ownerLogChannelId;
+
+ private ExecutionInfoLocation executionInfoLocation;
+ private List dataSamplers = List.of();
+ private final List dataSamplerStores = new ArrayList<>();
+ private Timer executionInfoTimer;
+ private LocalPipelineEngine pipeline;
+ private ITransform transform;
+ private boolean active;
+ private SparkExecutionDataAccumulator sampleDataAccumulator;
+
+ public SparkTransformExecutionSampling(
+ String transformName, String parentLogChannelId, int copyNr) {
+ this(transformName, parentLogChannelId, copyNr, null);
+ }
+
+ public SparkTransformExecutionSampling(
+ String transformName,
+ String parentLogChannelId,
+ int copyNr,
+ SparkExecutionDataAccumulator sampleDataAccumulator) {
+ this.transformName = transformName;
+ this.parentLogChannelId = parentLogChannelId;
+ this.copyNr = copyNr;
+ this.sampleDataAccumulator = sampleDataAccumulator;
+ String parent =
+ StringUtils.isNotEmpty(parentLogChannelId) ? parentLogChannelId : "spark-pipeline";
+ this.ownerLogChannelId = parent + "|" + transformName + "|" + copyNr;
+ }
+
+ /**
+ * Load location + data profile samplers from metadata. Sampling is only active when both an
+ * execution information location and a non-empty data profile (or extra samplers JSON) are
+ * present.
+ */
+ @SuppressWarnings("rawtypes")
+ public void lookup(
+ IVariables variables,
+ IHopMetadataProvider metadataProvider,
+ String runConfigName,
+ String dataSamplersJson)
+ throws HopException, JsonProcessingException {
+ active = false;
+ executionInfoLocation = null;
+ dataSamplers = new ArrayList<>();
+ dataSamplerStores.clear();
+
+ if (StringUtils.isEmpty(runConfigName) || metadataProvider == null) {
+ LogChannel.GENERAL.logDebug(
+ "Spark sampling inactive for '"
+ + transformName
+ + "': runConfigName or metadataProvider missing");
+ return;
+ }
+
+ PipelineRunConfiguration runConf =
+ metadataProvider.getSerializer(PipelineRunConfiguration.class).load(runConfigName);
+ if (runConf == null) {
+ LogChannel.GENERAL.logBasic(
+ "Spark sampling inactive for '"
+ + transformName
+ + "': run configuration '"
+ + runConfigName
+ + "' not found in metadata");
+ return;
+ }
+
+ String locationName = runConf.getExecutionInfoLocationName();
+ if (StringUtils.isEmpty(locationName)) {
+ LogChannel.GENERAL.logDebug(
+ "Spark sampling inactive for '"
+ + transformName
+ + "': no execution info location on run configuration '"
+ + runConfigName
+ + "'");
+ return;
+ }
+
+ ExecutionInfoLocation location =
+ metadataProvider.getSerializer(ExecutionInfoLocation.class).load(locationName);
+ if (location == null) {
+ LogChannel.GENERAL.logBasic(
+ "Spark sampling inactive for '"
+ + transformName
+ + "': execution info location '"
+ + locationName
+ + "' not found in metadata");
+ return;
+ }
+
+ String profileName = runConf.getExecutionDataProfileName();
+ if (StringUtils.isNotEmpty(profileName)) {
+ ExecutionDataProfile dataProfile =
+ metadataProvider.getSerializer(ExecutionDataProfile.class).load(profileName);
+ if (dataProfile != null && dataProfile.getSamplers() != null) {
+ dataSamplers.addAll(dataProfile.getSamplers());
+ } else {
+ LogChannel.GENERAL.logBasic(
+ "Spark sampling: data profile '"
+ + profileName
+ + "' missing or has no samplers for transform '"
+ + transformName
+ + "'");
+ }
+ }
+
+ if (StringUtils.isNotEmpty(dataSamplersJson) && !"[]".equals(dataSamplersJson.trim())) {
+ try {
+ IExecutionDataSampler[] extraSamplers =
+ HopJson.newMapper().readValue(dataSamplersJson, IExecutionDataSampler[].class);
+ if (extraSamplers != null) {
+ dataSamplers.addAll(Arrays.asList(extraSamplers));
+ }
+ } catch (LinkageError | Exception e) {
+ // local[*]: plugin CL may have a second Jackson → LinkageError on HopJson.newMapper().
+ // Profile samplers above are enough; extra GUI samplers are best-effort.
+ LogChannel.GENERAL.logError(
+ "Unable to deserialize extra data samplers JSON for transform '"
+ + transformName
+ + "' (non-fatal): "
+ + e.getMessage());
+ }
+ }
+
+ if (dataSamplers.isEmpty()) {
+ LogChannel.GENERAL.logBasic(
+ "Spark sampling inactive for '"
+ + transformName
+ + "': no data samplers (profile='"
+ + Const.NVL(profileName, "")
+ + "', extraJson empty="
+ + StringUtils.isEmpty(dataSamplersJson)
+ + ")");
+ return;
+ }
+
+ executionInfoLocation = location;
+ // Location init still needed for optional executor-local registerData (file locations)
+ IExecutionInfoLocation iLocation = executionInfoLocation.getExecutionInfoLocation();
+ iLocation.initialize(variables, metadataProvider);
+ active = true;
+ LogChannel.GENERAL.logBasic(
+ "Spark sampling active for '"
+ + transformName
+ + "' copy "
+ + copyNr
+ + " ("
+ + dataSamplers.size()
+ + " sampler(s), location='"
+ + locationName
+ + "', driverAccumulator="
+ + (sampleDataAccumulator != null)
+ + ")");
+ }
+
+ /** Register a transform execution node under the parent pipeline execution. */
+ public void registerExecutingTransform(LocalPipelineEngine localPipeline) {
+ if (!active || executionInfoLocation == null || localPipeline == null) {
+ return;
+ }
+ try {
+ ITransform t = localPipeline.getTransform(transformName, 0);
+ if (t == null) {
+ return;
+ }
+ List copies = localPipeline.getComponentCopies(transformName);
+ if (copies == null || copies.isEmpty()) {
+ return;
+ }
+ IEngineComponent transformComponent = copies.get(0);
+ Execution execution = ExecutionBuilder.fromTransform(localPipeline, t).build();
+ execution.setParentId(parentLogChannelId);
+ execution.setId(ownerLogChannelId);
+ execution.setCopyNr(Integer.toString(copyNr));
+ executionInfoLocation.getExecutionInfoLocation().registerExecution(execution);
+
+ ExecutionState transformState =
+ ExecutionStateBuilder.fromTransform(localPipeline, transformComponent).build();
+ transformState.setParentId(parentLogChannelId);
+ transformState.setId(ownerLogChannelId);
+ transformState.setCopyNr(Integer.toString(copyNr));
+ executionInfoLocation.getExecutionInfoLocation().updateExecutionState(transformState);
+ } catch (Exception e) {
+ LogChannel.GENERAL.logError(
+ "Error registering transform execution for sampling (non-fatal)", e);
+ }
+ }
+
+ /**
+ * Attach profile samplers to transform OUTPUT rows and start a timer to flush samples to the
+ * location.
+ */
+ @SuppressWarnings({"rawtypes", "unchecked"})
+ public void attach(
+ IVariables variables,
+ LocalPipelineEngine localPipeline,
+ ITransform transform,
+ IRowMeta inputRowMeta,
+ IRowMeta outputRowMeta) {
+ if (!active || executionInfoLocation == null || dataSamplers.isEmpty() || transform == null) {
+ return;
+ }
+ this.pipeline = localPipeline;
+ this.transform = transform;
+
+ ExecutionDataSamplerMeta dataSamplerMeta =
+ new ExecutionDataSamplerMeta(
+ transformName, Integer.toString(copyNr), ownerLogChannelId, false, false);
+
+ for (IExecutionDataSampler dataSampler : dataSamplers) {
+ IExecutionDataSamplerStore dataSamplerStore = dataSampler.createSamplerStore(dataSamplerMeta);
+ dataSamplerStore.init(variables, inputRowMeta, outputRowMeta);
+ dataSamplerStores.add(dataSamplerStore);
+ }
+
+ transform.addRowListener(
+ new RowAdapter() {
+ @Override
+ public void rowWrittenEvent(IRowMeta rowMeta, Object[] row) throws HopTransformException {
+ for (int s = 0; s < dataSamplers.size(); s++) {
+ IExecutionDataSampler sampler = dataSamplers.get(s);
+ IExecutionDataSamplerStore store = dataSamplerStores.get(s);
+ try {
+ sampler.sampleRow(store, IStream.StreamType.OUTPUT, rowMeta, row);
+ } catch (HopException e) {
+ throw new HopRuntimeException("Error sampling row on Spark partition", e);
+ }
+ }
+ }
+ });
+
+ long delay = Const.toLong(executionInfoLocation.getDataLoggingDelay(), 5000L);
+ long interval = Const.toLong(executionInfoLocation.getDataLoggingInterval(), 10000L);
+ TimerTask task =
+ new TimerTask() {
+ @Override
+ public void run() {
+ try {
+ sendSamplesToLocation(false);
+ } catch (Exception e) {
+ LogChannel.GENERAL.logError(
+ "Error sending transform samples to location (non-fatal)", e);
+ }
+ }
+ };
+ executionInfoTimer = new Timer("spark-sample-" + transformName + "-" + copyNr, true);
+ executionInfoTimer.schedule(task, delay, interval);
+ }
+
+ /**
+ * Push sampler stores + transform metrics. Prefer the driver accumulator (reliable for caching
+ * locations); also best-effort {@code registerData} on this process for file locations.
+ */
+ @SuppressWarnings("rawtypes")
+ public void sendSamplesToLocation(boolean finished) throws HopException {
+ if (!active || executionInfoLocation == null || pipeline == null || transform == null) {
+ return;
+ }
+
+ ExecutionDataBuilder dataBuilder =
+ ExecutionDataBuilder.of()
+ .withOwnerId(ownerLogChannelId)
+ .withParentId(parentLogChannelId)
+ .withExecutionType(ExecutionType.Transform)
+ .withCollectionDate(new Date())
+ .withFinished(finished);
+
+ for (IExecutionDataSamplerStore store : dataSamplerStores) {
+ dataBuilder.addDataSets(store.getSamples()).addSetMeta(store.getSamplesMetadata());
+ }
+
+ dataBuilder.addSetMeta(
+ ownerLogChannelId,
+ new ExecutionDataSetMeta(
+ ownerLogChannelId,
+ ownerLogChannelId,
+ transformName,
+ ownerLogChannelId,
+ transformName + "." + ownerLogChannelId + " (Metrics)"));
+ dataBuilder.addDataSet(
+ ownerLogChannelId,
+ new RowBuffer(
+ new RowMetaBuilder().addString("metric").addInteger("value").build(),
+ List.of(
+ new Object[] {Pipeline.METRIC_NAME_INPUT, transform.getLinesInput()},
+ new Object[] {Pipeline.METRIC_NAME_OUTPUT, transform.getLinesOutput()},
+ new Object[] {Pipeline.METRIC_NAME_READ, transform.getLinesRead()},
+ new Object[] {Pipeline.METRIC_NAME_WRITTEN, transform.getLinesWritten()},
+ new Object[] {Pipeline.METRIC_NAME_REJECTED, transform.getLinesRejected()},
+ new Object[] {Pipeline.METRIC_NAME_ERROR, transform.getErrors()})));
+
+ var executionData = dataBuilder.build();
+
+ // Primary path: ship JSON to the driver (owns parent CacheEntry for caching locations)
+ if (sampleDataAccumulator != null) {
+ try {
+ String json = HopJson.newMapper().writeValueAsString(executionData);
+ sampleDataAccumulator.addSample(ownerLogChannelId, json);
+ } catch (LinkageError e) {
+ // Dual Jackson on local[*] plugin CL — fall through to local registerData only
+ LogChannel.GENERAL.logError(
+ "Unable to serialize samples via HopJson for '"
+ + transformName
+ + "' (using local registerData only): "
+ + e.getMessage());
+ } catch (JsonProcessingException e) {
+ throw new HopException(
+ "Unable to serialize execution sample data for transform '" + transformName + "'", e);
+ }
+ }
+
+ // Best-effort local registration (works for file locations when parent folder exists)
+ IExecutionInfoLocation iLocation = executionInfoLocation.getExecutionInfoLocation();
+ try {
+ iLocation.registerData(executionData);
+ } catch (Exception e) {
+ if (sampleDataAccumulator == null) {
+ throw e instanceof HopException hopEx
+ ? hopEx
+ : new HopException("Error registering execution data locally", e);
+ }
+ LogChannel.GENERAL.logDebug(
+ "Local registerData skipped/failed for '"
+ + transformName
+ + "' (samples sent via driver accumulator): "
+ + e.getMessage());
+ }
+
+ List copies = pipeline.getComponentCopies(transformName);
+ if (copies != null && !copies.isEmpty()) {
+ ExecutionState transformState =
+ ExecutionStateBuilder.fromTransform(pipeline, copies.get(0)).build();
+ transformState.setParentId(parentLogChannelId);
+ transformState.setId(ownerLogChannelId);
+ transformState.setCopyNr(Integer.toString(copyNr));
+ try {
+ iLocation.updateExecutionState(transformState);
+ } catch (Exception e) {
+ LogChannel.GENERAL.logDebug(
+ "Local updateExecutionState skipped for '" + transformName + "': " + e.getMessage());
+ }
+ }
+ }
+
+ /** Final sample flush, cancel timer, close location instance for this partition. */
+ public void close() {
+ if (!active) {
+ return;
+ }
+ try {
+ if (executionInfoTimer != null) {
+ executionInfoTimer.cancel();
+ executionInfoTimer = null;
+ }
+ try {
+ sendSamplesToLocation(true);
+ } catch (Exception e) {
+ LogChannel.GENERAL.logError("Error sending final transform samples (non-fatal)", e);
+ }
+ if (executionInfoLocation != null
+ && executionInfoLocation.getExecutionInfoLocation() != null) {
+ try {
+ executionInfoLocation.getExecutionInfoLocation().close();
+ } catch (Exception e) {
+ LogChannel.GENERAL.logError(
+ "Error closing execution info location on partition (non-fatal)", e);
+ }
+ }
+ } finally {
+ active = false;
+ }
+ }
+
+ /** Serialize extra GUI samplers for the mapPartitions closure (Beam-compatible). */
+ public static String serializeDataSamplers(
+ List> dataSamplers)
+ throws HopException {
+ try {
+ if (dataSamplers == null || dataSamplers.isEmpty()) {
+ return "[]";
+ }
+ return HopJson.newMapper().writeValueAsString(dataSamplers);
+ } catch (JsonProcessingException e) {
+ throw new HopException("Unable to serialize execution data samplers", e);
+ }
+ }
+}
diff --git a/plugins/engines/spark/src/main/java/org/apache/hop/spark/gui/DrawSparkRunModeOnTransformExtensionPoint.java b/plugins/engines/spark/src/main/java/org/apache/hop/spark/gui/DrawSparkRunModeOnTransformExtensionPoint.java
new file mode 100644
index 00000000000..c360d185c29
--- /dev/null
+++ b/plugins/engines/spark/src/main/java/org/apache/hop/spark/gui/DrawSparkRunModeOnTransformExtensionPoint.java
@@ -0,0 +1,100 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hop.spark.gui;
+
+import org.apache.hop.core.extension.ExtensionPoint;
+import org.apache.hop.core.extension.IExtensionPoint;
+import org.apache.hop.core.gui.AreaOwner;
+import org.apache.hop.core.gui.IGc;
+import org.apache.hop.core.gui.Rectangle;
+import org.apache.hop.core.logging.ILogChannel;
+import org.apache.hop.core.svg.SvgFile;
+import org.apache.hop.core.variables.IVariables;
+import org.apache.hop.pipeline.PipelinePainterExtension;
+import org.apache.hop.pipeline.transform.TransformMeta;
+import org.apache.hop.spark.util.SparkRunMode;
+
+/**
+ * Draws a small badge at the bottom-left of a transform icon when a Spark run-mode override is set
+ * (Force distributed / Force Driver Only).
+ */
+@ExtensionPoint(
+ id = "DrawSparkRunModeOnTransformExtensionPoint",
+ description =
+ "Draw Spark run-mode override badge (bottom-left) on transforms with Force Distributed or Force Driver Only",
+ extensionPointId = "PipelinePainterTransform")
+public class DrawSparkRunModeOnTransformExtensionPoint
+ implements IExtensionPoint {
+
+ public static final String AREA_OWNER_PREFIX = "Spark run mode: ";
+
+ @Override
+ public void callExtensionPoint(
+ ILogChannel log, IVariables variables, PipelinePainterExtension ext) {
+ try {
+ TransformMeta transformMeta = ext.transformMeta;
+ if (transformMeta == null || !SparkRunMode.hasForcedOverride(transformMeta)) {
+ return;
+ }
+
+ String override = SparkRunMode.getOverride(transformMeta);
+ String svgName =
+ SparkRunMode.OVERRIDE_FORCE_DRIVER_ONLY.equals(override)
+ ? "spark-run-driver.svg"
+ : "spark-run-distributed.svg";
+ String label = SparkRunMode.displayLabelForOverride(override);
+
+ Rectangle r =
+ drawBadge(
+ ext.gc, ext.x1, ext.y1, ext.iconSize, svgName, this.getClass().getClassLoader());
+ ext.areaOwners.add(
+ new AreaOwner(
+ AreaOwner.AreaType.CUSTOM,
+ r.x,
+ r.y,
+ r.width,
+ r.height,
+ ext.offset,
+ transformMeta,
+ AREA_OWNER_PREFIX + label));
+ } catch (Exception e) {
+ // Not critical for canvas painting
+ log.logError("Error drawing Spark run-mode badge", e);
+ }
+ }
+
+ /** Bottom-left corner of the transform icon (badge overlaps the lower-left of the glyph). */
+ static Rectangle drawBadge(
+ IGc gc, int x, int y, int iconSize, String svgResource, ClassLoader classLoader)
+ throws Exception {
+ int imageWidth = 16;
+ int imageHeight = 16;
+ int locationX = x - (2 * imageWidth) / 3;
+ int locationY = y + iconSize - imageHeight / 3;
+
+ gc.drawImage(
+ new SvgFile(svgResource, classLoader),
+ locationX,
+ locationY,
+ imageWidth,
+ imageHeight,
+ gc.getMagnification(),
+ 0);
+ return new Rectangle(locationX, locationY, imageWidth, imageHeight);
+ }
+}
diff --git a/plugins/engines/spark/src/main/java/org/apache/hop/spark/gui/HopSparkGuiPlugin.java b/plugins/engines/spark/src/main/java/org/apache/hop/spark/gui/HopSparkGuiPlugin.java
new file mode 100644
index 00000000000..87e1335c45d
--- /dev/null
+++ b/plugins/engines/spark/src/main/java/org/apache/hop/spark/gui/HopSparkGuiPlugin.java
@@ -0,0 +1,232 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hop.spark.gui;
+
+import org.apache.commons.lang3.StringUtils;
+import org.apache.hop.core.Const;
+import org.apache.hop.core.action.GuiContextAction;
+import org.apache.hop.core.gui.plugin.GuiPlugin;
+import org.apache.hop.core.gui.plugin.action.GuiActionType;
+import org.apache.hop.core.gui.plugin.menu.GuiMenuElement;
+import org.apache.hop.core.variables.IVariables;
+import org.apache.hop.i18n.BaseMessages;
+import org.apache.hop.pipeline.PipelineMeta;
+import org.apache.hop.pipeline.transform.TransformMeta;
+import org.apache.hop.spark.pkg.SparkProjectPackage;
+import org.apache.hop.spark.util.SparkRunMode;
+import org.apache.hop.ui.core.dialog.BaseDialog;
+import org.apache.hop.ui.core.dialog.EnterSelectionDialog;
+import org.apache.hop.ui.core.dialog.ErrorDialog;
+import org.apache.hop.ui.core.gui.GuiResource;
+import org.apache.hop.ui.hopgui.HopGui;
+import org.apache.hop.ui.hopgui.file.pipeline.context.HopGuiPipelineTransformContext;
+import org.eclipse.swt.SWT;
+import org.eclipse.swt.widgets.MessageBox;
+import org.eclipse.swt.widgets.Shell;
+
+/**
+ * GUI actions for the Native Spark engine, including project package export for cluster execution
+ * and per-transform Spark run-mode overrides.
+ */
+@GuiPlugin
+public class HopSparkGuiPlugin {
+
+ public static final Class PKG = HopSparkGuiPlugin.class;
+
+ public static final String ID_MAIN_MENU_TOOLS_EXPORT_SPARK_PROJECT =
+ "40300-menu-tools-export-spark-project-package";
+
+ public static final String ID_MAIN_MENU_TOOLS_SPARK_PACKAGE_CONFIG =
+ "40310-menu-tools-spark-package-config";
+
+ public static final String ACTION_ID_PIPELINE_GRAPH_TRANSFORM_SPARK_RUN_MODE =
+ "pipeline-graph-transform-spark-run-mode";
+
+ private static HopSparkGuiPlugin instance;
+
+ public static HopSparkGuiPlugin getInstance() {
+ if (instance == null) {
+ instance = new HopSparkGuiPlugin();
+ }
+ return instance;
+ }
+
+ /**
+ * Per-transform override of generic mapPartitions run mode (Inherit / Force distributed / Force
+ * Driver Only). Stored on {@link TransformMeta} attributes; effective when running on Native
+ * Spark.
+ */
+ @GuiContextAction(
+ id = ACTION_ID_PIPELINE_GRAPH_TRANSFORM_SPARK_RUN_MODE,
+ parentId = HopGuiPipelineTransformContext.CONTEXT_ID,
+ type = GuiActionType.Modify,
+ name = "i18n::SparkGuiPlugin.ContextAction.SparkRunMode.Name",
+ tooltip = "i18n::SparkGuiPlugin.ContextAction.SparkRunMode.Tooltip",
+ image = "spark-run-driver.svg",
+ category = "i18n::SparkGuiPlugin.ContextAction.Category",
+ categoryOrder = "9")
+ public void setSparkRunMode(HopGuiPipelineTransformContext context) {
+ HopGui hopGui = HopGui.getInstance();
+ try {
+ TransformMeta transformMeta = context.getTransformMeta();
+ PipelineMeta pipelineMeta = context.getPipelineMeta();
+
+ String currentLabel =
+ SparkRunMode.displayLabelForOverride(SparkRunMode.getOverride(transformMeta));
+ String[] labels = SparkRunMode.overrideDisplayLabels();
+ int preselect = 0;
+ for (int i = 0; i < labels.length; i++) {
+ if (labels[i].equals(currentLabel)) {
+ preselect = i;
+ break;
+ }
+ }
+ EnterSelectionDialog dialog =
+ new EnterSelectionDialog(
+ hopGui.getShell(),
+ labels,
+ BaseMessages.getString(PKG, "SparkGuiPlugin.ContextAction.SparkRunMode.Dialog.Title"),
+ BaseMessages.getString(
+ PKG,
+ "SparkGuiPlugin.ContextAction.SparkRunMode.Dialog.Message",
+ transformMeta.getName(),
+ currentLabel));
+ dialog.setAvoidQuickSearch();
+ String choice = dialog.open(preselect);
+ if (choice == null) {
+ return;
+ }
+ SparkRunMode.setOverride(transformMeta, SparkRunMode.overrideFromDisplayLabel(choice));
+ transformMeta.setChanged();
+ pipelineMeta.setChanged();
+ context.getPipelineGraph().updateGui();
+ } catch (Exception e) {
+ new ErrorDialog(
+ hopGui.getShell(),
+ BaseMessages.getString(PKG, "SparkGuiPlugin.ContextAction.SparkRunMode.Error.Header"),
+ BaseMessages.getString(PKG, "SparkGuiPlugin.ContextAction.SparkRunMode.Error.Message"),
+ e);
+ }
+ }
+
+ /** Edit {@code spark-package.json} include/exclude rules for Native Spark project packages. */
+ @GuiMenuElement(
+ root = HopGui.ID_MAIN_MENU,
+ id = ID_MAIN_MENU_TOOLS_SPARK_PACKAGE_CONFIG,
+ label = "i18n::SparkGuiPlugin.Menu.SparkPackageConfig.Text",
+ parentId = HopGui.ID_MAIN_MENU_TOOLS_PARENT_ID,
+ image = "spark-file-input.svg",
+ separator = true)
+ public void menuToolsSparkPackageConfig() {
+ HopGui hopGui = HopGui.getInstance();
+ Shell shell = hopGui.getShell();
+ IVariables variables = hopGui.getVariables();
+ String projectHome = variables.getVariable("PROJECT_HOME");
+ if (StringUtils.isEmpty(projectHome)) {
+ MessageBox box = new MessageBox(shell, SWT.CLOSE | SWT.ICON_WARNING);
+ box.setText(
+ BaseMessages.getString(PKG, "SparkGuiPlugin.ExportSparkProject.NoProject.Header"));
+ box.setMessage(
+ BaseMessages.getString(PKG, "SparkGuiPlugin.ExportSparkProject.NoProject.Message"));
+ box.open();
+ return;
+ }
+ projectHome = variables.resolve(projectHome);
+ new SparkPackageConfigDialog(shell, projectHome).open();
+ }
+
+ /**
+ * Export the active project as a zip for Native Spark execution (MainSpark
+ * {@code --HopProjectPackage}). Not the same as File → Export current project to zip.
+ */
+ @GuiMenuElement(
+ root = HopGui.ID_MAIN_MENU,
+ id = ID_MAIN_MENU_TOOLS_EXPORT_SPARK_PROJECT,
+ label = "i18n::SparkGuiPlugin.Menu.ExportSparkProjectPackage.Text",
+ parentId = HopGui.ID_MAIN_MENU_TOOLS_PARENT_ID,
+ image = "spark-file-input.svg")
+ public void menuToolsExportSparkProjectPackage() {
+ HopGui hopGui = HopGui.getInstance();
+ Shell shell = hopGui.getShell();
+ IVariables variables = hopGui.getVariables();
+
+ MessageBox intro = new MessageBox(shell, SWT.OK | SWT.CANCEL | SWT.ICON_INFORMATION);
+ intro.setText(BaseMessages.getString(PKG, "SparkGuiPlugin.ExportSparkProject.Dialog.Header"));
+ intro.setMessage(
+ BaseMessages.getString(PKG, "SparkGuiPlugin.ExportSparkProject.Dialog.Message1")
+ + Const.CR
+ + Const.CR
+ + BaseMessages.getString(PKG, "SparkGuiPlugin.ExportSparkProject.Dialog.Message2")
+ + Const.CR
+ + Const.CR
+ + BaseMessages.getString(PKG, "SparkGuiPlugin.ExportSparkProject.Dialog.Message3"));
+ if ((intro.open() & SWT.CANCEL) != 0) {
+ return;
+ }
+
+ String projectHome = variables.getVariable("PROJECT_HOME");
+ if (StringUtils.isEmpty(projectHome)) {
+ MessageBox box = new MessageBox(shell, SWT.CLOSE | SWT.ICON_WARNING);
+ box.setText(
+ BaseMessages.getString(PKG, "SparkGuiPlugin.ExportSparkProject.NoProject.Header"));
+ box.setMessage(
+ BaseMessages.getString(PKG, "SparkGuiPlugin.ExportSparkProject.NoProject.Message"));
+ box.open();
+ return;
+ }
+
+ String zipFilename =
+ BaseDialog.presentFileDialog(
+ true,
+ shell,
+ new String[] {"*.zip", "*.*"},
+ new String[] {
+ BaseMessages.getString(PKG, "SparkGuiPlugin.FileTypes.Zip.Label"),
+ BaseMessages.getString(PKG, "SparkGuiPlugin.FileTypes.All.Label")
+ },
+ true);
+ if (zipFilename == null) {
+ return;
+ }
+
+ try {
+ SparkProjectPackage.exportProject(
+ projectHome, zipFilename, hopGui.getMetadataProvider(), variables);
+
+ GuiResource.getInstance().toClipboard(zipFilename);
+
+ MessageBox done = new MessageBox(shell, SWT.CLOSE | SWT.ICON_INFORMATION);
+ done.setText(BaseMessages.getString(PKG, "SparkGuiPlugin.ExportSparkProject.Done.Header"));
+ done.setMessage(
+ BaseMessages.getString(
+ PKG, "SparkGuiPlugin.ExportSparkProject.Done.Message1", zipFilename)
+ + Const.CR
+ + Const.CR
+ + BaseMessages.getString(PKG, "SparkGuiPlugin.ExportSparkProject.Done.Message2")
+ + Const.CR
+ + BaseMessages.getString(PKG, "SparkGuiPlugin.ExportSparkProject.Done.Message3"));
+ done.open();
+ } catch (Exception e) {
+ new ErrorDialog(
+ shell,
+ BaseMessages.getString(PKG, "SparkGuiPlugin.ExportSparkProject.Error.Header"),
+ BaseMessages.getString(PKG, "SparkGuiPlugin.ExportSparkProject.Error.Message"),
+ e);
+ }
+ }
+}
diff --git a/plugins/engines/spark/src/main/java/org/apache/hop/spark/gui/SparkPackageConfigDialog.java b/plugins/engines/spark/src/main/java/org/apache/hop/spark/gui/SparkPackageConfigDialog.java
new file mode 100644
index 00000000000..626ebb572cb
--- /dev/null
+++ b/plugins/engines/spark/src/main/java/org/apache/hop/spark/gui/SparkPackageConfigDialog.java
@@ -0,0 +1,261 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hop.spark.gui;
+
+import java.util.Arrays;
+import java.util.List;
+import java.util.stream.Collectors;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.hop.core.Const;
+import org.apache.hop.i18n.BaseMessages;
+import org.apache.hop.spark.pkg.PackageExportFilter;
+import org.apache.hop.ui.core.PropsUi;
+import org.apache.hop.ui.core.dialog.BaseDialog;
+import org.apache.hop.ui.core.dialog.ErrorDialog;
+import org.apache.hop.ui.core.gui.WindowProperty;
+import org.apache.hop.ui.pipeline.transform.BaseTransformDialog;
+import org.eclipse.swt.SWT;
+import org.eclipse.swt.custom.StyledText;
+import org.eclipse.swt.layout.FormAttachment;
+import org.eclipse.swt.layout.FormData;
+import org.eclipse.swt.layout.FormLayout;
+import org.eclipse.swt.widgets.Button;
+import org.eclipse.swt.widgets.Dialog;
+import org.eclipse.swt.widgets.Label;
+import org.eclipse.swt.widgets.Shell;
+import org.eclipse.swt.widgets.Text;
+
+/**
+ * Edit {@link PackageExportFilter#CONFIG_FILENAME} for the active project (Native Spark package
+ * include/exclude rules).
+ */
+public class SparkPackageConfigDialog extends Dialog {
+ private static final Class PKG = HopSparkGuiPlugin.class;
+
+ private final String projectHome;
+ private Shell shell;
+ private PropsUi props;
+
+ private Text wProjectHome;
+ private StyledText wExcludeDirs;
+ private StyledText wExcludeGlobs;
+ private StyledText wIncludePaths;
+ private Button wReplaceDefaults;
+
+ public SparkPackageConfigDialog(Shell parent, String projectHome) {
+ super(parent, SWT.NONE);
+ this.projectHome = projectHome;
+ this.props = PropsUi.getInstance();
+ }
+
+ public void open() {
+ Shell parent = getParent();
+
+ shell = new Shell(parent, SWT.DIALOG_TRIM | SWT.RESIZE | SWT.MAX | SWT.MIN);
+ props.setLook(shell);
+ shell.setText(BaseMessages.getString(PKG, "SparkPackageConfigDialog.Shell.Title"));
+
+ FormLayout formLayout = new FormLayout();
+ formLayout.marginWidth = PropsUi.getFormMargin();
+ formLayout.marginHeight = PropsUi.getFormMargin();
+ shell.setLayout(formLayout);
+
+ int middle = props.getMiddlePct();
+ int margin = PropsUi.getMargin();
+
+ Button wOk = new Button(shell, SWT.PUSH);
+ wOk.setText(BaseMessages.getString(PKG, "System.Button.OK"));
+ Button wCancel = new Button(shell, SWT.PUSH);
+ wCancel.setText(BaseMessages.getString(PKG, "System.Button.Cancel"));
+ BaseTransformDialog.positionBottomButtons(shell, new Button[] {wOk, wCancel}, margin, null);
+ wOk.addListener(SWT.Selection, e -> ok());
+ wCancel.addListener(SWT.Selection, e -> cancel());
+
+ Label wlIntro = new Label(shell, SWT.LEFT | SWT.WRAP);
+ props.setLook(wlIntro);
+ wlIntro.setText(BaseMessages.getString(PKG, "SparkPackageConfigDialog.Intro"));
+ FormData fdlIntro = new FormData();
+ fdlIntro.left = new FormAttachment(0, 0);
+ fdlIntro.top = new FormAttachment(0, margin);
+ fdlIntro.right = new FormAttachment(100, 0);
+ wlIntro.setLayoutData(fdlIntro);
+
+ Label wlHome = new Label(shell, SWT.RIGHT);
+ props.setLook(wlHome);
+ wlHome.setText(BaseMessages.getString(PKG, "SparkPackageConfigDialog.ProjectHome.Label"));
+ FormData fdlHome = new FormData();
+ fdlHome.left = new FormAttachment(0, 0);
+ fdlHome.top = new FormAttachment(wlIntro, margin * 2);
+ fdlHome.right = new FormAttachment(middle, -margin);
+ wlHome.setLayoutData(fdlHome);
+ wProjectHome = new Text(shell, SWT.SINGLE | SWT.LEFT | SWT.BORDER | SWT.READ_ONLY);
+ props.setLook(wProjectHome);
+ FormData fdHome = new FormData();
+ fdHome.left = new FormAttachment(middle, 0);
+ fdHome.top = new FormAttachment(wlIntro, margin * 2);
+ fdHome.right = new FormAttachment(100, 0);
+ wProjectHome.setLayoutData(fdHome);
+
+ Label wlDefaults = new Label(shell, SWT.LEFT | SWT.WRAP);
+ props.setLook(wlDefaults);
+ wlDefaults.setText(
+ BaseMessages.getString(
+ PKG,
+ "SparkPackageConfigDialog.Defaults.Label",
+ String.join(", ", PackageExportFilter.DEFAULT_EXCLUDE_DIRS)));
+ FormData fdlDefaults = new FormData();
+ fdlDefaults.left = new FormAttachment(0, 0);
+ fdlDefaults.top = new FormAttachment(wProjectHome, margin);
+ fdlDefaults.right = new FormAttachment(100, 0);
+ wlDefaults.setLayoutData(fdlDefaults);
+
+ wReplaceDefaults = new Button(shell, SWT.CHECK);
+ props.setLook(wReplaceDefaults);
+ wReplaceDefaults.setText(
+ BaseMessages.getString(PKG, "SparkPackageConfigDialog.ReplaceDefaults.Label"));
+ FormData fdReplace = new FormData();
+ fdReplace.left = new FormAttachment(middle, 0);
+ fdReplace.top = new FormAttachment(wlDefaults, margin);
+ fdReplace.right = new FormAttachment(100, 0);
+ wReplaceDefaults.setLayoutData(fdReplace);
+
+ wExcludeDirs =
+ addMultiLine(
+ shell,
+ wReplaceDefaults,
+ middle,
+ margin,
+ "SparkPackageConfigDialog.ExcludeDirs.Label",
+ "SparkPackageConfigDialog.ExcludeDirs.Tooltip",
+ 60);
+ wExcludeGlobs =
+ addMultiLine(
+ shell,
+ wExcludeDirs,
+ middle,
+ margin,
+ "SparkPackageConfigDialog.ExcludeGlobs.Label",
+ "SparkPackageConfigDialog.ExcludeGlobs.Tooltip",
+ 60);
+ wIncludePaths =
+ addMultiLine(
+ shell,
+ wExcludeGlobs,
+ middle,
+ margin,
+ "SparkPackageConfigDialog.IncludePaths.Label",
+ "SparkPackageConfigDialog.IncludePaths.Tooltip",
+ 80);
+
+ // stretch last field to bottom buttons
+ FormData fdInc = (FormData) wIncludePaths.getLayoutData();
+ fdInc.bottom = new FormAttachment(wOk, -margin);
+
+ getData();
+ // Blocks until the shell is disposed (OK / Cancel / Escape / window close).
+ // Do not add listeners or run another event loop after this returns — the shell is gone.
+ BaseDialog.defaultShellHandling(shell, c -> ok(), c -> cancel());
+ }
+
+ private StyledText addMultiLine(
+ Shell shell,
+ org.eclipse.swt.widgets.Control above,
+ int middle,
+ int margin,
+ String labelKey,
+ String tooltipKey,
+ int height) {
+ Label wl = new Label(shell, SWT.RIGHT);
+ props.setLook(wl);
+ wl.setText(BaseMessages.getString(PKG, labelKey));
+ FormData fdl = new FormData();
+ fdl.left = new FormAttachment(0, 0);
+ fdl.top = new FormAttachment(above, margin);
+ fdl.right = new FormAttachment(middle, -margin);
+ wl.setLayoutData(fdl);
+ StyledText w =
+ new StyledText(shell, SWT.MULTI | SWT.LEFT | SWT.BORDER | SWT.V_SCROLL | SWT.H_SCROLL);
+ props.setLook(w);
+ w.setToolTipText(BaseMessages.getString(PKG, tooltipKey));
+ FormData fd = new FormData();
+ fd.left = new FormAttachment(middle, 0);
+ fd.top = new FormAttachment(above, margin);
+ fd.right = new FormAttachment(100, 0);
+ fd.height = height;
+ w.setLayoutData(fd);
+ return w;
+ }
+
+ private void getData() {
+ wProjectHome.setText(Const.NVL(projectHome, ""));
+ try {
+ PackageExportFilter filter = PackageExportFilter.loadFromProjectHome(projectHome);
+ wExcludeDirs.setText(String.join(Const.CR, filter.getExcludeDirs()));
+ wExcludeGlobs.setText(String.join(Const.CR, filter.getExcludeGlobs()));
+ wIncludePaths.setText(String.join(Const.CR, filter.getIncludePaths()));
+ wReplaceDefaults.setSelection(filter.isReplaceDefaultExcludeDirs());
+ } catch (Exception e) {
+ new ErrorDialog(
+ shell,
+ BaseMessages.getString(PKG, "SparkPackageConfigDialog.Error.Load.Title"),
+ BaseMessages.getString(PKG, "SparkPackageConfigDialog.Error.Load.Message"),
+ e);
+ }
+ }
+
+ private void ok() {
+ try {
+ PackageExportFilter filter =
+ new PackageExportFilter(
+ linesToList(wExcludeDirs.getText()),
+ linesToList(wExcludeGlobs.getText()),
+ linesToList(wIncludePaths.getText()),
+ wReplaceDefaults.getSelection());
+ PackageExportFilter.saveToProjectHome(projectHome, filter);
+ dispose();
+ } catch (Exception e) {
+ new ErrorDialog(
+ shell,
+ BaseMessages.getString(PKG, "SparkPackageConfigDialog.Error.Save.Title"),
+ BaseMessages.getString(PKG, "SparkPackageConfigDialog.Error.Save.Message"),
+ e);
+ }
+ }
+
+ private static List linesToList(String text) {
+ if (StringUtils.isBlank(text)) {
+ return List.of();
+ }
+ return Arrays.stream(text.split("\\R"))
+ .map(String::trim)
+ .filter(StringUtils::isNotBlank)
+ .collect(Collectors.toList());
+ }
+
+ private void cancel() {
+ dispose();
+ }
+
+ private void dispose() {
+ if (shell == null || shell.isDisposed()) {
+ return;
+ }
+ props.setScreen(new WindowProperty(shell));
+ shell.dispose();
+ }
+}
diff --git a/plugins/engines/spark/src/main/java/org/apache/hop/spark/metadata/SparkCatalog.java b/plugins/engines/spark/src/main/java/org/apache/hop/spark/metadata/SparkCatalog.java
new file mode 100644
index 00000000000..13c22a88b17
--- /dev/null
+++ b/plugins/engines/spark/src/main/java/org/apache/hop/spark/metadata/SparkCatalog.java
@@ -0,0 +1,192 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hop.spark.metadata;
+
+import java.io.Serializable;
+import lombok.Getter;
+import lombok.Setter;
+import org.apache.hop.core.gui.plugin.GuiElementType;
+import org.apache.hop.core.gui.plugin.GuiPlugin;
+import org.apache.hop.core.gui.plugin.GuiWidgetElement;
+import org.apache.hop.i18n.BaseMessages;
+import org.apache.hop.metadata.api.HopMetadata;
+import org.apache.hop.metadata.api.HopMetadataBase;
+import org.apache.hop.metadata.api.HopMetadataCategory;
+import org.apache.hop.metadata.api.HopMetadataProperty;
+import org.apache.hop.metadata.api.IHopMetadata;
+import org.apache.hop.spark.metadata.template.SparkCatalogTemplate;
+import org.apache.hop.spark.table.SparkLakeFormats;
+import org.apache.hop.ui.core.dialog.EnterSelectionDialog;
+import org.apache.hop.ui.hopgui.HopGui;
+import org.eclipse.swt.SWT;
+import org.eclipse.swt.widgets.MessageBox;
+import org.eclipse.swt.widgets.Shell;
+
+/**
+ * Reusable Spark SQL catalog configuration for lakehouse TABLE mode (Iceberg Hadoop / REST, etc.).
+ * Applied as {@code spark.sql.catalog..*} when building a native Spark session.
+ */
+@Getter
+@Setter
+@GuiPlugin
+@HopMetadata(
+ key = "SparkCatalog",
+ name = "i18n::SparkCatalog.Name",
+ description = "i18n::SparkCatalog.Description",
+ image = "spark-catalog.svg",
+ category = HopMetadataCategory.CONNECTIONS,
+ documentationUrl = "/metadata-types/spark-catalog.html")
+public class SparkCatalog extends HopMetadataBase implements Serializable, IHopMetadata {
+
+ private static final Class PKG = SparkCatalog.class;
+
+ public static final String TYPE_HADOOP = "hadoop";
+ public static final String TYPE_REST = "rest";
+ public static final String TYPE_CUSTOM = "custom";
+
+ /** Advanced — confExtra only; not packaged by default. */
+ public static final String TYPE_HIVE = "hive";
+
+ /** Advanced — confExtra only; not packaged by default. */
+ public static final String TYPE_GLUE = "glue";
+
+ private static final String PARENT = SparkCatalogEditor.GUI_WIDGETS_PARENT_ID;
+
+ /**
+ * Fill catalog fields from a named scenario (Iceberg Hadoop/REST, Hive, Glue, …). Mutates {@code
+ * object} (the live editor model); widgets are refreshed by GuiCompositeWidgets.
+ */
+ @GuiWidgetElement(
+ id = "load-spark-catalog-template",
+ order = "09990-catalog-template",
+ parentId = PARENT,
+ type = GuiElementType.BUTTON,
+ label = "i18n::SparkCatalog.LoadTemplate.Label",
+ toolTip = "i18n::SparkCatalog.LoadTemplate.ToolTip")
+ public void loadCatalogTemplate(Object object) {
+ if (!(object instanceof SparkCatalog catalog)) {
+ return;
+ }
+ Shell shell = HopGui.getInstance().getShell();
+ String[] labels = SparkCatalogTemplate.displayNames();
+ EnterSelectionDialog dialog =
+ new EnterSelectionDialog(
+ shell,
+ labels,
+ BaseMessages.getString(PKG, "SparkCatalog.LoadTemplate.Dialog.Title"),
+ BaseMessages.getString(PKG, "SparkCatalog.LoadTemplate.Dialog.Message"));
+ dialog.setAvoidQuickSearch();
+ String choice = dialog.open();
+ if (choice == null) {
+ return;
+ }
+ SparkCatalogTemplate template = SparkCatalogTemplate.fromDisplayName(choice);
+ if (template == null) {
+ return;
+ }
+ if (SparkCatalogTemplate.looksCustomized(catalog)) {
+ MessageBox box = new MessageBox(shell, SWT.YES | SWT.NO | SWT.ICON_QUESTION);
+ box.setText(BaseMessages.getString(PKG, "SparkCatalog.LoadTemplate.Confirm.Title"));
+ box.setMessage(
+ BaseMessages.getString(
+ PKG, "SparkCatalog.LoadTemplate.Confirm.Message", template.getDisplayName()));
+ if (box.open() != SWT.YES) {
+ return;
+ }
+ }
+ template.applyTo(catalog);
+ }
+
+ @GuiWidgetElement(
+ id = "10000-catalog-name",
+ parentId = PARENT,
+ type = GuiElementType.TEXT,
+ label = "i18n::SparkCatalog.CatalogName.Label",
+ toolTip = "i18n::SparkCatalog.CatalogName.Tooltip")
+ @HopMetadataProperty
+ private String catalogName;
+
+ @GuiWidgetElement(
+ id = "10010-catalog-type",
+ parentId = PARENT,
+ type = GuiElementType.COMBO,
+ label = "i18n::SparkCatalog.CatalogType.Label",
+ toolTip = "i18n::SparkCatalog.CatalogType.Tooltip",
+ comboValuesMethod = "getCatalogTypeValues")
+ @HopMetadataProperty
+ private String catalogType = TYPE_HADOOP;
+
+ @GuiWidgetElement(
+ id = "10020-implementation",
+ parentId = PARENT,
+ type = GuiElementType.TEXT,
+ label = "i18n::SparkCatalog.Implementation.Label",
+ toolTip = "i18n::SparkCatalog.Implementation.Tooltip")
+ @HopMetadataProperty
+ private String implementation;
+
+ @GuiWidgetElement(
+ id = "10030-warehouse",
+ parentId = PARENT,
+ type = GuiElementType.TEXT,
+ label = "i18n::SparkCatalog.Warehouse.Label",
+ toolTip = "i18n::SparkCatalog.Warehouse.Tooltip")
+ @HopMetadataProperty
+ private String warehouse;
+
+ @GuiWidgetElement(
+ id = "10040-uri",
+ parentId = PARENT,
+ type = GuiElementType.TEXT,
+ label = "i18n::SparkCatalog.Uri.Label",
+ toolTip = "i18n::SparkCatalog.Uri.Tooltip")
+ @HopMetadataProperty
+ private String uri;
+
+ @GuiWidgetElement(
+ id = "10050-credential",
+ parentId = PARENT,
+ type = GuiElementType.TEXT,
+ password = true,
+ label = "i18n::SparkCatalog.Credential.Label",
+ toolTip = "i18n::SparkCatalog.Credential.Tooltip")
+ @HopMetadataProperty(password = true)
+ private String credential;
+
+ @GuiWidgetElement(
+ id = "10060-conf-extra",
+ parentId = PARENT,
+ type = GuiElementType.MULTI_LINE_TEXT,
+ multiLineTextHeight = 5,
+ label = "i18n::SparkCatalog.ConfExtra.Label",
+ toolTip = "i18n::SparkCatalog.ConfExtra.Tooltip")
+ @HopMetadataProperty
+ private String confExtra;
+
+ public SparkCatalog() {
+ this.catalogType = TYPE_HADOOP;
+ this.implementation = SparkLakeFormats.ICEBERG_CATALOG;
+ }
+
+ /** Combo values for catalog type widget (GuiCompositeWidgets signature). */
+ public java.util.List getCatalogTypeValues(
+ org.apache.hop.core.logging.ILogChannel log,
+ org.apache.hop.metadata.api.IHopMetadataProvider metadataProvider) {
+ return java.util.List.of(TYPE_HADOOP, TYPE_REST, TYPE_CUSTOM, TYPE_HIVE, TYPE_GLUE);
+ }
+}
diff --git a/plugins/engines/spark/src/main/java/org/apache/hop/spark/metadata/SparkCatalogEditor.java b/plugins/engines/spark/src/main/java/org/apache/hop/spark/metadata/SparkCatalogEditor.java
new file mode 100644
index 00000000000..c15e970f9c3
--- /dev/null
+++ b/plugins/engines/spark/src/main/java/org/apache/hop/spark/metadata/SparkCatalogEditor.java
@@ -0,0 +1,156 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hop.spark.metadata;
+
+import org.apache.hop.core.Const;
+import org.apache.hop.core.gui.plugin.GuiPlugin;
+import org.apache.hop.i18n.BaseMessages;
+import org.apache.hop.ui.core.PropsUi;
+import org.apache.hop.ui.core.gui.GuiCompositeWidgets;
+import org.apache.hop.ui.core.gui.GuiCompositeWidgetsAdapter;
+import org.apache.hop.ui.core.gui.IGuiPluginCompositeButtonsListener;
+import org.apache.hop.ui.core.metadata.MetadataEditor;
+import org.apache.hop.ui.core.metadata.MetadataManager;
+import org.apache.hop.ui.hopgui.HopGui;
+import org.eclipse.swt.SWT;
+import org.eclipse.swt.layout.FormAttachment;
+import org.eclipse.swt.layout.FormData;
+import org.eclipse.swt.layout.FormLayout;
+import org.eclipse.swt.widgets.Composite;
+import org.eclipse.swt.widgets.Control;
+import org.eclipse.swt.widgets.Label;
+import org.eclipse.swt.widgets.Text;
+
+@GuiPlugin(description = "Editor for Spark catalog metadata")
+public class SparkCatalogEditor extends MetadataEditor {
+
+ private static final Class PKG = SparkCatalog.class;
+
+ public static final String GUI_WIDGETS_PARENT_ID = "SparkCatalogEditor-GuiWidgetsParent";
+
+ private Text wName;
+ private Composite wWidgetsComposite;
+ private GuiCompositeWidgets guiCompositeWidgets;
+
+ public SparkCatalogEditor(
+ HopGui hopGui, MetadataManager manager, SparkCatalog metadata) {
+ super(hopGui, manager, metadata);
+ }
+
+ @Override
+ public void createControl(Composite parent) {
+ PropsUi props = PropsUi.getInstance();
+ int middle = props.getMiddlePct();
+ int margin = PropsUi.getMargin() + 2;
+
+ Label wIcon = new Label(parent, SWT.RIGHT);
+ wIcon.setImage(getImage());
+ FormData fdlIcon = new FormData();
+ fdlIcon.top = new FormAttachment(0, 0);
+ fdlIcon.right = new FormAttachment(100, 0);
+ wIcon.setLayoutData(fdlIcon);
+ PropsUi.setLook(wIcon);
+
+ Label wlName = new Label(parent, SWT.RIGHT);
+ PropsUi.setLook(wlName);
+ wlName.setText(BaseMessages.getString(PKG, "SparkCatalog.Name.Label"));
+ FormData fdlName = new FormData();
+ fdlName.top = new FormAttachment(0, margin);
+ fdlName.left = new FormAttachment(0, 0);
+ fdlName.right = new FormAttachment(middle, -margin);
+ wlName.setLayoutData(fdlName);
+ wName = new Text(parent, SWT.SINGLE | SWT.LEFT | SWT.BORDER);
+ PropsUi.setLook(wName);
+ FormData fdName = new FormData();
+ fdName.top = new FormAttachment(wlName, 0, SWT.CENTER);
+ fdName.left = new FormAttachment(middle, 0);
+ fdName.right = new FormAttachment(wIcon, -margin);
+ wName.setLayoutData(fdName);
+ Control lastControl = wName;
+
+ wWidgetsComposite = new Composite(parent, SWT.NONE);
+ PropsUi.setLook(wWidgetsComposite);
+ wWidgetsComposite.setLayout(new FormLayout());
+ FormData fdWidgetsComposite = new FormData();
+ fdWidgetsComposite.top = new FormAttachment(lastControl, margin);
+ fdWidgetsComposite.left = new FormAttachment(0, 0);
+ fdWidgetsComposite.right = new FormAttachment(100, 0);
+ fdWidgetsComposite.bottom = new FormAttachment(100, 0);
+ wWidgetsComposite.setLayoutData(fdWidgetsComposite);
+
+ guiCompositeWidgets = new GuiCompositeWidgets(manager.getVariables());
+ guiCompositeWidgets.createCompositeWidgets(
+ metadata, null, wWidgetsComposite, GUI_WIDGETS_PARENT_ID, lastControl);
+ guiCompositeWidgets.setWidgetsListener(
+ new GuiCompositeWidgetsAdapter() {
+ @Override
+ public void widgetModified(
+ GuiCompositeWidgets compositeWidgets, Control changedWidget, String widgetId) {
+ setChanged();
+ }
+ });
+ // Flush form → model before button methods; re-bind model → form after mutations
+ // (e.g. Load catalog template). Do not touch wName (Hop metadata name stays independent).
+ guiCompositeWidgets.setCompositeButtonsListener(
+ new IGuiPluginCompositeButtonsListener() {
+ @Override
+ public void buttonPressed(Object sourceObject) {
+ Object model = sourceObject != null ? sourceObject : metadata;
+ guiCompositeWidgets.getWidgetsContents(model, GUI_WIDGETS_PARENT_ID);
+ }
+
+ @Override
+ public void afterButtonPressed(Object sourceObject) {
+ Object model = sourceObject != null ? sourceObject : metadata;
+ if (guiCompositeWidgets != null
+ && wWidgetsComposite != null
+ && !wWidgetsComposite.isDisposed()) {
+ guiCompositeWidgets.setWidgetsContents(
+ model, wWidgetsComposite, GUI_WIDGETS_PARENT_ID);
+ wWidgetsComposite.layout(true, true);
+ }
+ setChanged();
+ }
+ });
+
+ setWidgetsContent();
+ resetChanged();
+ wName.addModifyListener(e -> setChanged());
+ }
+
+ @Override
+ public void setWidgetsContent() {
+ SparkCatalog meta = this.getMetadata();
+ wName.setText(Const.NVL(meta.getName(), ""));
+ guiCompositeWidgets.setWidgetsContents(metadata, wWidgetsComposite, GUI_WIDGETS_PARENT_ID);
+ }
+
+ @Override
+ public void getWidgetsContent(SparkCatalog meta) {
+ meta.setName(wName.getText());
+ guiCompositeWidgets.getWidgetsContents(metadata, GUI_WIDGETS_PARENT_ID);
+ }
+
+ @Override
+ public boolean setFocus() {
+ if (wName == null || wName.isDisposed()) {
+ return false;
+ }
+ return wName.setFocus();
+ }
+}
diff --git a/plugins/engines/spark/src/main/java/org/apache/hop/spark/metadata/template/SparkCatalogTemplate.java b/plugins/engines/spark/src/main/java/org/apache/hop/spark/metadata/template/SparkCatalogTemplate.java
new file mode 100644
index 00000000000..adb989395d4
--- /dev/null
+++ b/plugins/engines/spark/src/main/java/org/apache/hop/spark/metadata/template/SparkCatalogTemplate.java
@@ -0,0 +1,258 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hop.spark.metadata.template;
+
+import java.util.Arrays;
+import java.util.Objects;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.hop.spark.metadata.SparkCatalog;
+import org.apache.hop.spark.table.SparkLakeFormats;
+
+/**
+ * Named presets for {@link SparkCatalog} fields. Pure data — no SWT. Apply via {@link
+ * #applyTo(SparkCatalog)}.
+ *
+ *
Advanced presets include a commented {@code # docs: …} line in conf extra (ignored by {@code
+ * SparkCatalogApplier}) so operators can open vendor documentation without leaving Hop.
+ */
+public enum SparkCatalogTemplate {
+ ICEBERG_HADOOP_LOCAL(
+ "Iceberg Hadoop (local)",
+ "Named Iceberg Hadoop catalog on a local warehouse path. Use TABLE mode as lake.db.table.",
+ "lake",
+ SparkCatalog.TYPE_HADOOP,
+ SparkLakeFormats.ICEBERG_CATALOG,
+ "file:///tmp/hop-warehouse",
+ "",
+ confWithDocs(Docs.ICEBERG_SPARK, "")),
+ ICEBERG_HADOOP_OBJECT_STORE(
+ "Iceberg Hadoop (object store)",
+ "Iceberg Hadoop warehouse on object storage (replace bucket). Credentials via env/Hadoop conf.",
+ "lake",
+ SparkCatalog.TYPE_HADOOP,
+ SparkLakeFormats.ICEBERG_CATALOG,
+ "s3a://bucket/warehouse",
+ "",
+ confWithDocs(Docs.ICEBERG_AWS, "io-impl=org.apache.iceberg.aws.s3.S3FileIO")),
+ ICEBERG_REST(
+ "Iceberg REST",
+ "Iceberg REST catalog. Replace the URI with your catalog service endpoint.",
+ "lake",
+ SparkCatalog.TYPE_REST,
+ SparkLakeFormats.ICEBERG_CATALOG,
+ "",
+ "https://catalog.example.com/v1",
+ confWithDocs(Docs.ICEBERG_REST, "")),
+ ICEBERG_REST_AUTH(
+ "Iceberg REST (authenticated)",
+ "Iceberg REST with token auth. Put the secret in Credential (maps to catalog .token).",
+ "lake",
+ SparkCatalog.TYPE_REST,
+ SparkLakeFormats.ICEBERG_CATALOG,
+ "",
+ "https://catalog.example.com/v1",
+ confWithDocs(
+ Docs.ICEBERG_REST,
+ "# Bearer/token: use the Credential field (applied as spark.sql.catalog..token)")),
+ HIVE_METASTORE(
+ "Hive Metastore (advanced)",
+ "Iceberg via Hive Metastore. Requires Hive/Iceberg deps on the cluster (not packaged by default).",
+ "hive",
+ SparkCatalog.TYPE_HIVE,
+ SparkLakeFormats.ICEBERG_CATALOG,
+ "",
+ "",
+ confWithDocs(Docs.ICEBERG_HIVE, "uri=thrift://hive-metastore:9083")),
+ AWS_GLUE(
+ "AWS Glue (advanced)",
+ "Iceberg via AWS Glue. Requires AWS/Iceberg deps and IAM on the cluster (not packaged by default).",
+ "glue",
+ SparkCatalog.TYPE_GLUE,
+ SparkLakeFormats.ICEBERG_CATALOG,
+ "s3a://bucket/warehouse",
+ "",
+ confWithDocs(
+ Docs.ICEBERG_AWS,
+ "warehouse=s3a://bucket/warehouse\nio-impl=org.apache.iceberg.aws.s3.S3FileIO")),
+ NESSIE(
+ "Nessie (advanced)",
+ "Iceberg Nessie catalog. Requires Nessie/Iceberg deps; replace URI, ref, and warehouse.",
+ "nessie",
+ SparkCatalog.TYPE_CUSTOM,
+ SparkLakeFormats.ICEBERG_CATALOG,
+ "file:///tmp/nessie-warehouse",
+ "http://localhost:19120/api/v1",
+ confWithDocs(
+ Docs.NESSIE_SPARK,
+ "catalog-impl=org.apache.iceberg.nessie.NessieCatalog\n"
+ + "uri=http://localhost:19120/api/v1\n"
+ + "ref=main\n"
+ + "warehouse=file:///tmp/nessie-warehouse")),
+ DATABRICKS_UNITY(
+ "Databricks Unity Catalog (advanced)",
+ "Skeleton for Unity Catalog on Databricks. Prefer workspace-managed conf; adjust URI/auth for your env.",
+ "unity",
+ SparkCatalog.TYPE_CUSTOM,
+ SparkLakeFormats.ICEBERG_CATALOG,
+ "",
+ "",
+ confWithDocs(
+ Docs.DATABRICKS_UNITY,
+ "# Typically configured on the Databricks cluster or spark-submit.\n"
+ + "# Example short keys (uncomment and replace if your runtime expects them):\n"
+ + "# uri=https:///api/2.1/unity-catalog\n"
+ + "# warehouse=s3:///unity")),
+ DELTA_NAMED_CATALOG(
+ "Delta named catalog (advanced)",
+ "Named DeltaCatalog (not spark_catalog). Prefer spark_catalog for Delta when coexisting with Iceberg.",
+ "delta_cat",
+ SparkCatalog.TYPE_CUSTOM,
+ SparkLakeFormats.DELTA_CATALOG,
+ "",
+ "",
+ confWithDocs(
+ Docs.DELTA,
+ "# Prefer spark.sql.catalog.spark_catalog=DeltaCatalog for default Delta tables.\n"
+ + "# Use a distinct catalog name only when you must register a second Delta catalog."));
+
+ /**
+ * Documentation URLs for conf-extra {@code # docs:} lines. Nested interface so enum constants can
+ * reference them without illegal forward references.
+ */
+ public interface Docs {
+ String ICEBERG_SPARK = "https://iceberg.apache.org/docs/latest/spark-configuration/";
+ String ICEBERG_REST = "https://iceberg.apache.org/docs/latest/rest-catalog/";
+ String ICEBERG_AWS = "https://iceberg.apache.org/docs/latest/aws/";
+ String ICEBERG_HIVE = "https://iceberg.apache.org/docs/latest/hive/";
+ String NESSIE_SPARK = "https://projectnessie.org/iceberg/spark/";
+ String DATABRICKS_UNITY =
+ "https://docs.databricks.com/en/data-governance/unity-catalog/index.html";
+ String DELTA = "https://docs.delta.io/latest/index.html";
+ }
+
+ private final String displayName;
+ private final String description;
+ private final String catalogName;
+ private final String catalogType;
+ private final String implementation;
+ private final String warehouse;
+ private final String uri;
+ private final String confExtra;
+
+ SparkCatalogTemplate(
+ String displayName,
+ String description,
+ String catalogName,
+ String catalogType,
+ String implementation,
+ String warehouse,
+ String uri,
+ String confExtra) {
+ this.displayName = displayName;
+ this.description = description;
+ this.catalogName = catalogName;
+ this.catalogType = catalogType;
+ this.implementation = implementation;
+ this.warehouse = warehouse;
+ this.uri = uri;
+ this.confExtra = confExtra;
+ }
+
+ /**
+ * Build conf-extra text with a leading commented documentation URL (ignored by the applier) and
+ * optional body lines.
+ */
+ static String confWithDocs(String docsUrl, String body) {
+ String header = "# docs: " + docsUrl;
+ if (StringUtils.isBlank(body)) {
+ return header;
+ }
+ return header + "\n" + body;
+ }
+
+ public String getDisplayName() {
+ return displayName;
+ }
+
+ public String getDescription() {
+ return description;
+ }
+
+ /** Labels for selection dialogs (same order as {@link #values()}). */
+ public static String[] displayNames() {
+ return Arrays.stream(values()).map(SparkCatalogTemplate::getDisplayName).toArray(String[]::new);
+ }
+
+ public static SparkCatalogTemplate fromDisplayName(String name) {
+ if (name == null) {
+ return null;
+ }
+ for (SparkCatalogTemplate t : values()) {
+ if (t.displayName.equals(name)) {
+ return t;
+ }
+ }
+ return null;
+ }
+
+ /**
+ * Apply this template onto {@code catalog}. Does not change the Hop metadata object name; leaves
+ * credential empty (never put secrets in presets).
+ */
+ public void applyTo(SparkCatalog catalog) {
+ Objects.requireNonNull(catalog, "catalog");
+ catalog.setCatalogName(catalogName);
+ catalog.setCatalogType(catalogType);
+ catalog.setImplementation(implementation);
+ catalog.setWarehouse(warehouse);
+ catalog.setUri(uri);
+ catalog.setCredential("");
+ catalog.setConfExtra(confExtra);
+ }
+
+ /**
+ * True if the catalog looks customized relative to a fresh {@link SparkCatalog} default (used for
+ * overwrite confirmation).
+ */
+ public static boolean looksCustomized(SparkCatalog catalog) {
+ if (catalog == null) {
+ return false;
+ }
+ SparkCatalog def = new SparkCatalog();
+ if (StringUtils.isNotEmpty(catalog.getCatalogName())) {
+ return true;
+ }
+ if (!Objects.equals(nz(catalog.getCatalogType()), nz(def.getCatalogType()))) {
+ return true;
+ }
+ if (!Objects.equals(nz(catalog.getImplementation()), nz(def.getImplementation()))) {
+ return true;
+ }
+ if (StringUtils.isNotEmpty(catalog.getWarehouse())
+ || StringUtils.isNotEmpty(catalog.getUri())
+ || StringUtils.isNotEmpty(catalog.getCredential())
+ || StringUtils.isNotEmpty(catalog.getConfExtra())) {
+ return true;
+ }
+ return false;
+ }
+
+ private static String nz(String s) {
+ return s == null ? "" : s;
+ }
+}
diff --git a/plugins/engines/spark/src/main/java/org/apache/hop/spark/pipeline/HopPipelineMetaToSparkConverter.java b/plugins/engines/spark/src/main/java/org/apache/hop/spark/pipeline/HopPipelineMetaToSparkConverter.java
new file mode 100644
index 00000000000..8d16d7e3044
--- /dev/null
+++ b/plugins/engines/spark/src/main/java/org/apache/hop/spark/pipeline/HopPipelineMetaToSparkConverter.java
@@ -0,0 +1,526 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hop.spark.pipeline;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import org.apache.hop.core.exception.HopException;
+import org.apache.hop.core.exception.HopRowException;
+import org.apache.hop.core.logging.ILogChannel;
+import org.apache.hop.core.logging.LogChannel;
+import org.apache.hop.core.metadata.SerializableMetadataProvider;
+import org.apache.hop.core.row.IRowMeta;
+import org.apache.hop.core.variables.IVariables;
+import org.apache.hop.metadata.api.IHopMetadataProvider;
+import org.apache.hop.pipeline.PipelineHopMeta;
+import org.apache.hop.pipeline.PipelineMeta;
+import org.apache.hop.pipeline.config.PipelineRunConfiguration;
+import org.apache.hop.pipeline.transform.BaseTransform;
+import org.apache.hop.pipeline.transform.TransformMeta;
+import org.apache.hop.spark.core.HopSparkUtil;
+import org.apache.hop.spark.core.SparkExecutionDataAccumulator;
+import org.apache.hop.spark.core.SparkTransformMetricsAccumulator;
+import org.apache.hop.spark.engines.ISparkPipelineEngineRunConfiguration;
+import org.apache.hop.spark.pipeline.handler.SparkBaseTransformHandler;
+import org.apache.hop.spark.pipeline.handler.SparkFileInputHandler;
+import org.apache.hop.spark.pipeline.handler.SparkFileOutputHandler;
+import org.apache.hop.spark.pipeline.handler.SparkGenericTransformHandler;
+import org.apache.hop.spark.pipeline.handler.SparkLakeTableInputHandler;
+import org.apache.hop.spark.pipeline.handler.SparkLakeTableMaintenanceHandler;
+import org.apache.hop.spark.pipeline.handler.SparkLakeTableMergeHandler;
+import org.apache.hop.spark.pipeline.handler.SparkLakeTableOutputHandler;
+import org.apache.hop.spark.pipeline.handler.SparkMemoryGroupByHandler;
+import org.apache.hop.spark.pipeline.handler.SparkMergeJoinHandler;
+import org.apache.hop.spark.pipeline.handler.SparkSortRowsHandler;
+import org.apache.hop.spark.pipeline.handler.SparkUniqueRowsHandler;
+import org.apache.hop.spark.util.SparkConst;
+import org.apache.spark.sql.Dataset;
+import org.apache.spark.sql.Row;
+import org.apache.spark.sql.SparkSession;
+
+/**
+ * Converts a Hop {@link PipelineMeta} into a Spark {@link Dataset} lineage. Stateless transforms
+ * use mapPartitions; shuffle-aware transforms use native Dataset handlers.
+ */
+public class HopPipelineMetaToSparkConverter {
+
+ /**
+ * Plugin ids with dedicated native handlers. Keep in lockstep with {@link
+ * #addDefaultTransformHandlers()} and {@link
+ * org.apache.hop.spark.engines.SparkPipelineEngine#supports}.
+ */
+ public static final Set EXPLICIT_HANDLER_PLUGIN_IDS =
+ Set.of(
+ SparkConst.MEMORY_GROUP_BY_PLUGIN_ID,
+ SparkConst.MERGE_JOIN_PLUGIN_ID,
+ SparkConst.UNIQUE_ROWS_PLUGIN_ID,
+ SparkConst.SORT_ROWS_PLUGIN_ID,
+ SparkConst.SPARK_FILE_INPUT_PLUGIN_ID,
+ SparkConst.SPARK_FILE_OUTPUT_PLUGIN_ID,
+ SparkConst.SPARK_LAKE_TABLE_INPUT_PLUGIN_ID,
+ SparkConst.SPARK_LAKE_TABLE_OUTPUT_PLUGIN_ID,
+ SparkConst.SPARK_LAKE_TABLE_MERGE_PLUGIN_ID,
+ SparkConst.SPARK_LAKE_TABLE_MAINTENANCE_PLUGIN_ID);
+
+ /**
+ * Plugin ids that must not run as partition-local Hop mini-pipelines. Keep in lockstep with
+ * {@link org.apache.hop.spark.engines.SparkPipelineEngine#supports}.
+ */
+ public static final Map HARD_BANNED_PLUGIN_IDS =
+ Map.of(
+ SparkConst.GROUP_BY_PLUGIN_ID,
+ "Group By is not supported on the native Spark engine. Use Memory Group By (native Spark shuffle) instead, or run on Local/Beam.");
+
+ private final IVariables variables;
+ private final PipelineMeta pipelineMeta;
+ private final SerializableMetadataProvider metadataProvider;
+ private final String metaStoreJson;
+ private final String runConfigName;
+ private final ISparkPipelineEngineRunConfiguration sparkRunConfiguration;
+ private final Map transformHandlers;
+ private final SparkGenericTransformHandler genericTransformHandler;
+ private SparkTransformMetricsAccumulator metricsAccumulator;
+ private SparkExecutionDataAccumulator sampleDataAccumulator;
+ private String parentLogChannelId;
+ private String dataSamplersJson;
+
+ public HopPipelineMetaToSparkConverter(
+ IVariables variables,
+ PipelineMeta pipelineMeta,
+ IHopMetadataProvider metadataProvider,
+ String runConfigName)
+ throws HopException {
+ this.variables = variables;
+ this.pipelineMeta = pipelineMeta;
+ this.metadataProvider = new SerializableMetadataProvider(metadataProvider);
+ this.metaStoreJson = this.metadataProvider.toJson();
+ this.runConfigName = runConfigName;
+
+ PipelineRunConfiguration runConfiguration =
+ metadataProvider.getSerializer(PipelineRunConfiguration.class).load(runConfigName);
+ if (runConfiguration == null) {
+ throw new HopException("Unable to load pipeline run configuration '" + runConfigName + "'");
+ }
+ if (!(runConfiguration.getEngineRunConfiguration()
+ instanceof ISparkPipelineEngineRunConfiguration sparkConfig)) {
+ throw new HopException(
+ "Run configuration '"
+ + runConfigName
+ + "' is not a native Spark pipeline engine configuration");
+ }
+ this.sparkRunConfiguration = sparkConfig;
+ this.transformHandlers = new HashMap<>();
+ this.genericTransformHandler = new SparkGenericTransformHandler();
+ addDefaultTransformHandlers();
+ }
+
+ /**
+ * Constructor used when the run configuration object is already available (unit tests / engine).
+ */
+ public HopPipelineMetaToSparkConverter(
+ IVariables variables,
+ PipelineMeta pipelineMeta,
+ IHopMetadataProvider metadataProvider,
+ String runConfigName,
+ ISparkPipelineEngineRunConfiguration sparkRunConfiguration)
+ throws HopException {
+ this.variables = variables;
+ this.pipelineMeta = pipelineMeta;
+ this.metadataProvider = new SerializableMetadataProvider(metadataProvider);
+ this.metaStoreJson = this.metadataProvider.toJson();
+ this.runConfigName = runConfigName;
+ this.sparkRunConfiguration = sparkRunConfiguration;
+ this.transformHandlers = new HashMap<>();
+ this.genericTransformHandler = new SparkGenericTransformHandler();
+ addDefaultTransformHandlers();
+ }
+
+ protected void addDefaultTransformHandlers() {
+ transformHandlers.put(SparkConst.MEMORY_GROUP_BY_PLUGIN_ID, new SparkMemoryGroupByHandler());
+ transformHandlers.put(SparkConst.MERGE_JOIN_PLUGIN_ID, new SparkMergeJoinHandler());
+ transformHandlers.put(SparkConst.UNIQUE_ROWS_PLUGIN_ID, new SparkUniqueRowsHandler());
+ transformHandlers.put(SparkConst.SORT_ROWS_PLUGIN_ID, new SparkSortRowsHandler());
+ transformHandlers.put(SparkConst.SPARK_FILE_INPUT_PLUGIN_ID, new SparkFileInputHandler());
+ transformHandlers.put(SparkConst.SPARK_FILE_OUTPUT_PLUGIN_ID, new SparkFileOutputHandler());
+ transformHandlers.put(
+ SparkConst.SPARK_LAKE_TABLE_INPUT_PLUGIN_ID, new SparkLakeTableInputHandler());
+ transformHandlers.put(
+ SparkConst.SPARK_LAKE_TABLE_OUTPUT_PLUGIN_ID, new SparkLakeTableOutputHandler());
+ transformHandlers.put(
+ SparkConst.SPARK_LAKE_TABLE_MERGE_PLUGIN_ID, new SparkLakeTableMergeHandler());
+ transformHandlers.put(
+ SparkConst.SPARK_LAKE_TABLE_MAINTENANCE_PLUGIN_ID, new SparkLakeTableMaintenanceHandler());
+ }
+
+ public void validatePipeline() throws HopException {
+ for (TransformMeta transformMeta : pipelineMeta.getTransforms()) {
+ validateTransformSparkUsage(transformMeta.getTransformPluginId(), transformMeta.getName());
+ }
+ }
+
+ public static void validateTransformSparkUsage(String pluginId, String transformName)
+ throws HopException {
+ if (pluginId == null) {
+ return;
+ }
+ String reason = HARD_BANNED_PLUGIN_IDS.get(pluginId);
+ if (reason != null) {
+ throw new HopException("Transform '" + transformName + "' (" + pluginId + "): " + reason);
+ }
+ }
+
+ /**
+ * Build the Dataset graph and return the sink Dataset (last transform with no successors, or the
+ * last in topological order). Calling an action on the result materializes the pipeline.
+ */
+ public Dataset createDataset(SparkSession spark) throws HopException {
+ try {
+ ILogChannel log = LogChannel.GENERAL;
+ validatePipeline();
+
+ Map> transformDatasetMap = new HashMap<>();
+ List transforms = getSortedTransformsList();
+
+ for (TransformMeta transformMeta : transforms) {
+ String pluginId = transformMeta.getTransformPluginId();
+ ISparkPipelineTransformHandler handler =
+ transformHandlers.getOrDefault(pluginId, genericTransformHandler);
+ boolean nativeHandler = transformHandlers.containsKey(pluginId);
+
+ List previousTransforms =
+ pipelineMeta.findPreviousTransforms(transformMeta, false);
+
+ Dataset input = null;
+ IRowMeta rowMeta;
+ if (previousTransforms.isEmpty()) {
+ rowMeta = new org.apache.hop.core.row.RowMeta();
+ } else if (nativeHandler && previousTransforms.size() > 1) {
+ // Merge Join (etc.) resolves named inputs itself from transformDatasetMap
+ TransformMeta previous = previousTransforms.get(0);
+ input = lookupPreviousDataset(transformDatasetMap, previous, transformMeta, log);
+ rowMeta =
+ input != null
+ ? pipelineMeta.getTransformFields(variables, previous, transformMeta, null)
+ : new org.apache.hop.core.row.RowMeta();
+ } else {
+ // One or more main predecessors: resolve each Dataset (incl. target-stream keys) and
+ // union when layouts match (Hop multi-hop merge semantics).
+ ResolvedInputs resolved =
+ resolveAndUnionInputs(
+ log,
+ variables,
+ pipelineMeta,
+ transformMeta,
+ previousTransforms,
+ transformDatasetMap);
+ input = resolved.dataset();
+ rowMeta = resolved.rowMeta();
+ if (input == null) {
+ throw new HopException(
+ "Previous Dataset(s) for transform '"
+ + transformMeta.getName()
+ + "' could not be found (previous="
+ + previousTransforms.stream().map(TransformMeta::getName).toList()
+ + "). Check that hops into this transform are enabled.");
+ }
+ }
+
+ handler.handleTransform(
+ log,
+ variables,
+ runConfigName,
+ sparkRunConfiguration,
+ metadataProvider,
+ metaStoreJson,
+ pipelineMeta,
+ transformMeta,
+ transformDatasetMap,
+ spark,
+ rowMeta,
+ previousTransforms,
+ input);
+ }
+
+ // Prefer a leaf transform as the result
+ for (int i = transforms.size() - 1; i >= 0; i--) {
+ TransformMeta candidate = transforms.get(i);
+ if (pipelineMeta.findNextTransforms(candidate).isEmpty()) {
+ Dataset leaf = transformDatasetMap.get(candidate.getName());
+ if (leaf != null) {
+ return leaf;
+ }
+ }
+ }
+ if (transforms.isEmpty()) {
+ throw new HopException("Pipeline has no transforms to execute on Spark");
+ }
+ Dataset last = transformDatasetMap.get(transforms.get(transforms.size() - 1).getName());
+ if (last == null) {
+ throw new HopException("No Spark Dataset was produced for the pipeline");
+ }
+ return last;
+ } catch (HopException e) {
+ throw e;
+ } catch (Exception e) {
+ throw new HopException("Error converting Hop pipeline to Spark", e);
+ }
+ }
+
+ /**
+ * Resolve Datasets for all main previous transforms (preferring target-stream keys), verify that
+ * Hop row layouts match, and {@code union} them. Matches Beam Flatten / Hop multi-hop merge.
+ */
+ static ResolvedInputs resolveAndUnionInputs(
+ ILogChannel log,
+ IVariables variables,
+ PipelineMeta pipelineMeta,
+ TransformMeta transformMeta,
+ List previousTransforms,
+ Map> transformDatasetMap)
+ throws HopException {
+ if (previousTransforms == null || previousTransforms.isEmpty()) {
+ return new ResolvedInputs(null, new org.apache.hop.core.row.RowMeta());
+ }
+
+ List> datasets = new ArrayList<>();
+ IRowMeta referenceRowMeta = null;
+ List sourceNames = new ArrayList<>();
+
+ for (TransformMeta previous : previousTransforms) {
+ Dataset ds = lookupPreviousDataset(transformDatasetMap, previous, transformMeta, log);
+ if (ds == null) {
+ throw new HopException(
+ "Previous Dataset for transform '"
+ + previous.getName()
+ + "' could not be found when handling '"
+ + transformMeta.getName()
+ + "'");
+ }
+ // Pass current as targetTransform so hops like Workflow Executor → results keep
+ // execution-result fields (getFields clears the row when nextTransform is null).
+ IRowMeta prevRowMeta =
+ pipelineMeta.getTransformFields(variables, previous, transformMeta, null);
+ if (referenceRowMeta == null) {
+ referenceRowMeta = prevRowMeta;
+ } else if (!transformMeta.getTransform().excludeFromRowLayoutVerification()) {
+ try {
+ BaseTransform.safeModeChecking(referenceRowMeta, prevRowMeta);
+ } catch (HopRowException e) {
+ throw new HopException(
+ "Cannot combine data into transform '"
+ + transformMeta.getName()
+ + "': previous transforms do not share the same row layout. Hop requires"
+ + " identical field names, order, and types when multiple hops feed one"
+ + " transform. Mismatch involving '"
+ + previous.getName()
+ + "': "
+ + e.getMessage(),
+ e);
+ }
+ }
+ datasets.add(ds);
+ sourceNames.add(previous.getName());
+ }
+
+ Dataset unioned = datasets.get(0);
+ for (int i = 1; i < datasets.size(); i++) {
+ // unionByName is more resilient if Spark column order drifts; layouts already checked
+ unioned = unioned.unionByName(datasets.get(i));
+ }
+ if (datasets.size() > 1) {
+ log.logBasic(
+ "Combined "
+ + datasets.size()
+ + " previous Dataset(s) into transform '"
+ + transformMeta.getName()
+ + "': "
+ + sourceNames);
+ }
+ return new ResolvedInputs(
+ unioned,
+ referenceRowMeta != null ? referenceRowMeta : new org.apache.hop.core.row.RowMeta());
+ }
+
+ /**
+ * Prefer a target-stream Dataset when {@code previous} routed to {@code current} (Filter/Switch);
+ * otherwise use the previous transform's main Dataset.
+ */
+ static Dataset lookupPreviousDataset(
+ Map> transformDatasetMap,
+ TransformMeta previous,
+ TransformMeta current,
+ ILogChannel log) {
+ String targetKey = HopSparkUtil.createTargetTupleId(previous.getName(), current.getName());
+ Dataset input = transformDatasetMap.get(targetKey);
+ if (input != null) {
+ if (log != null) {
+ log.logBasic(
+ "Transform '"
+ + current.getName()
+ + "' reading from previous target stream key: "
+ + targetKey);
+ }
+ return input;
+ }
+ return transformDatasetMap.get(previous.getName());
+ }
+
+ /** Result of resolving one or more previous Datasets for a transform. */
+ record ResolvedInputs(Dataset dataset, IRowMeta rowMeta) {}
+
+ /**
+ * Transforms that participate in at least one enabled hop (Beam-style active
+ * graph). Fully disconnected transforms and transforms only linked by disabled hops are skipped
+ * so sinks without an active input are not executed.
+ *
+ *
Unlike {@link PipelineMeta#getPipelineHopTransforms(boolean)}, this does not re-add
+ * unused canvas transforms (those exist for painting only).
+ */
+ public static List collectActiveTransforms(PipelineMeta pipelineMeta) {
+ if (pipelineMeta == null) {
+ return List.of();
+ }
+ Set active = new LinkedHashSet<>();
+ List hops = pipelineMeta.getPipelineHops();
+ if (hops != null) {
+ for (PipelineHopMeta hop : hops) {
+ if (hop == null || !hop.isEnabled()) {
+ continue;
+ }
+ if (hop.getFromTransform() != null) {
+ active.add(hop.getFromTransform());
+ }
+ if (hop.getToTransform() != null) {
+ active.add(hop.getToTransform());
+ }
+ }
+ }
+ // Single-transform pipeline with no hops (rare design canvas)
+ if (active.isEmpty() && pipelineMeta.nrTransforms() == 1) {
+ active.add(pipelineMeta.getTransform(0));
+ }
+ return new ArrayList<>(active);
+ }
+
+ /** Topological order of active transforms (Kahn-style via repeated previous-set checks). */
+ protected List getSortedTransformsList() throws HopException {
+ List transforms = collectActiveTransforms(pipelineMeta);
+ if (transforms.isEmpty() && pipelineMeta.nrTransforms() > 0) {
+ throw new HopException(
+ "No active transforms to execute on Spark: enable at least one hop between transforms"
+ + " (disabled hops and disconnected transforms are skipped).");
+ }
+ List sorted = new ArrayList<>();
+ Set handled = new HashSet<>();
+ Set activeSet = new HashSet<>(transforms);
+
+ while (sorted.size() < transforms.size()) {
+ boolean progress = false;
+ for (TransformMeta transformMeta : transforms) {
+ if (handled.contains(transformMeta)) {
+ continue;
+ }
+ // Include informational predecessors so Stream Lookup sources are built first
+ List previous = pipelineMeta.findPreviousTransforms(transformMeta, true);
+ boolean allPreviousHandled = true;
+ for (TransformMeta prev : previous) {
+ // Only require predecessors that are themselves active in the enabled graph
+ if (activeSet.contains(prev) && !handled.contains(prev)) {
+ allPreviousHandled = false;
+ break;
+ }
+ }
+ if (allPreviousHandled) {
+ sorted.add(transformMeta);
+ handled.add(transformMeta);
+ progress = true;
+ }
+ }
+ if (!progress) {
+ // Cycle or unresolved graph — fall back to original order for remaining
+ for (TransformMeta transformMeta : transforms) {
+ if (!handled.contains(transformMeta)) {
+ sorted.add(transformMeta);
+ handled.add(transformMeta);
+ }
+ }
+ }
+ }
+ return sorted;
+ }
+
+ public ISparkPipelineEngineRunConfiguration getSparkRunConfiguration() {
+ return sparkRunConfiguration;
+ }
+
+ /**
+ * Registers the driver-side metrics accumulator used by generic mapPartitions transforms and
+ * native Dataset handlers. Must be called after the accumulator is registered with the Spark
+ * context.
+ */
+ public void setMetricsAccumulator(SparkTransformMetricsAccumulator metricsAccumulator) {
+ this.metricsAccumulator = metricsAccumulator;
+ this.genericTransformHandler.setMetricsAccumulator(metricsAccumulator);
+ for (ISparkPipelineTransformHandler handler : transformHandlers.values()) {
+ if (handler instanceof SparkBaseTransformHandler baseHandler) {
+ baseHandler.setMetricsAccumulator(metricsAccumulator);
+ }
+ }
+ }
+
+ public SparkTransformMetricsAccumulator getMetricsAccumulator() {
+ return metricsAccumulator;
+ }
+
+ /**
+ * Accumulator that carries sample {@code ExecutionData} JSON from executors to the driver for
+ * registration at the driver's execution info location.
+ */
+ public void setSampleDataAccumulator(SparkExecutionDataAccumulator sampleDataAccumulator) {
+ this.sampleDataAccumulator = sampleDataAccumulator;
+ this.genericTransformHandler.setSampleDataAccumulator(sampleDataAccumulator);
+ }
+
+ public SparkExecutionDataAccumulator getSampleDataAccumulator() {
+ return sampleDataAccumulator;
+ }
+
+ /**
+ * Context for executor-side execution data sampling. Call after the engine log channel exists.
+ * Samples are shipped to the driver via {@link #setSampleDataAccumulator}.
+ */
+ public void setExecutionSamplingContext(String parentLogChannelId, String dataSamplersJson) {
+ this.parentLogChannelId = parentLogChannelId;
+ this.dataSamplersJson = dataSamplersJson;
+ this.genericTransformHandler.setExecutionSamplingContext(
+ runConfigName, parentLogChannelId, dataSamplersJson);
+ }
+
+ public String getParentLogChannelId() {
+ return parentLogChannelId;
+ }
+
+ public String getDataSamplersJson() {
+ return dataSamplersJson;
+ }
+}
diff --git a/plugins/engines/spark/src/main/java/org/apache/hop/spark/pipeline/ISparkPipelineTransformHandler.java b/plugins/engines/spark/src/main/java/org/apache/hop/spark/pipeline/ISparkPipelineTransformHandler.java
new file mode 100644
index 00000000000..83d493d55f7
--- /dev/null
+++ b/plugins/engines/spark/src/main/java/org/apache/hop/spark/pipeline/ISparkPipelineTransformHandler.java
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hop.spark.pipeline;
+
+import java.util.List;
+import java.util.Map;
+import org.apache.hop.core.exception.HopException;
+import org.apache.hop.core.logging.ILogChannel;
+import org.apache.hop.core.row.IRowMeta;
+import org.apache.hop.core.variables.IVariables;
+import org.apache.hop.metadata.api.IHopMetadataProvider;
+import org.apache.hop.pipeline.PipelineMeta;
+import org.apache.hop.pipeline.transform.TransformMeta;
+import org.apache.hop.spark.engines.ISparkPipelineEngineRunConfiguration;
+import org.apache.spark.sql.Dataset;
+import org.apache.spark.sql.Row;
+import org.apache.spark.sql.SparkSession;
+
+public interface ISparkPipelineTransformHandler {
+
+ boolean isInput();
+
+ boolean isOutput();
+
+ void handleTransform(
+ ILogChannel log,
+ IVariables variables,
+ String runConfigurationName,
+ ISparkPipelineEngineRunConfiguration runConfiguration,
+ IHopMetadataProvider metadataProvider,
+ String metastoreJson,
+ PipelineMeta pipelineMeta,
+ TransformMeta transformMeta,
+ Map> transformDatasetMap,
+ SparkSession spark,
+ IRowMeta rowMeta,
+ List previousTransforms,
+ Dataset input)
+ throws HopException;
+}
diff --git a/plugins/engines/spark/src/main/java/org/apache/hop/spark/pipeline/handler/SparkBaseTransformHandler.java b/plugins/engines/spark/src/main/java/org/apache/hop/spark/pipeline/handler/SparkBaseTransformHandler.java
new file mode 100644
index 00000000000..2ee63a591ec
--- /dev/null
+++ b/plugins/engines/spark/src/main/java/org/apache/hop/spark/pipeline/handler/SparkBaseTransformHandler.java
@@ -0,0 +1,88 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hop.spark.pipeline.handler;
+
+import org.apache.hop.core.exception.HopException;
+import org.apache.hop.core.xml.XmlHandler;
+import org.apache.hop.metadata.api.IHopMetadataProvider;
+import org.apache.hop.pipeline.PipelineMeta;
+import org.apache.hop.pipeline.transform.ITransformMeta;
+import org.apache.hop.pipeline.transform.TransformMeta;
+import org.apache.hop.spark.core.SparkNativeMetrics;
+import org.apache.hop.spark.core.SparkTransformMetricsAccumulator;
+import org.apache.hop.spark.pipeline.ISparkPipelineTransformHandler;
+import org.apache.spark.sql.Dataset;
+import org.apache.spark.sql.Row;
+import org.w3c.dom.Node;
+
+public abstract class SparkBaseTransformHandler implements ISparkPipelineTransformHandler {
+
+ private SparkTransformMetricsAccumulator metricsAccumulator;
+
+ public void setMetricsAccumulator(SparkTransformMetricsAccumulator metricsAccumulator) {
+ this.metricsAccumulator = metricsAccumulator;
+ }
+
+ protected SparkTransformMetricsAccumulator getMetricsAccumulator() {
+ return metricsAccumulator;
+ }
+
+ /**
+ * Attach native row tracking for this transform when a metrics accumulator is registered. Returns
+ * {@code dataset} unchanged when metrics are disabled.
+ */
+ protected Dataset trackMetrics(
+ Dataset dataset, TransformMeta transformMeta, SparkNativeMetrics.Role role) {
+ if (dataset == null || transformMeta == null) {
+ return dataset;
+ }
+ return SparkNativeMetrics.track(dataset, transformMeta.getName(), metricsAccumulator, role);
+ }
+
+ @Override
+ public boolean isInput() {
+ return false;
+ }
+
+ @Override
+ public boolean isOutput() {
+ return false;
+ }
+
+ protected Node getTransformXmlNode(TransformMeta transformMeta) throws HopException {
+ String xml = transformMeta.getXml();
+ return XmlHandler.getSubNode(XmlHandler.loadXmlString(xml), TransformMeta.XML_TAG);
+ }
+
+ protected void loadTransformMetadata(
+ ITransformMeta meta,
+ TransformMeta transformMeta,
+ IHopMetadataProvider metadataProvider,
+ PipelineMeta pipelineMeta)
+ throws HopException {
+ // Prefer live instance when classloaders match (normal Hop driver)
+ if (meta.getClass().isInstance(transformMeta.getTransform())) {
+ ITransformMeta live = transformMeta.getTransform();
+ // Copy via XML round-trip so we never mutate the live pipeline meta
+ meta.loadXml(getTransformXmlNode(transformMeta), metadataProvider);
+ } else {
+ meta.loadXml(getTransformXmlNode(transformMeta), metadataProvider);
+ }
+ meta.searchInfoAndTargetTransforms(pipelineMeta.getTransforms());
+ }
+}
diff --git a/plugins/engines/spark/src/main/java/org/apache/hop/spark/pipeline/handler/SparkFileInputHandler.java b/plugins/engines/spark/src/main/java/org/apache/hop/spark/pipeline/handler/SparkFileInputHandler.java
new file mode 100644
index 00000000000..851e57c27bd
--- /dev/null
+++ b/plugins/engines/spark/src/main/java/org/apache/hop/spark/pipeline/handler/SparkFileInputHandler.java
@@ -0,0 +1,380 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hop.spark.pipeline.handler;
+
+import static org.apache.spark.sql.functions.col;
+import static org.apache.spark.sql.functions.lit;
+import static org.apache.spark.sql.functions.to_date;
+import static org.apache.spark.sql.functions.to_timestamp;
+import static org.apache.spark.sql.functions.trim;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.hop.core.exception.HopException;
+import org.apache.hop.core.logging.ILogChannel;
+import org.apache.hop.core.row.IRowMeta;
+import org.apache.hop.core.row.IValueMeta;
+import org.apache.hop.core.variables.IVariables;
+import org.apache.hop.metadata.api.IHopMetadataProvider;
+import org.apache.hop.pipeline.PipelineMeta;
+import org.apache.hop.pipeline.transform.TransformMeta;
+import org.apache.hop.spark.core.SparkNativeMetrics;
+import org.apache.hop.spark.engines.ISparkPipelineEngineRunConfiguration;
+import org.apache.hop.spark.pkg.SparkProjectPackage;
+import org.apache.hop.spark.transforms.io.SparkField;
+import org.apache.hop.spark.transforms.io.SparkFileInputMeta;
+import org.apache.hop.spark.util.SparkPathDialect;
+import org.apache.spark.sql.Column;
+import org.apache.spark.sql.DataFrameReader;
+import org.apache.spark.sql.Dataset;
+import org.apache.spark.sql.Row;
+import org.apache.spark.sql.SparkSession;
+import org.apache.spark.sql.types.DataTypes;
+
+/**
+ * Native Spark file read for {@link SparkFileInputMeta}.
+ *
+ *
CSV/text with an explicit field list is read as strings first, then projected/cast