issue #7547: a Hop marketplace

.gitignore

@@ -83,3 +83,6 @@ screenshots/
 # Docker script leaving audit data
 docker/local-data/
 /docker/marketplace-nexus/.env
+
+# Jenkins-style local deploy output (verify-ci-snapshot-zips.sh)
+local-snapshots-dir/

Jenkinsfile

@@ -113,12 +113,23 @@ pipeline {
                 }
             }
         }
+        stage('Assembly size check') {
+            when {
+                  anyOf { changeset pattern: "^(?!docs).*^(?!integration-tests).*" , comparator: "REGEXP" ; equals expected: true, actual: params.FORCE_BUILD }
+                }
+            steps {
+                // ASF Artifactory rejects packages over ~850MB; fail early with margin.
+                sh "./tools/check-assembly-size.sh"
+            }
+        }
         stage('Unzip Apache Hop'){
             when {
                   anyOf { changeset pattern: "^(?!docs).*^(?!integration-tests).*" , comparator: "REGEXP" ; equals expected: true, actual: params.FORCE_BUILD }
                 }
             steps{
                 sh "unzip ./assemblies/client/target/hop-client-*.zip -d ./assemblies/client/target/"
+                // Optional Wave 1 plugins are marketplace-only; install into the extracted client for IT/Docker
+                sh "./tools/install-wave1-plugins.sh ./assemblies/client/target/hop"
                 sh "unzip ./assemblies/web/target/hop.war -d ./assemblies/web/target/webapp"
             }
         }

assemblies/client/pom.xml

@@ -61,18 +61,17 @@
             <version>${project.version}</version>
             <type>zip</type>
         </dependency>
+        <!-- Slim Beam-related libs only: lib/spark-client + shared lib/core extras.
+             Beam SDKs/runners ship with the marketplace plugin under plugins/engines/beam/lib-beam.
+             Use -Pbeam to bake the full Beam plugin (+ lib-beam) into hop-client (IT / fat Beam builds). -->
         <dependency>
             <groupId>org.apache.hop</groupId>
             <artifactId>hop-engines-beam</artifactId>
             <version>${project.version}</version>
+            <classifier>libs</classifier>
             <type>zip</type>
         </dependency>
-        <dependency>
-            <groupId>org.apache.hop</groupId>
-            <artifactId>hop-engines-spark</artifactId>
-            <version>${project.version}</version>
-            <type>zip</type>
-        </dependency>
+        <!-- hop-engines-spark is a marketplace-optional plugin (not bundled in the client). -->
         <dependency>
             <groupId>org.apache.hop</groupId>
             <artifactId>hop-ui</artifactId>
@@ -144,4 +143,25 @@
         </dependency>
     </dependencies>
 
+    <!--
+      Opt-in Beam engine + SDKs in hop-client (plugins/engines/beam including lib-beam).
+      Analogous to HOP_SPARK_CLIENT_VERSION=native for Spark: default lean client has no Beam
+      runners on the classpath; use -Pbeam for IT / local full Beam installs.
+
+        ./mvnw -pl assemblies/client -am package -Pbeam
+    -->
+    <profiles>
+        <profile>
+            <id>beam</id>
+            <dependencies>
+                <dependency>
+                    <groupId>org.apache.hop</groupId>
+                    <artifactId>hop-engines-beam</artifactId>
+                    <version>${project.version}</version>
+                    <type>zip</type>
+                </dependency>
+            </dependencies>
+        </profile>
+    </profiles>
+
 </project>

assemblies/client/src/assembly/assembly.xml

@@ -40,6 +40,9 @@
                 <include>org.apache.hop:hop-core:zip</include>
                 <include>org.apache.hop:hop-engine:zip</include>
                 <include>org.apache.hop:hop-ui:zip</include>
+                <!-- Slim: spark-client + shared lib/core extras (no Beam SDKs) -->
+                <include>org.apache.hop:hop-engines-beam:zip:libs</include>
+                <!-- Full Beam plugin + lib-beam only when dependency present (-Pbeam) -->
                 <include>org.apache.hop:hop-engines-beam:zip</include>
                 <include>org.apache.hop:hop-ui-rcp:zip</include>
             </includes>

assemblies/plugins/pom.xml

@@ -618,12 +618,7 @@
             <version>${project.version}</version>
             <type>zip</type>
         </dependency>
-        <dependency>
-            <groupId>org.apache.hop</groupId>
-            <artifactId>hop-engines-spark</artifactId>
-            <version>${project.version}</version>
-            <type>zip</type>
-        </dependency>
+        <!-- marketplace optional: hop-engines-spark -->
         <dependency>
             <groupId>org.apache.hop</groupId>
             <artifactId>hop-misc-async</artifactId>
@@ -636,27 +631,18 @@
             <version>${project.version}</version>
             <type>zip</type>
         </dependency>
-        <dependency>
-            <groupId>org.apache.hop</groupId>
-            <artifactId>hop-misc-documentation</artifactId>
-            <version>${project.version}</version>
-            <type>zip</type>
-        </dependency>
+        <!-- marketplace optional: hop-misc-documentation -->
         <dependency>
             <groupId>org.apache.hop</groupId>
             <artifactId>hop-misc-git</artifactId>
             <version>${project.version}</version>
             <type>zip</type>
         </dependency>
+        <!-- marketplace optional: hop-misc-import -->
+        <!-- marketplace optional: hop-misc-mail -->
         <dependency>
             <groupId>org.apache.hop</groupId>
-            <artifactId>hop-misc-import</artifactId>
-            <version>${project.version}</version>
-            <type>zip</type>
-        </dependency>
-        <dependency>
-            <groupId>org.apache.hop</groupId>
-            <artifactId>hop-misc-mail</artifactId>
+            <artifactId>hop-misc-marketplace</artifactId>
             <version>${project.version}</version>
             <type>zip</type>
         </dependency>
@@ -672,18 +658,14 @@
             <version>${project.version}</version>
             <type>zip</type>
         </dependency>
-        <dependency>
-            <groupId>org.apache.hop</groupId>
-            <artifactId>hop-misc-py4j</artifactId>
-            <version>2.19.0-SNAPSHOT</version>
-            <type>zip</type>
-        </dependency>
+        <!-- marketplace optional: hop-misc-py4j -->
         <dependency>
             <groupId>org.apache.hop</groupId>
             <artifactId>hop-misc-reflection</artifactId>
             <version>${project.version}</version>
             <type>zip</type>
         </dependency>
+        <!-- marketplace optional: hop-misc-rest -->
         <dependency>
             <groupId>org.apache.hop</groupId>
             <artifactId>hop-misc-rest</artifactId>
@@ -708,97 +690,20 @@
             <version>2.19.0-SNAPSHOT</version>
             <type>zip</type>
         </dependency>
-
-        <dependency>
-            <groupId>org.apache.hop</groupId>
-            <artifactId>hop-tech-arrow</artifactId>
-            <version>${project.version}</version>
-            <type>zip</type>
-        </dependency>
-        <dependency>
-            <groupId>org.apache.hop</groupId>
-            <artifactId>hop-tech-avro</artifactId>
-            <version>${project.version}</version>
-            <type>zip</type>
-        </dependency>
-        <dependency>
-            <groupId>org.apache.hop</groupId>
-            <artifactId>hop-tech-aws</artifactId>
-            <version>${project.version}</version>
-            <type>zip</type>
-        </dependency>
-        <dependency>
-            <groupId>org.apache.hop</groupId>
-            <artifactId>hop-tech-azure</artifactId>
-            <version>${project.version}</version>
-            <type>zip</type>
-        </dependency>
-        <dependency>
-            <groupId>org.apache.hop</groupId>
-            <artifactId>hop-tech-cassandra</artifactId>
-            <version>${project.version}</version>
-            <type>zip</type>
-        </dependency>
-        <dependency>
-            <groupId>org.apache.hop</groupId>
-            <artifactId>hop-tech-databricks</artifactId>
-            <version>${project.version}</version>
-            <type>zip</type>
-        </dependency>
-        <dependency>
-            <groupId>org.apache.hop</groupId>
-            <artifactId>hop-tech-dropbox</artifactId>
-            <version>${project.version}</version>
-            <type>zip</type>
-        </dependency>
-        <dependency>
-            <groupId>org.apache.hop</groupId>
-            <artifactId>hop-tech-elastic</artifactId>
-            <version>${project.version}</version>
-            <type>zip</type>
-        </dependency>
-        <dependency>
-            <groupId>org.apache.hop</groupId>
-            <artifactId>hop-tech-google</artifactId>
-            <version>${project.version}</version>
-            <type>zip</type>
-        </dependency>
-        <dependency>
-            <groupId>org.apache.hop</groupId>
-            <artifactId>hop-tech-minio</artifactId>
-            <version>${project.version}</version>
-            <type>zip</type>
-        </dependency>
-        <dependency>
-            <groupId>org.apache.hop</groupId>
-            <artifactId>hop-tech-mongodb</artifactId>
-            <version>${project.version}</version>
-            <type>zip</type>
-        </dependency>
-        <dependency>
-            <groupId>org.apache.hop</groupId>
-            <artifactId>hop-tech-neo4j</artifactId>
-            <version>${project.version}</version>
-            <type>zip</type>
-        </dependency>
-        <dependency>
-            <groupId>org.apache.hop</groupId>
-            <artifactId>hop-tech-opensearch</artifactId>
-            <version>${project.version}</version>
-            <type>zip</type>
-        </dependency>
-        <dependency>
-            <groupId>org.apache.hop</groupId>
-            <artifactId>hop-tech-parquet</artifactId>
-            <version>${project.version}</version>
-            <type>zip</type>
-        </dependency>
-        <dependency>
-            <groupId>org.apache.hop</groupId>
-            <artifactId>hop-tech-salesforce</artifactId>
-            <version>${project.version}</version>
-            <type>zip</type>
-        </dependency>
+        <!-- marketplace optional: hop-tech-arrow -->
+        <!-- marketplace optional: hop-tech-avro -->
+        <!-- marketplace optional: hop-tech-aws -->
+        <!-- marketplace optional: hop-tech-azure -->
+        <!-- marketplace optional: hop-tech-cassandra -->
+        <!-- marketplace optional: hop-tech-databricks -->
+        <!-- marketplace optional: hop-tech-dropbox -->
+        <!-- marketplace optional: hop-tech-elastic -->
+        <!-- marketplace optional: hop-tech-minio -->
+        <!-- marketplace optional: hop-tech-mongodb -->
+        <!-- marketplace optional: hop-tech-neo4j -->
+        <!-- marketplace optional: hop-tech-opensearch -->
+        <!-- marketplace optional: hop-tech-parquet -->
+        <!-- marketplace optional: hop-tech-salesforce -->
         <dependency>
             <groupId>org.apache.hop</groupId>
             <artifactId>hop-tech-vault</artifactId>
@@ -1009,24 +914,14 @@
             <version>${project.version}</version>
             <type>zip</type>
         </dependency>
-        <dependency>
-            <groupId>org.apache.hop</groupId>
-            <artifactId>hop-transform-drools</artifactId>
-            <version>${project.version}</version>
-            <type>zip</type>
-        </dependency>
+        <!-- marketplace optional: hop-transform-drools -->
         <dependency>
             <groupId>org.apache.hop</groupId>
             <artifactId>hop-transform-dynamicsqlrow</artifactId>
             <version>${project.version}</version>
             <type>zip</type>
         </dependency>
-        <dependency>
-            <groupId>org.apache.hop</groupId>
-            <artifactId>hop-transform-edi2xml</artifactId>
-            <version>${project.version}</version>
-            <type>zip</type>
-        </dependency>
+        <!-- marketplace optional: hop-transform-edi2xml -->
         <dependency>
             <groupId>org.apache.hop</groupId>
             <artifactId>hop-transform-excel</artifactId>
@@ -1051,12 +946,7 @@
             <version>${project.version}</version>
             <type>zip</type>
         </dependency>
-        <dependency>
-            <groupId>org.apache.hop</groupId>
-            <artifactId>hop-transform-fake</artifactId>
-            <version>${project.version}</version>
-            <type>zip</type>
-        </dependency>
+        <!-- marketplace optional: hop-transform-fake -->
         <dependency>
             <groupId>org.apache.hop</groupId>
             <artifactId>hop-transform-fieldschangesequence</artifactId>
@@ -1417,12 +1307,7 @@
             <version>${project.version}</version>
             <type>zip</type>
         </dependency>
-        <dependency>
-            <groupId>org.apache.hop</groupId>
-            <artifactId>hop-transform-script</artifactId>
-            <version>${project.version}</version>
-            <type>zip</type>
-        </dependency>
+        <!-- marketplace optional: hop-transform-script -->
         <dependency>
             <groupId>org.apache.hop</groupId>
             <artifactId>hop-transform-selectvalues</artifactId>
@@ -1507,12 +1392,7 @@
             <version>${project.version}</version>
             <type>zip</type>
         </dependency>
-        <dependency>
-            <groupId>org.apache.hop</groupId>
-            <artifactId>hop-transform-stanfordnlp</artifactId>
-            <version>${project.version}</version>
-            <type>zip</type>
-        </dependency>
+        <!-- marketplace optional: hop-transform-stanfordnlp -->
         <dependency>
             <groupId>org.apache.hop</groupId>
             <artifactId>hop-transform-streamlookup</artifactId>
@@ -1591,12 +1471,7 @@
             <version>${project.version}</version>
             <type>zip</type>
         </dependency>
-        <dependency>
-            <groupId>org.apache.hop</groupId>
-            <artifactId>hop-transform-tika</artifactId>
-            <version>${project.version}</version>
-            <type>zip</type>
-        </dependency>
+        <!-- marketplace optional: hop-transform-tika -->
         <dependency>
             <groupId>org.apache.hop</groupId>
             <artifactId>hop-transform-tokenreplacement</artifactId>
@@ -1633,12 +1508,7 @@
             <version>${project.version}</version>
             <type>zip</type>
         </dependency>
-        <dependency>
-            <groupId>org.apache.hop</groupId>
-            <artifactId>hop-transform-vcard</artifactId>
-            <version>${project.version}</version>
-            <type>zip</type>
-        </dependency>
+        <!-- marketplace optional: hop-transform-vcard -->
         <dependency>
             <groupId>org.apache.hop</groupId>
             <artifactId>hop-transform-webserviceavailable</artifactId>