[#14002] improvement(test): Run Kafka, LocalStack, ClickHouse and OceanBase on official Testcontainers modules

agent-skills/trino-test/trino-test-guide.md

@@ -544,16 +544,15 @@ Use 5-digit number prefix for easy sorting and reference:
 
 #### Issue 3: Docker container startup failure
 
-**Cause**: Port conflicts or insufficient resources
+**Cause**: Insufficient resources, or containers left from an earlier run
 
 **Solution**:
 ```bash
-# Cleanup old containers
-./integration-test-common/docker-script/shutdown.sh
+# Stop an environment started with --env_only
+./trino-connector/integration-test/trino-test-tools/trino_integration_test.sh --stop
 
-# Check port usage
-lsof -i :8080
-lsof -i :9083
+# List test containers; Testcontainers removes them once their test JVM is gone
+docker ps -a --filter label=org.testcontainers=true
 ```
 
 ## Summary

authorizations/authorization-chain/src/test/java/org/apache/gravitino/authorization/chain/integration/test/TestChainedAuthorizationIT.java

@@ -80,22 +80,20 @@ public void startIntegrationTest() throws Exception {
     registerCustomConfigs(configs);
 
     super.startIntegrationTest();
-    RangerITEnv.init(RangerBaseE2EIT.metalakeName, false);
-    RangerITEnv.startHiveRangerContainer();
+    RangerITEnv.init(ranger, RangerBaseE2EIT.metalakeName, false);
+    hiveRanger = RangerITEnv.startHiveRangerContainer();
 
     HIVE_METASTORE_URIS =
         String.format(
             "thrift://%s:%d",
-            containerSuite.getHiveRangerContainer().getContainerIpAddress(),
-            HiveContainer.HIVE_METASTORE_PORT);
+            hiveRanger.getContainerIpAddress(), HiveContainer.HIVE_METASTORE_PORT);
 
     generateRangerSparkSecurityXML("authorization-chain");
 
     DEFAULT_FS =
         String.format(
             "hdfs://%s:%d/user/hive/warehouse",
-            containerSuite.getHiveRangerContainer().getContainerIpAddress(),
-            HiveContainer.HDFS_DEFAULTFS_PORT);
+            hiveRanger.getContainerIpAddress(), HiveContainer.HDFS_DEFAULTFS_PORT);
     BaseIT.runInEnv(
         "HADOOP_USER_NAME",
         "test123",
@@ -326,67 +324,80 @@ public void testCreateSchema() throws InterruptedException {
   }
 
   @Test
-  void testCreateTable() throws InterruptedException {
+  @Override
+  protected void testCreateTable() {
     // TODO
   }
 
   @Test
-  void testSelectModifyTableWithMetalakeLevelRole() throws InterruptedException {
+  @Override
+  protected void testSelectModifyTableWithMetalakeLevelRole() {
     // TODO
   }
 
   @Test
-  void testSelectModifyTableWithTableLevelRole() throws InterruptedException {
+  @Override
+  protected void testSelectModifyTableWithTableLevelRole() {
     // TODO
   }
 
   @Test
-  void testSelectOnlyTable() throws InterruptedException {
+  @Override
+  protected void testSelectOnlyTable() {
     // TODO
   }
 
   @Test
-  void testModifyOnlyTable() throws InterruptedException {
+  @Override
+  protected void testModifyOnlyTable() {
     // TODO
   }
 
   @Test
-  void testCreateAllPrivilegesRole() throws InterruptedException {
+  @Override
+  protected void testCreateAllPrivilegesRole() {
     // TODO
   }
 
   @Test
-  void testDeleteAndRecreateRole() throws InterruptedException {
+  @Override
+  protected void testDeleteAndRecreateRole() {
     // TODO
   }
 
   @Test
-  void testDeleteAndRecreateMetadataObject() throws InterruptedException {
+  @Override
+  protected void testDeleteAndRecreateMetadataObject() {
     // TODO
   }
 
   @Test
-  void testRenameMetadataObject() throws InterruptedException {
+  @Override
+  protected void testRenameMetadataObject() {
     // TODO
   }
 
   @Test
-  void testRenameMetadataObjectPrivilege() throws InterruptedException {
+  @Override
+  protected void testRenameMetadataObjectPrivilege() {
     // TODO
   }
 
   @Test
-  void testChangeOwner() throws InterruptedException {
+  @Override
+  protected void testChangeOwner() {
     // TODO
   }
 
   @Test
-  void testDenyPrivileges() throws InterruptedException {
+  @Override
+  protected void testDenyPrivileges() {
     // TODO
   }
 
   @Test
-  void testGrantPrivilegesForMetalake() throws InterruptedException {
+  @Override
+  protected void testGrantPrivilegesForMetalake() {
     // TODO
   }
 

authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerAuthorizationHDFSPluginIT.java

@@ -40,6 +40,8 @@
 import org.apache.gravitino.authorization.common.PathBasedSecurableObject;
 import org.apache.gravitino.authorization.ranger.RangerAuthorizationPlugin;
 import org.apache.gravitino.catalog.CatalogDispatcher;
+import org.apache.gravitino.integration.test.container.RangerContainer;
+import org.apache.gravitino.integration.test.container.RangerFixture;
 import org.junit.jupiter.api.AfterAll;
 import org.junit.jupiter.api.Assertions;
 import org.junit.jupiter.api.BeforeAll;
@@ -49,6 +51,7 @@
 import org.mockito.Mockito;
 
 @Tag("gravitino-docker-test")
+@RangerFixture
 public class RangerAuthorizationHDFSPluginIT {
 
   private static RangerAuthorizationPlugin rangerAuthPlugin;
@@ -57,8 +60,8 @@ public class RangerAuthorizationHDFSPluginIT {
   private static CatalogDispatcher previousInternalCatalogDispatcher;
 
   @BeforeAll
-  public static void setup() throws Exception {
-    RangerITEnv.init(RangerITEnv.currentFunName(), true);
+  public static void setup(RangerContainer ranger) throws Exception {
+    RangerITEnv.init(ranger, RangerITEnv.currentFunName(), true);
     rangerAuthPlugin = RangerITEnv.rangerAuthHDFSPlugin;
     previousCatalogDispatcher =
         (CatalogDispatcher)

authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerAuthorizationPluginIT.java

@@ -33,19 +33,22 @@
 import org.apache.gravitino.authorization.ranger.RangerAuthorizationPlugin;
 import org.apache.gravitino.authorization.ranger.RangerHadoopSQLMetadataObject;
 import org.apache.gravitino.authorization.ranger.RangerHelper;
+import org.apache.gravitino.integration.test.container.RangerContainer;
+import org.apache.gravitino.integration.test.container.RangerFixture;
 import org.junit.jupiter.api.AfterAll;
 import org.junit.jupiter.api.Assertions;
 import org.junit.jupiter.api.BeforeAll;
 import org.junit.jupiter.api.Tag;
 import org.junit.jupiter.api.Test;
 
 @Tag("gravitino-docker-test")
+@RangerFixture
 public class RangerAuthorizationPluginIT {
   private static RangerAuthorizationPlugin rangerAuthPlugin;
 
   @BeforeAll
-  public static void setup() {
-    RangerITEnv.init(RangerITEnv.currentFunName(), true);
+  public static void setup(RangerContainer ranger) {
+    RangerITEnv.init(ranger, RangerITEnv.currentFunName(), true);
     rangerAuthPlugin = RangerITEnv.rangerAuthHivePlugin;
   }
 

authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerBaseE2EIT.java

@@ -42,6 +42,10 @@
 import org.apache.gravitino.authorization.SecurableObject;
 import org.apache.gravitino.authorization.SecurableObjects;
 import org.apache.gravitino.client.GravitinoMetalake;
+import org.apache.gravitino.integration.test.container.FixtureContainer;
+import org.apache.gravitino.integration.test.container.HiveContainer;
+import org.apache.gravitino.integration.test.container.RangerContainer;
+import org.apache.gravitino.integration.test.container.RangerFixture;
 import org.apache.gravitino.integration.test.util.BaseIT;
 import org.apache.gravitino.integration.test.util.GravitinoITUtils;
 import org.apache.gravitino.rel.TableChange;
@@ -54,6 +58,7 @@
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+@RangerFixture
 public abstract class RangerBaseE2EIT extends BaseIT {
   private static final Logger LOG = LoggerFactory.getLogger(RangerBaseE2EIT.class);
   public static final String catalogName = GravitinoITUtils.genRandomName("catalog").toLowerCase();
@@ -105,6 +110,11 @@ public abstract class RangerBaseE2EIT extends BaseIT {
 
   protected static final String SQL_DROP_TABLE = String.format("DROP TABLE %s", tableName);
 
+  @FixtureContainer protected RangerContainer ranger;
+
+  // The Ranger-enabled Hive; subclasses start it after RangerITEnv.init().
+  protected static HiveContainer hiveRanger;
+
   protected static void generateRangerSparkSecurityXML(String modeName) throws IOException {
     String templatePath =
         String.join(
@@ -279,7 +289,7 @@ void checkCreateTable() {
   }
 
   @Test
-  void testCreateTable() {
+  protected void testCreateTable() {
     // Choose a catalog
     reset();
 
@@ -345,7 +355,7 @@ void checkCreateTable() {
   }
 
   @Test
-  void testSelectModifyTableWithMetalakeLevelRole() {
+  protected void testSelectModifyTableWithMetalakeLevelRole() {
     // Choose a catalog
     reset();
 
@@ -430,7 +440,7 @@ protected abstract static class TestSelectModifyTableChecker {
   }
 
   @Test
-  void testSelectModifyTableWithTableLevelRole() {
+  protected void testSelectModifyTableWithTableLevelRole() {
     // Choose a catalog
     reset();
 
@@ -496,7 +506,7 @@ void testSelectModifyTableWithTableLevelRole() {
   }
 
   @Test
-  void testSelectOnlyTable() {
+  protected void testSelectOnlyTable() {
     // Choose a catalog
     reset();
 
@@ -603,7 +613,7 @@ protected abstract static class TestSelectOnlyTableChecker {
   }
 
   @Test
-  void testModifyOnlyTable() {
+  protected void testModifyOnlyTable() {
     // Choose a catalog
     reset();
 
@@ -686,7 +696,7 @@ protected abstract static class TestModifyOnlyTableChecker {
   }
 
   @Test
-  void testCreateAllPrivilegesRole() {
+  protected void testCreateAllPrivilegesRole() {
     // Choose a catalog
     reset();
 
@@ -735,7 +745,7 @@ void testCreateAllPrivilegesRole() {
   }
 
   @Test
-  void testDeleteAndRecreateRole() {
+  protected void testDeleteAndRecreateRole() {
     // Choose a catalog
     reset();
 
@@ -801,7 +811,7 @@ void checkCreateSchema() {
   }
 
   @Test
-  void testDeleteAndRecreateMetadataObject() {
+  protected void testDeleteAndRecreateMetadataObject() {
     // Choose a catalog
     reset();
 
@@ -872,7 +882,7 @@ void checkRecreateSchema() {
   }
 
   @Test
-  void testRenameMetadataObject() {
+  protected void testRenameMetadataObject() {
     // Choose a catalog
     reset();
 
@@ -910,7 +920,7 @@ void testRenameMetadataObject() {
   }
 
   @Test
-  void testRenameMetadataObjectPrivilege() {
+  protected void testRenameMetadataObjectPrivilege() {
     // Choose a catalog
     reset();
 
@@ -959,7 +969,7 @@ void testRenameMetadataObjectPrivilege() {
   }
 
   @Test
-  void testChangeOwner() {
+  protected void testChangeOwner() {
     // Choose a catalog
     reset();
 
@@ -1183,7 +1193,7 @@ void checkShowSchemasAgain() {
     }
   }
 
-  void testDenyPrivileges() {
+  protected void testDenyPrivileges() {
     // Choose a catalog
     reset();
 
@@ -1248,7 +1258,7 @@ void testDenyPrivileges() {
 
   // ISSUE-5892 Fix to grant privilege for the metalake
   @Test
-  void testGrantPrivilegesForMetalake() {
+  protected void testGrantPrivilegesForMetalake() {
     // Choose a catalog
     reset();
 

authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerFilesetIT.java

@@ -52,8 +52,10 @@
 import org.apache.gravitino.client.GravitinoMetalake;
 import org.apache.gravitino.file.Fileset;
 import org.apache.gravitino.file.FilesetChange;
+import org.apache.gravitino.integration.test.container.FixtureContainer;
 import org.apache.gravitino.integration.test.container.HiveContainer;
 import org.apache.gravitino.integration.test.container.RangerContainer;
+import org.apache.gravitino.integration.test.container.RangerFixture;
 import org.apache.gravitino.integration.test.util.BaseIT;
 import org.apache.gravitino.integration.test.util.GravitinoITUtils;
 import org.apache.hadoop.conf.Configuration;
@@ -73,6 +75,7 @@
 import org.slf4j.LoggerFactory;
 
 @Tag("gravitino-docker-test")
+@RangerFixture
 public class RangerFilesetIT extends BaseIT {
   private static final Logger LOG = LoggerFactory.getLogger(RangerFilesetIT.class);
 
@@ -85,6 +88,8 @@ public class RangerFilesetIT extends BaseIT {
   private FileSystem fileSystem;
   private GravitinoMetalake metalake;
   private Catalog catalog;
+  @FixtureContainer private RangerContainer ranger;
+  private static HiveContainer hiveRanger;
 
   @BeforeAll
   public void startIntegrationTest() throws Exception {
@@ -100,13 +105,11 @@ public void startIntegrationTest() throws Exception {
     registerCustomConfigs(configs);
     super.startIntegrationTest();
 
-    RangerITEnv.init(metalakeName, false);
-    RangerITEnv.startHiveRangerContainer();
+    RangerITEnv.init(ranger, metalakeName, false);
+    hiveRanger = RangerITEnv.startHiveRangerContainer();
 
     RANGER_ADMIN_URL =
-        String.format(
-            "http://%s:%d",
-            containerSuite.getRangerContainer().getContainerIpAddress(), RANGER_SERVER_PORT);
+        String.format("http://%s:%d", ranger.getContainerIpAddress(), RANGER_SERVER_PORT);
 
     Configuration conf = new Configuration();
     conf.set("fs.defaultFS", defaultBaseLocation());
@@ -599,7 +602,7 @@ private String defaultBaseLocation() {
       defaultBaseLocation =
           String.format(
               "hdfs://%s:%d/user/hadoop/%s",
-              containerSuite.getHiveRangerContainer().getContainerIpAddress(),
+              hiveRanger.getContainerIpAddress(),
               HiveContainer.HDFS_DEFAULTFS_PORT,
               schemaName.toLowerCase());
     }