From b95f0d8a47f089aa29377741ebbc283f83b1a8ae Mon Sep 17 00:00:00 2001 From: Nevin Zheng Date: Wed, 21 Oct 2026 14:10:04 -0700 Subject: [PATCH 01/14] [#14105] test: Replace the last JUnit 4 code and declare the test libraries each module uses --- .../test/TestChainedAuthorizationIT.java | 39 ++++++++---- .../integration/test/RangerBaseE2EIT.java | 26 ++++---- build.gradle.kts | 12 ++++ .../integration/test/CatalogHologresIT.java | 2 +- .../integration/test/FilesetABSCatalogIT.java | 2 +- .../integration/test/FilesetCOSCatalogIT.java | 2 +- .../integration/test/FilesetOSSCatalogIT.java | 2 +- catalogs/catalog-jdbc-common/build.gradle.kts | 1 + .../operation/TestDorisTableOperations.java | 4 +- .../TestDorisTablePartitionOperations.java | 8 +-- .../catalog/doris/utils/TestDorisUtils.java | 6 +- .../TestStarRocksTableOperations.java | 4 +- ...TestStarRocksTablePartitionOperations.java | 8 +-- .../starrocks/utils/TestStarRocksUtils.java | 6 +- .../integration/test/CatalogPaimonBaseIT.java | 2 +- .../gravitino/cli/TestCatalogCommands.java | 4 +- .../gravitino/cli/TestColumnCommands.java | 4 +- .../gravitino/cli/TestFilesetCommands.java | 4 +- .../gravitino/cli/TestGroupCommands.java | 4 +- .../gravitino/cli/TestMetalakeCommands.java | 4 +- .../gravitino/cli/TestOwnerCommands.java | 4 +- .../apache/gravitino/cli/TestParseType.java | 51 ++++++++------- .../gravitino/cli/TestRoleCommands.java | 4 +- .../gravitino/cli/TestSchemaCommands.java | 6 +- .../gravitino/cli/TestTableCommands.java | 4 +- .../apache/gravitino/cli/TestTagCommands.java | 4 +- .../gravitino/cli/TestTopicCommands.java | 4 +- .../gravitino/cli/TestUserCommands.java | 4 +- .../cli/command/TestOwnerDetails.java | 26 ++++---- .../cli/commands/TestListProperties.java | 2 +- .../gravitino/cli/output/TestPlainFormat.java | 2 +- .../gravitino/cli/output/TestTableFormat.java | 2 +- .../client/TestGravitinoVersion.java | 2 +- .../authorization/CatalogAuthorizationIT.java | 26 ++++---- .../authorization/FilesetAuthorizationIT.java | 38 ++++++------ .../FunctionAuthorizationIT.java | 2 +- .../authorization/GroupAuthorizationIT.java | 10 +-- .../HierarchicalSchemaAuthorizationIT.java | 2 +- .../MetadataObjectRoleAuthorizationIT.java | 2 +- .../MetalakeAuthorizationIT.java | 34 +++++----- .../authorization/ModelAuthorizationIT.java | 50 +++++++-------- .../authorization/OwnerAuthorizationIT.java | 62 +++++++++---------- .../PartitionAuthorizationIT.java | 26 ++++---- .../PermissionAuthorizationIT.java | 26 ++++---- .../authorization/PolicyAuthorizationIT.java | 30 ++++----- .../authorization/RoleAuthorizationIT.java | 32 +++++----- .../authorization/SchemaAuthorizationIT.java | 28 ++++----- .../StatisticAuthorizationIT.java | 22 +++---- .../authorization/TableAuthorizationIT.java | 42 ++++++------- .../TagOperationsAuthorizationIT.java | 44 ++++++------- .../authorization/TopicAuthorizationIT.java | 30 ++++----- .../authorization/UserAuthorizationIT.java | 18 +++--- ...ileSystemGCSCredentialAuthorizationIT.java | 2 +- ...FileSystemS3CredentialAuthorizationIT.java | 2 +- ...itinoVirtualFileSystemABSCredentialIT.java | 2 +- .../test/GravitinoVirtualFileSystemABSIT.java | 2 +- ...itinoVirtualFileSystemCOSCredentialIT.java | 2 +- .../test/GravitinoVirtualFileSystemCOSIT.java | 2 +- ...itinoVirtualFileSystemOSSCredentialIT.java | 2 +- .../test/GravitinoVirtualFileSystemOSSIT.java | 2 +- ...vitinoVirtualFileSystemS3CredentialIT.java | 2 +- config/checkstyle/checkstyle.xml | 35 +++++++++++ core/build.gradle.kts | 2 + .../gravitino/storage/TestSQLScripts.java | 9 ++- .../store/TestGravitinoCatalogStore.java | 24 +++---- gradle/libs.versions.toml | 7 +++ .../integration/test/IcebergRESTOSSIT.java | 2 +- .../IcebergRESTS3TokenAuthorizationIT.java | 2 +- .../test/IcebergRESTS3TokenIT.java | 2 +- .../iceberg-rest-trino-it/build.gradle.kts | 1 + integration-test-common/build.gradle.kts | 2 + .../integration/test/MiniGravitino.java | 2 +- lance/lance-rest-server/build.gradle.kts | 1 + plugins/idp-basic/build.gradle.kts | 1 + server-common/build.gradle.kts | 1 + server/build.gradle.kts | 1 + .../authorization/SparkAuthorizationIT.java | 14 ++--- .../integration/test/sql/SparkTestConfig.java | 2 +- spark-connector/v3.5/spark/build.gradle.kts | 1 + spark-connector/v4.0/spark/build.gradle.kts | 1 + .../trino-connector-440-445/build.gradle.kts | 2 + .../trino-connector-446-451/build.gradle.kts | 2 + .../trino-connector-452-468/build.gradle.kts | 2 + .../trino-connector-469-472/build.gradle.kts | 2 + .../trino-connector-473-478/build.gradle.kts | 1 + .../trino-connector/build.gradle.kts | 2 + 86 files changed, 503 insertions(+), 419 deletions(-) create mode 100644 config/checkstyle/checkstyle.xml diff --git a/authorizations/authorization-chain/src/test/java/org/apache/gravitino/authorization/chain/integration/test/TestChainedAuthorizationIT.java b/authorizations/authorization-chain/src/test/java/org/apache/gravitino/authorization/chain/integration/test/TestChainedAuthorizationIT.java index 582a4794ec5..30b45b2fef1 100644 --- a/authorizations/authorization-chain/src/test/java/org/apache/gravitino/authorization/chain/integration/test/TestChainedAuthorizationIT.java +++ b/authorizations/authorization-chain/src/test/java/org/apache/gravitino/authorization/chain/integration/test/TestChainedAuthorizationIT.java @@ -326,67 +326,80 @@ public void testCreateSchema() throws InterruptedException { } @Test - void testCreateTable() throws InterruptedException { + @Override + protected void testCreateTable() { // TODO } @Test - void testSelectModifyTableWithMetalakeLevelRole() throws InterruptedException { + @Override + protected void testSelectModifyTableWithMetalakeLevelRole() { // TODO } @Test - void testSelectModifyTableWithTableLevelRole() throws InterruptedException { + @Override + protected void testSelectModifyTableWithTableLevelRole() { // TODO } @Test - void testSelectOnlyTable() throws InterruptedException { + @Override + protected void testSelectOnlyTable() { // TODO } @Test - void testModifyOnlyTable() throws InterruptedException { + @Override + protected void testModifyOnlyTable() { // TODO } @Test - void testCreateAllPrivilegesRole() throws InterruptedException { + @Override + protected void testCreateAllPrivilegesRole() { // TODO } @Test - void testDeleteAndRecreateRole() throws InterruptedException { + @Override + protected void testDeleteAndRecreateRole() { // TODO } @Test - void testDeleteAndRecreateMetadataObject() throws InterruptedException { + @Override + protected void testDeleteAndRecreateMetadataObject() { // TODO } @Test - void testRenameMetadataObject() throws InterruptedException { + @Override + protected void testRenameMetadataObject() { // TODO } @Test - void testRenameMetadataObjectPrivilege() throws InterruptedException { + @Override + protected void testRenameMetadataObjectPrivilege() { // TODO } @Test - void testChangeOwner() throws InterruptedException { + @Override + protected void testChangeOwner() { // TODO } @Test - void testDenyPrivileges() throws InterruptedException { + @Override + protected void testDenyPrivileges() { // TODO } @Test - void testGrantPrivilegesForMetalake() throws InterruptedException { + @Override + protected void testGrantPrivilegesForMetalake() { // TODO } diff --git a/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerBaseE2EIT.java b/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerBaseE2EIT.java index e404e77a362..b1f1dc32051 100644 --- a/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerBaseE2EIT.java +++ b/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerBaseE2EIT.java @@ -279,7 +279,7 @@ void checkCreateTable() { } @Test - void testCreateTable() { + protected void testCreateTable() { // Choose a catalog reset(); @@ -345,7 +345,7 @@ void checkCreateTable() { } @Test - void testSelectModifyTableWithMetalakeLevelRole() { + protected void testSelectModifyTableWithMetalakeLevelRole() { // Choose a catalog reset(); @@ -430,7 +430,7 @@ protected abstract static class TestSelectModifyTableChecker { } @Test - void testSelectModifyTableWithTableLevelRole() { + protected void testSelectModifyTableWithTableLevelRole() { // Choose a catalog reset(); @@ -496,7 +496,7 @@ void testSelectModifyTableWithTableLevelRole() { } @Test - void testSelectOnlyTable() { + protected void testSelectOnlyTable() { // Choose a catalog reset(); @@ -603,7 +603,7 @@ protected abstract static class TestSelectOnlyTableChecker { } @Test - void testModifyOnlyTable() { + protected void testModifyOnlyTable() { // Choose a catalog reset(); @@ -686,7 +686,7 @@ protected abstract static class TestModifyOnlyTableChecker { } @Test - void testCreateAllPrivilegesRole() { + protected void testCreateAllPrivilegesRole() { // Choose a catalog reset(); @@ -735,7 +735,7 @@ void testCreateAllPrivilegesRole() { } @Test - void testDeleteAndRecreateRole() { + protected void testDeleteAndRecreateRole() { // Choose a catalog reset(); @@ -801,7 +801,7 @@ void checkCreateSchema() { } @Test - void testDeleteAndRecreateMetadataObject() { + protected void testDeleteAndRecreateMetadataObject() { // Choose a catalog reset(); @@ -872,7 +872,7 @@ void checkRecreateSchema() { } @Test - void testRenameMetadataObject() { + protected void testRenameMetadataObject() { // Choose a catalog reset(); @@ -910,7 +910,7 @@ void testRenameMetadataObject() { } @Test - void testRenameMetadataObjectPrivilege() { + protected void testRenameMetadataObjectPrivilege() { // Choose a catalog reset(); @@ -959,7 +959,7 @@ void testRenameMetadataObjectPrivilege() { } @Test - void testChangeOwner() { + protected void testChangeOwner() { // Choose a catalog reset(); @@ -1183,7 +1183,7 @@ void checkShowSchemasAgain() { } } - void testDenyPrivileges() { + protected void testDenyPrivileges() { // Choose a catalog reset(); @@ -1248,7 +1248,7 @@ void testDenyPrivileges() { // ISSUE-5892 Fix to grant privilege for the metalake @Test - void testGrantPrivilegesForMetalake() { + protected void testGrantPrivilegesForMetalake() { // Choose a catalog reset(); diff --git a/build.gradle.kts b/build.gradle.kts index a1295f06c33..46fa0bded6d 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -583,6 +583,7 @@ fun excludePackagesForSparkConnector(project: Project) { } val commonsBeanutilsVersion: String = libs.versions.commons.beanutils.get() +val checkstyleVersion: String = libs.versions.checkstyle.get() subprojects { // Gravitino Python client project didn't need to apply the java plugin @@ -613,6 +614,17 @@ subprojects { exclude(group = "log4j", module = "log4j") } + // Checkstyle carries only the import bans Spotless cannot express, such as JUnit 4. + apply(plugin = "checkstyle") + configure { + toolVersion = checkstyleVersion + configFile = rootProject.file("config/checkstyle/checkstyle.xml") + maxWarnings = 0 + } + // The import bans need no type information; an empty classpath keeps checkstyle from building + // other modules' test jars. + tasks.withType().configureEach { classpath = files() } + repositories { mavenCentral() mavenLocal() diff --git a/catalogs-contrib/catalog-jdbc-hologres/src/test/java/org/apache/gravitino/catalog/hologres/integration/test/CatalogHologresIT.java b/catalogs-contrib/catalog-jdbc-hologres/src/test/java/org/apache/gravitino/catalog/hologres/integration/test/CatalogHologresIT.java index 670d7aabe78..ae35d3731df 100644 --- a/catalogs-contrib/catalog-jdbc-hologres/src/test/java/org/apache/gravitino/catalog/hologres/integration/test/CatalogHologresIT.java +++ b/catalogs-contrib/catalog-jdbc-hologres/src/test/java/org/apache/gravitino/catalog/hologres/integration/test/CatalogHologresIT.java @@ -27,6 +27,7 @@ import java.util.Map; import java.util.Set; import java.util.stream.Collectors; +import org.apache.commons.lang3.StringUtils; import org.apache.gravitino.Catalog; import org.apache.gravitino.NameIdentifier; import org.apache.gravitino.Namespace; @@ -63,7 +64,6 @@ import org.junit.jupiter.api.TestInstance.Lifecycle; import org.junit.jupiter.api.condition.EnabledIf; import org.junit.jupiter.api.condition.EnabledIfEnvironmentVariable; -import org.junit.platform.commons.util.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/catalogs/catalog-fileset/src/test/java/org/apache/gravitino/catalog/fileset/integration/test/FilesetABSCatalogIT.java b/catalogs/catalog-fileset/src/test/java/org/apache/gravitino/catalog/fileset/integration/test/FilesetABSCatalogIT.java index 209f66bce79..5394f7c4506 100644 --- a/catalogs/catalog-fileset/src/test/java/org/apache/gravitino/catalog/fileset/integration/test/FilesetABSCatalogIT.java +++ b/catalogs/catalog-fileset/src/test/java/org/apache/gravitino/catalog/fileset/integration/test/FilesetABSCatalogIT.java @@ -25,6 +25,7 @@ import java.io.IOException; import java.net.URI; import java.util.Map; +import org.apache.commons.lang3.StringUtils; import org.apache.gravitino.Catalog; import org.apache.gravitino.NameIdentifier; import org.apache.gravitino.Schema; @@ -39,7 +40,6 @@ import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.condition.EnabledIf; -import org.junit.platform.commons.util.StringUtils; @EnabledIf("absIsConfigured") public class FilesetABSCatalogIT extends FilesetCatalogIT { diff --git a/catalogs/catalog-fileset/src/test/java/org/apache/gravitino/catalog/fileset/integration/test/FilesetCOSCatalogIT.java b/catalogs/catalog-fileset/src/test/java/org/apache/gravitino/catalog/fileset/integration/test/FilesetCOSCatalogIT.java index 232fd638642..1735189f7c1 100644 --- a/catalogs/catalog-fileset/src/test/java/org/apache/gravitino/catalog/fileset/integration/test/FilesetCOSCatalogIT.java +++ b/catalogs/catalog-fileset/src/test/java/org/apache/gravitino/catalog/fileset/integration/test/FilesetCOSCatalogIT.java @@ -26,6 +26,7 @@ import java.io.IOException; import java.net.URI; import java.util.Map; +import org.apache.commons.lang3.StringUtils; import org.apache.gravitino.Catalog; import org.apache.gravitino.NameIdentifier; import org.apache.gravitino.Schema; @@ -40,7 +41,6 @@ import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.condition.EnabledIf; -import org.junit.platform.commons.util.StringUtils; /** * Integration test for the fileset catalog backed by Tencent Cloud COS. It is disabled unless the diff --git a/catalogs/catalog-fileset/src/test/java/org/apache/gravitino/catalog/fileset/integration/test/FilesetOSSCatalogIT.java b/catalogs/catalog-fileset/src/test/java/org/apache/gravitino/catalog/fileset/integration/test/FilesetOSSCatalogIT.java index 4ffd93f1b0a..57d76dac730 100644 --- a/catalogs/catalog-fileset/src/test/java/org/apache/gravitino/catalog/fileset/integration/test/FilesetOSSCatalogIT.java +++ b/catalogs/catalog-fileset/src/test/java/org/apache/gravitino/catalog/fileset/integration/test/FilesetOSSCatalogIT.java @@ -26,6 +26,7 @@ import java.io.IOException; import java.net.URI; import java.util.Map; +import org.apache.commons.lang3.StringUtils; import org.apache.gravitino.Catalog; import org.apache.gravitino.NameIdentifier; import org.apache.gravitino.Schema; @@ -40,7 +41,6 @@ import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.condition.EnabledIf; -import org.junit.platform.commons.util.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/catalogs/catalog-jdbc-common/build.gradle.kts b/catalogs/catalog-jdbc-common/build.gradle.kts index b29ae7786fd..6e4a8b92483 100644 --- a/catalogs/catalog-jdbc-common/build.gradle.kts +++ b/catalogs/catalog-jdbc-common/build.gradle.kts @@ -68,6 +68,7 @@ dependencies { testImplementation(libs.mockito.core) testImplementation(libs.sqlite.jdbc) testImplementation(libs.testcontainers) + testImplementation(libs.testcontainers.jdbc) testImplementation(libs.testcontainers.mysql) testImplementation(libs.testcontainers.postgresql) diff --git a/catalogs/catalog-jdbc-doris/src/test/java/org/apache/gravitino/catalog/doris/operation/TestDorisTableOperations.java b/catalogs/catalog-jdbc-doris/src/test/java/org/apache/gravitino/catalog/doris/operation/TestDorisTableOperations.java index f5fcbaff39e..4b23b267fd9 100644 --- a/catalogs/catalog-jdbc-doris/src/test/java/org/apache/gravitino/catalog/doris/operation/TestDorisTableOperations.java +++ b/catalogs/catalog-jdbc-doris/src/test/java/org/apache/gravitino/catalog/doris/operation/TestDorisTableOperations.java @@ -18,8 +18,8 @@ */ package org.apache.gravitino.catalog.doris.operation; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; import com.google.common.collect.Maps; import java.sql.Connection; diff --git a/catalogs/catalog-jdbc-doris/src/test/java/org/apache/gravitino/catalog/doris/operation/TestDorisTablePartitionOperations.java b/catalogs/catalog-jdbc-doris/src/test/java/org/apache/gravitino/catalog/doris/operation/TestDorisTablePartitionOperations.java index 18c91f2a51d..8fba1189d9f 100644 --- a/catalogs/catalog-jdbc-doris/src/test/java/org/apache/gravitino/catalog/doris/operation/TestDorisTablePartitionOperations.java +++ b/catalogs/catalog-jdbc-doris/src/test/java/org/apache/gravitino/catalog/doris/operation/TestDorisTablePartitionOperations.java @@ -18,10 +18,10 @@ */ package org.apache.gravitino.catalog.doris.operation; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertThrows; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; import com.google.common.collect.ImmutableMap; import java.sql.Connection; diff --git a/catalogs/catalog-jdbc-doris/src/test/java/org/apache/gravitino/catalog/doris/utils/TestDorisUtils.java b/catalogs/catalog-jdbc-doris/src/test/java/org/apache/gravitino/catalog/doris/utils/TestDorisUtils.java index 45e804cbffd..0db49094185 100644 --- a/catalogs/catalog-jdbc-doris/src/test/java/org/apache/gravitino/catalog/doris/utils/TestDorisUtils.java +++ b/catalogs/catalog-jdbc-doris/src/test/java/org/apache/gravitino/catalog/doris/utils/TestDorisUtils.java @@ -18,9 +18,9 @@ */ package org.apache.gravitino.catalog.doris.utils; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; import java.util.Collections; import java.util.HashMap; diff --git a/catalogs/catalog-jdbc-starrocks/src/test/java/org/apache/gravitino/catalog/starrocks/operation/TestStarRocksTableOperations.java b/catalogs/catalog-jdbc-starrocks/src/test/java/org/apache/gravitino/catalog/starrocks/operation/TestStarRocksTableOperations.java index fb4a4c0ae4d..817015b08ab 100644 --- a/catalogs/catalog-jdbc-starrocks/src/test/java/org/apache/gravitino/catalog/starrocks/operation/TestStarRocksTableOperations.java +++ b/catalogs/catalog-jdbc-starrocks/src/test/java/org/apache/gravitino/catalog/starrocks/operation/TestStarRocksTableOperations.java @@ -18,8 +18,8 @@ */ package org.apache.gravitino.catalog.starrocks.operation; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; import com.google.common.collect.Maps; import java.sql.Connection; diff --git a/catalogs/catalog-jdbc-starrocks/src/test/java/org/apache/gravitino/catalog/starrocks/operation/TestStarRocksTablePartitionOperations.java b/catalogs/catalog-jdbc-starrocks/src/test/java/org/apache/gravitino/catalog/starrocks/operation/TestStarRocksTablePartitionOperations.java index 52b0f952201..d233b312607 100644 --- a/catalogs/catalog-jdbc-starrocks/src/test/java/org/apache/gravitino/catalog/starrocks/operation/TestStarRocksTablePartitionOperations.java +++ b/catalogs/catalog-jdbc-starrocks/src/test/java/org/apache/gravitino/catalog/starrocks/operation/TestStarRocksTablePartitionOperations.java @@ -18,10 +18,10 @@ */ package org.apache.gravitino.catalog.starrocks.operation; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertThrows; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; import com.google.common.collect.ImmutableMap; import java.sql.Connection; diff --git a/catalogs/catalog-jdbc-starrocks/src/test/java/org/apache/gravitino/catalog/starrocks/utils/TestStarRocksUtils.java b/catalogs/catalog-jdbc-starrocks/src/test/java/org/apache/gravitino/catalog/starrocks/utils/TestStarRocksUtils.java index e04db6f02cd..e72bfd0575a 100644 --- a/catalogs/catalog-jdbc-starrocks/src/test/java/org/apache/gravitino/catalog/starrocks/utils/TestStarRocksUtils.java +++ b/catalogs/catalog-jdbc-starrocks/src/test/java/org/apache/gravitino/catalog/starrocks/utils/TestStarRocksUtils.java @@ -18,9 +18,9 @@ */ package org.apache.gravitino.catalog.starrocks.utils; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; import java.util.Collections; import java.util.HashMap; diff --git a/catalogs/catalog-lakehouse-paimon/src/test/java/org/apache/gravitino/catalog/lakehouse/paimon/integration/test/CatalogPaimonBaseIT.java b/catalogs/catalog-lakehouse-paimon/src/test/java/org/apache/gravitino/catalog/lakehouse/paimon/integration/test/CatalogPaimonBaseIT.java index 00e97ac3043..e94faf01714 100644 --- a/catalogs/catalog-lakehouse-paimon/src/test/java/org/apache/gravitino/catalog/lakehouse/paimon/integration/test/CatalogPaimonBaseIT.java +++ b/catalogs/catalog-lakehouse-paimon/src/test/java/org/apache/gravitino/catalog/lakehouse/paimon/integration/test/CatalogPaimonBaseIT.java @@ -36,6 +36,7 @@ import java.util.Map; import java.util.Set; import org.apache.commons.lang3.ArrayUtils; +import org.apache.commons.lang3.StringUtils; import org.apache.gravitino.Catalog; import org.apache.gravitino.NameIdentifier; import org.apache.gravitino.Namespace; @@ -89,7 +90,6 @@ import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; -import org.junit.platform.commons.util.StringUtils; public abstract class CatalogPaimonBaseIT extends BaseIT { diff --git a/clients/cli/src/test/java/org/apache/gravitino/cli/TestCatalogCommands.java b/clients/cli/src/test/java/org/apache/gravitino/cli/TestCatalogCommands.java index 553820db1b7..95aff52a0ff 100644 --- a/clients/cli/src/test/java/org/apache/gravitino/cli/TestCatalogCommands.java +++ b/clients/cli/src/test/java/org/apache/gravitino/cli/TestCatalogCommands.java @@ -19,8 +19,8 @@ package org.apache.gravitino.cli; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertThrows; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.argThat; import static org.mockito.ArgumentMatchers.eq; diff --git a/clients/cli/src/test/java/org/apache/gravitino/cli/TestColumnCommands.java b/clients/cli/src/test/java/org/apache/gravitino/cli/TestColumnCommands.java index 521b1e2b61d..5e249097e74 100644 --- a/clients/cli/src/test/java/org/apache/gravitino/cli/TestColumnCommands.java +++ b/clients/cli/src/test/java/org/apache/gravitino/cli/TestColumnCommands.java @@ -19,8 +19,8 @@ package org.apache.gravitino.cli; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertThrows; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyBoolean; import static org.mockito.ArgumentMatchers.eq; diff --git a/clients/cli/src/test/java/org/apache/gravitino/cli/TestFilesetCommands.java b/clients/cli/src/test/java/org/apache/gravitino/cli/TestFilesetCommands.java index 9533069557a..b51aef0f148 100644 --- a/clients/cli/src/test/java/org/apache/gravitino/cli/TestFilesetCommands.java +++ b/clients/cli/src/test/java/org/apache/gravitino/cli/TestFilesetCommands.java @@ -19,8 +19,8 @@ package org.apache.gravitino.cli; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertThrows; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; import static org.mockito.ArgumentMatchers.isNull; import static org.mockito.Mockito.any; import static org.mockito.Mockito.doReturn; diff --git a/clients/cli/src/test/java/org/apache/gravitino/cli/TestGroupCommands.java b/clients/cli/src/test/java/org/apache/gravitino/cli/TestGroupCommands.java index eb8b923112c..871969dfc5c 100644 --- a/clients/cli/src/test/java/org/apache/gravitino/cli/TestGroupCommands.java +++ b/clients/cli/src/test/java/org/apache/gravitino/cli/TestGroupCommands.java @@ -19,8 +19,8 @@ package org.apache.gravitino.cli; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertThrows; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.ArgumentMatchers.isNull; diff --git a/clients/cli/src/test/java/org/apache/gravitino/cli/TestMetalakeCommands.java b/clients/cli/src/test/java/org/apache/gravitino/cli/TestMetalakeCommands.java index 4707d24387e..06d00492d55 100644 --- a/clients/cli/src/test/java/org/apache/gravitino/cli/TestMetalakeCommands.java +++ b/clients/cli/src/test/java/org/apache/gravitino/cli/TestMetalakeCommands.java @@ -19,8 +19,8 @@ package org.apache.gravitino.cli; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertThrows; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; import static org.mockito.Mockito.any; import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.eq; diff --git a/clients/cli/src/test/java/org/apache/gravitino/cli/TestOwnerCommands.java b/clients/cli/src/test/java/org/apache/gravitino/cli/TestOwnerCommands.java index f3c8115185a..73951a3d623 100644 --- a/clients/cli/src/test/java/org/apache/gravitino/cli/TestOwnerCommands.java +++ b/clients/cli/src/test/java/org/apache/gravitino/cli/TestOwnerCommands.java @@ -19,8 +19,8 @@ package org.apache.gravitino.cli; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertThrows; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyBoolean; import static org.mockito.ArgumentMatchers.eq; diff --git a/clients/cli/src/test/java/org/apache/gravitino/cli/TestParseType.java b/clients/cli/src/test/java/org/apache/gravitino/cli/TestParseType.java index 3bb9376ff5f..a2cbf12c8ca 100644 --- a/clients/cli/src/test/java/org/apache/gravitino/cli/TestParseType.java +++ b/clients/cli/src/test/java/org/apache/gravitino/cli/TestParseType.java @@ -19,9 +19,8 @@ package org.apache.gravitino.cli; -import static org.hamcrest.CoreMatchers.instanceOf; -import static org.hamcrest.MatcherAssert.assertThat; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertInstanceOf; import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertThrows; @@ -34,14 +33,14 @@ public class TestParseType { @Test public void testParseTypeVarcharWithLength() { Type type = ParseType.toType("varchar(10)"); - assertThat(type, instanceOf(Types.VarCharType.class)); + assertInstanceOf(Types.VarCharType.class, type); assertEquals(10, ((Types.VarCharType) type).length()); } @Test public void testParseTypeDecimalWithPrecisionAndScale() { Type type = ParseType.toType("decimal(10,5)"); - assertThat(type, instanceOf(Types.DecimalType.class)); + assertInstanceOf(Types.DecimalType.class, type); assertEquals(10, ((Types.DecimalType) type).precision()); assertEquals(5, ((Types.DecimalType) type).scale()); } @@ -49,15 +48,15 @@ public void testParseTypeDecimalWithPrecisionAndScale() { @Test public void testParseTypeVariant() { Type type = ParseType.toType("variant"); - assertThat(type, instanceOf(Types.VariantType.class)); + assertInstanceOf(Types.VariantType.class, type); } @Test public void testParseTypeListValidInput() { Type type = ParseType.toType("list(integer)"); - assertThat(type, instanceOf(Types.ListType.class)); + assertInstanceOf(Types.ListType.class, type); Type elementType = ((Types.ListType) type).elementType(); - assertThat(elementType, instanceOf(Types.IntegerType.class)); + assertInstanceOf(Types.IntegerType.class, elementType); } @Test @@ -72,11 +71,11 @@ public void testParseTypeListMalformedInput() { @Test public void testParseTypeMapValidInput() { Type type = ParseType.toType("map(string,integer)"); - assertThat(type, instanceOf(Types.MapType.class)); + assertInstanceOf(Types.MapType.class, type); Type keyType = ((Types.MapType) type).keyType(); Type valueType = ((Types.MapType) type).valueType(); - assertThat(keyType, instanceOf(Types.StringType.class)); - assertThat(valueType, instanceOf(Types.IntegerType.class)); + assertInstanceOf(Types.StringType.class, keyType); + assertInstanceOf(Types.IntegerType.class, valueType); } @Test @@ -110,47 +109,47 @@ public void testParseTypeMalformedInput() { @Test public void testParseTypeListWithSpaces() { Type type = ParseType.toType("list( integer )"); - assertThat(type, instanceOf(Types.ListType.class)); + assertInstanceOf(Types.ListType.class, type); Type elementType = ((Types.ListType) type).elementType(); - assertThat(elementType, instanceOf(Types.IntegerType.class)); + assertInstanceOf(Types.IntegerType.class, elementType); } @Test public void testParseTypeMapWithSpaces() { Type type = ParseType.toType("map( string , integer )"); - assertThat(type, instanceOf(Types.MapType.class)); + assertInstanceOf(Types.MapType.class, type); Type keyType = ((Types.MapType) type).keyType(); Type valueType = ((Types.MapType) type).valueType(); - assertThat(keyType, instanceOf(Types.StringType.class)); - assertThat(valueType, instanceOf(Types.IntegerType.class)); + assertInstanceOf(Types.StringType.class, keyType); + assertInstanceOf(Types.IntegerType.class, valueType); } @Test public void testParseTypeNestedList() { Type type = ParseType.toType("list(list(integer))"); - assertThat(type, instanceOf(Types.ListType.class)); + assertInstanceOf(Types.ListType.class, type); Type innerList = ((Types.ListType) type).elementType(); - assertThat(innerList, instanceOf(Types.ListType.class)); + assertInstanceOf(Types.ListType.class, innerList); Type elementType = ((Types.ListType) innerList).elementType(); - assertThat(elementType, instanceOf(Types.IntegerType.class)); + assertInstanceOf(Types.IntegerType.class, elementType); } @Test public void testParseTypeMapWithNestedValue() { Type type = ParseType.toType("map(string,list(integer))"); - assertThat(type, instanceOf(Types.MapType.class)); + assertInstanceOf(Types.MapType.class, type); Type keyType = ((Types.MapType) type).keyType(); Type valueType = ((Types.MapType) type).valueType(); - assertThat(keyType, instanceOf(Types.StringType.class)); - assertThat(valueType, instanceOf(Types.ListType.class)); + assertInstanceOf(Types.StringType.class, keyType); + assertInstanceOf(Types.ListType.class, valueType); Type elementType = ((Types.ListType) valueType).elementType(); - assertThat(elementType, instanceOf(Types.IntegerType.class)); + assertInstanceOf(Types.IntegerType.class, elementType); } @Test public void testParseTypeDecimalWithSpaces() { Type type = ParseType.toType("decimal(10, 5)"); - assertThat(type, instanceOf(Types.DecimalType.class)); + assertInstanceOf(Types.DecimalType.class, type); assertEquals(10, ((Types.DecimalType) type).precision()); assertEquals(5, ((Types.DecimalType) type).scale()); } @@ -158,7 +157,7 @@ public void testParseTypeDecimalWithSpaces() { @Test public void testParseTypeVarcharWithSpaces() { Type type = ParseType.toType("varchar( 10 )"); - assertThat(type, instanceOf(Types.VarCharType.class)); + assertInstanceOf(Types.VarCharType.class, type); assertEquals(10, ((Types.VarCharType) type).length()); } @@ -166,7 +165,7 @@ public void testParseTypeVarcharWithSpaces() { @Test public void testParseTypeDecimalWithPrecisionOnly() { Type type = ParseType.toType("decimal(10)"); - assertThat(type, instanceOf(Types.DecimalType.class)); + assertInstanceOf(Types.DecimalType.class, type); assertEquals(10, ((Types.DecimalType) type).precision()); assertEquals(0, ((Types.DecimalType) type).scale()); } @@ -175,7 +174,7 @@ public void testParseTypeDecimalWithPrecisionOnly() { @Test public void testParseTypeDecimalWithPrecisionOnlySpacesAndCase() { Type type = ParseType.toType("DECIMAL ( 10 ) "); - assertThat(type, instanceOf(Types.DecimalType.class)); + assertInstanceOf(Types.DecimalType.class, type); assertEquals(10, ((Types.DecimalType) type).precision()); assertEquals(0, ((Types.DecimalType) type).scale()); } diff --git a/clients/cli/src/test/java/org/apache/gravitino/cli/TestRoleCommands.java b/clients/cli/src/test/java/org/apache/gravitino/cli/TestRoleCommands.java index 8d21679688f..e0c48bf6c93 100644 --- a/clients/cli/src/test/java/org/apache/gravitino/cli/TestRoleCommands.java +++ b/clients/cli/src/test/java/org/apache/gravitino/cli/TestRoleCommands.java @@ -19,8 +19,8 @@ package org.apache.gravitino.cli; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertThrows; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; import static org.mockito.ArgumentMatchers.argThat; import static org.mockito.Mockito.any; import static org.mockito.Mockito.doReturn; diff --git a/clients/cli/src/test/java/org/apache/gravitino/cli/TestSchemaCommands.java b/clients/cli/src/test/java/org/apache/gravitino/cli/TestSchemaCommands.java index 12b3d72ba1f..89abc66f0e9 100644 --- a/clients/cli/src/test/java/org/apache/gravitino/cli/TestSchemaCommands.java +++ b/clients/cli/src/test/java/org/apache/gravitino/cli/TestSchemaCommands.java @@ -19,9 +19,9 @@ package org.apache.gravitino.cli; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertThrows; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.doReturn; diff --git a/clients/cli/src/test/java/org/apache/gravitino/cli/TestTableCommands.java b/clients/cli/src/test/java/org/apache/gravitino/cli/TestTableCommands.java index 77166babd29..92a63591409 100644 --- a/clients/cli/src/test/java/org/apache/gravitino/cli/TestTableCommands.java +++ b/clients/cli/src/test/java/org/apache/gravitino/cli/TestTableCommands.java @@ -19,8 +19,8 @@ package org.apache.gravitino.cli; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertThrows; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.ArgumentMatchers.isNull; diff --git a/clients/cli/src/test/java/org/apache/gravitino/cli/TestTagCommands.java b/clients/cli/src/test/java/org/apache/gravitino/cli/TestTagCommands.java index 4a7549510b3..dde2f00417d 100644 --- a/clients/cli/src/test/java/org/apache/gravitino/cli/TestTagCommands.java +++ b/clients/cli/src/test/java/org/apache/gravitino/cli/TestTagCommands.java @@ -19,8 +19,8 @@ package org.apache.gravitino.cli; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertThrows; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; import static org.mockito.ArgumentMatchers.argThat; import static org.mockito.ArgumentMatchers.isNull; import static org.mockito.Mockito.any; diff --git a/clients/cli/src/test/java/org/apache/gravitino/cli/TestTopicCommands.java b/clients/cli/src/test/java/org/apache/gravitino/cli/TestTopicCommands.java index 695edb9dd48..94701978802 100644 --- a/clients/cli/src/test/java/org/apache/gravitino/cli/TestTopicCommands.java +++ b/clients/cli/src/test/java/org/apache/gravitino/cli/TestTopicCommands.java @@ -19,8 +19,8 @@ package org.apache.gravitino.cli; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertThrows; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.ArgumentMatchers.isNull; diff --git a/clients/cli/src/test/java/org/apache/gravitino/cli/TestUserCommands.java b/clients/cli/src/test/java/org/apache/gravitino/cli/TestUserCommands.java index 2d004ab9599..694945d44ff 100644 --- a/clients/cli/src/test/java/org/apache/gravitino/cli/TestUserCommands.java +++ b/clients/cli/src/test/java/org/apache/gravitino/cli/TestUserCommands.java @@ -19,8 +19,8 @@ package org.apache.gravitino.cli; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertThrows; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.ArgumentMatchers.isNull; diff --git a/clients/cli/src/test/java/org/apache/gravitino/cli/command/TestOwnerDetails.java b/clients/cli/src/test/java/org/apache/gravitino/cli/command/TestOwnerDetails.java index 94222348b10..ef369ad8365 100644 --- a/clients/cli/src/test/java/org/apache/gravitino/cli/command/TestOwnerDetails.java +++ b/clients/cli/src/test/java/org/apache/gravitino/cli/command/TestOwnerDetails.java @@ -18,10 +18,10 @@ */ package org.apache.gravitino.cli.command; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; import org.apache.commons.cli.CommandLine; import org.apache.gravitino.cli.CommandContext; @@ -29,23 +29,23 @@ import org.apache.gravitino.cli.Main; import org.apache.gravitino.cli.commands.Command; import org.apache.gravitino.cli.commands.OwnerDetails; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; import org.mockito.Mockito; public class TestOwnerDetails { private CommandContext context; - @Before + @BeforeEach public void setUp() { CommandLine mockCommandLine = Mockito.mock(CommandLine.class); context = new CommandContext(mockCommandLine); Main.useExit = false; } - @After + @AfterEach public void tearDown() { Main.useExit = true; } @@ -62,11 +62,7 @@ public void testValidate_withValidEntityType_shouldPass() { public void testValidate_withInvalidEntityType_shouldExit() { OwnerDetails command = new OwnerDetails(context, "metalake1", "entity1", "INVALID_TYPE"); - try { - command.validate(); - fail("Expected RuntimeException due to intercepted exit"); - } catch (RuntimeException e) { - assertTrue(e.getMessage().contains("Exit with code")); - } + RuntimeException e = assertThrows(RuntimeException.class, command::validate); + assertTrue(e.getMessage().contains("Exit with code")); } } diff --git a/clients/cli/src/test/java/org/apache/gravitino/cli/commands/TestListProperties.java b/clients/cli/src/test/java/org/apache/gravitino/cli/commands/TestListProperties.java index 1a1170464f9..fdda21444bc 100644 --- a/clients/cli/src/test/java/org/apache/gravitino/cli/commands/TestListProperties.java +++ b/clients/cli/src/test/java/org/apache/gravitino/cli/commands/TestListProperties.java @@ -19,7 +19,7 @@ package org.apache.gravitino.cli.commands; -import static org.junit.Assert.assertEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; diff --git a/clients/cli/src/test/java/org/apache/gravitino/cli/output/TestPlainFormat.java b/clients/cli/src/test/java/org/apache/gravitino/cli/output/TestPlainFormat.java index e773031a2c9..0a76fffef14 100644 --- a/clients/cli/src/test/java/org/apache/gravitino/cli/output/TestPlainFormat.java +++ b/clients/cli/src/test/java/org/apache/gravitino/cli/output/TestPlainFormat.java @@ -19,7 +19,7 @@ package org.apache.gravitino.cli.output; -import static org.junit.Assert.assertThrows; +import static org.junit.jupiter.api.Assertions.assertThrows; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; diff --git a/clients/cli/src/test/java/org/apache/gravitino/cli/output/TestTableFormat.java b/clients/cli/src/test/java/org/apache/gravitino/cli/output/TestTableFormat.java index 48fe9e580e2..a858b26620b 100644 --- a/clients/cli/src/test/java/org/apache/gravitino/cli/output/TestTableFormat.java +++ b/clients/cli/src/test/java/org/apache/gravitino/cli/output/TestTableFormat.java @@ -21,7 +21,7 @@ import static org.apache.gravitino.rel.Column.DEFAULT_VALUE_NOT_SET; import static org.apache.gravitino.rel.expressions.NamedReference.field; -import static org.junit.Assert.assertThrows; +import static org.junit.jupiter.api.Assertions.assertThrows; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; diff --git a/clients/client-java/src/test/java/org/apache/gravitino/client/TestGravitinoVersion.java b/clients/client-java/src/test/java/org/apache/gravitino/client/TestGravitinoVersion.java index 9da527a97a7..bd5a824eb45 100644 --- a/clients/client-java/src/test/java/org/apache/gravitino/client/TestGravitinoVersion.java +++ b/clients/client-java/src/test/java/org/apache/gravitino/client/TestGravitinoVersion.java @@ -18,8 +18,8 @@ */ package org.apache.gravitino.client; -import static org.junit.Assert.assertFalse; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; diff --git a/clients/client-java/src/test/java/org/apache/gravitino/client/integration/test/authorization/CatalogAuthorizationIT.java b/clients/client-java/src/test/java/org/apache/gravitino/client/integration/test/authorization/CatalogAuthorizationIT.java index 004b20fdc94..22dae4f4b11 100644 --- a/clients/client-java/src/test/java/org/apache/gravitino/client/integration/test/authorization/CatalogAuthorizationIT.java +++ b/clients/client-java/src/test/java/org/apache/gravitino/client/integration/test/authorization/CatalogAuthorizationIT.java @@ -17,9 +17,9 @@ package org.apache.gravitino.client.integration.test.authorization; -import static org.junit.Assert.assertThrows; import static org.junit.jupiter.api.Assertions.assertArrayEquals; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; import com.google.common.collect.ImmutableList; import com.google.common.collect.Maps; @@ -72,21 +72,21 @@ public void testCreateCatalog() throws Exception { Map properties = Maps.newHashMap(); properties.put("metastore.uris", hmsUri); assertThrows( - "Can not access metadata {" + catalog1 + "}.", ForbiddenException.class, () -> { normalUserClient .loadMetalake(METALAKE) .testConnection(catalog1, Catalog.Type.RELATIONAL, "hive", "comment", properties); - }); + }, + "Can not access metadata {" + catalog1 + "}."); assertThrows( - "Can not access metadata {" + catalog1 + "}.", ForbiddenException.class, () -> { normalUserClient .loadMetalake(METALAKE) .createCatalog(catalog1, Catalog.Type.RELATIONAL, "hive", "comment", properties); - }); + }, + "Can not access metadata {" + catalog1 + "}."); client .loadMetalake(METALAKE) .testConnection(catalog1, Catalog.Type.RELATIONAL, "hive", "comment", properties); @@ -100,13 +100,13 @@ public void testCreateCatalog() throws Exception { .loadMetalake(METALAKE) .createCatalog(catalog2, Catalog.Type.RELATIONAL, "hive", "comment", properties); assertThrows( - "Can not access metadata {" + catalog1 + "}.", ForbiddenException.class, () -> { normalUserClient .loadMetalake(METALAKE) .createCatalog(catalog1, Catalog.Type.RELATIONAL, "hive", "comment", properties); - }); + }, + "Can not access metadata {" + catalog1 + "}."); } @Test @@ -141,9 +141,9 @@ public void testTestExistingCatalogConnection() throws Exception { GravitinoMetalake adminMetalake = client.loadMetalake(METALAKE); CatalogChange proposedChange = CatalogChange.updateComment("proposed comment"); assertThrows( - "Can not access metadata {" + catalog1 + "}.", ForbiddenException.class, - () -> normalUserClient.loadMetalake(METALAKE).testConnection(catalog1)); + () -> normalUserClient.loadMetalake(METALAKE).testConnection(catalog1), + "Can not access metadata {" + catalog1 + "}."); // USE_CATALOG allows testing the stored configuration, but not proposed changes. String role = "testConnectionRole"; @@ -158,9 +158,9 @@ public void testTestExistingCatalogConnection() throws Exception { GravitinoMetalake normalUserMetalake = normalUserClient.loadMetalake(METALAKE); normalUserMetalake.testConnection(catalog1); assertThrows( - "Can not access metadata {" + catalog1 + "}.", ForbiddenException.class, - () -> normalUserMetalake.testConnection(catalog1, proposedChange)); + () -> normalUserMetalake.testConnection(catalog1, proposedChange), + "Can not access metadata {" + catalog1 + "}."); } finally { adminMetalake.revokeRolesFromUser(ImmutableList.of(role), NORMAL_USER); adminMetalake.deleteRole(role); @@ -178,11 +178,11 @@ public void testDeleteCatalog() { assertEquals(2, catalogs.length); assertArrayEquals(new String[] {catalog1, catalog2}, catalogs); assertThrows( - "Can not access metadata {" + catalog1 + "}.", ForbiddenException.class, () -> { normalUserClient.loadMetalake(METALAKE).dropCatalog(catalog1, true); - }); + }, + "Can not access metadata {" + catalog1 + "}."); client.loadMetalake(METALAKE).dropCatalog(catalog1, true); catalogs = client.loadMetalake(METALAKE).listCatalogs(); assertEquals(1, catalogs.length); diff --git a/clients/client-java/src/test/java/org/apache/gravitino/client/integration/test/authorization/FilesetAuthorizationIT.java b/clients/client-java/src/test/java/org/apache/gravitino/client/integration/test/authorization/FilesetAuthorizationIT.java index a651dda5148..6d774b7b3c0 100644 --- a/clients/client-java/src/test/java/org/apache/gravitino/client/integration/test/authorization/FilesetAuthorizationIT.java +++ b/clients/client-java/src/test/java/org/apache/gravitino/client/integration/test/authorization/FilesetAuthorizationIT.java @@ -17,9 +17,9 @@ package org.apache.gravitino.client.integration.test.authorization; -import static org.junit.Assert.assertThrows; import static org.junit.jupiter.api.Assertions.assertArrayEquals; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; import com.google.common.collect.ImmutableList; @@ -92,7 +92,6 @@ public void startIntegrationTest() throws Exception { .createSchema(SCHEMA, "test", new HashMap<>()); // try to load the schema as normal user, expect failure assertThrows( - "Can not access metadata {" + CATALOG + "." + SCHEMA + "}.", ForbiddenException.class, () -> { normalUserClient @@ -100,7 +99,8 @@ public void startIntegrationTest() throws Exception { .loadCatalog(CATALOG) .asSchemas() .loadSchema(SCHEMA); - }); + }, + "Can not access metadata {" + CATALOG + "." + SCHEMA + "}."); // grant tester privilege List securableObjects = new ArrayList<>(); GravitinoMetalake gravitinoMetalake = client.loadMetalake(METALAKE); @@ -113,11 +113,11 @@ public void startIntegrationTest() throws Exception { Catalog catalogLoadByNormalUser = normalUserClient.loadMetalake(METALAKE).loadCatalog(CATALOG); assertEquals(CATALOG, catalogLoadByNormalUser.name()); assertThrows( - "Can not access metadata {" + CATALOG + "." + SCHEMA + "}.", ForbiddenException.class, () -> { catalogLoadByNormalUser.asSchemas().loadSchema(SCHEMA); - }); + }, + "Can not access metadata {" + CATALOG + "." + SCHEMA + "}."); httpClient = HttpClients.createDefault(); } @@ -147,7 +147,6 @@ public void testCreateFileset() { FilesetCatalog filesetCatalogNormalUser = normalUserClient.loadMetalake(METALAKE).loadCatalog(CATALOG).asFilesetCatalog(); assertThrows( - "Can not access metadata {" + CATALOG + "." + SCHEMA + "}.", ForbiddenException.class, () -> { filesetCatalogNormalUser.createFileset( @@ -157,12 +156,13 @@ public void testCreateFileset() { Fileset.Type.MANAGED, storageLocation(GravitinoITUtils.genRandomName("FilesetAuthorizationIT_fileset2")), new HashMap<>()); - }); + }, + "Can not access metadata {" + CATALOG + "." + SCHEMA + "}."); assertThrows( - "Can not access metadata {" + CATALOG + "." + SCHEMA + "}.", ForbiddenException.class, - () -> filesetCatalogNormalUser.listFilesets(Namespace.of(SCHEMA))); + () -> filesetCatalogNormalUser.listFilesets(Namespace.of(SCHEMA)), + "Can not access metadata {" + CATALOG + "." + SCHEMA + "}."); // grant privileges GravitinoMetalake gravitinoMetalake = client.loadMetalake(METALAKE); @@ -219,11 +219,11 @@ public void testLoadFileset() { normalUserClient.loadMetalake(METALAKE).loadCatalog(CATALOG).asFilesetCatalog(); // normal user can load fileset2 and fileset3, but not fileset1 assertThrows( - String.format("Can not access metadata {%s.%s.%s}.", CATALOG, SCHEMA, "fileset1"), ForbiddenException.class, () -> { filesetCatalogNormalUser.loadFileset(NameIdentifier.of(SCHEMA, "fileset1")); - }); + }, + String.format("Can not access metadata {%s.%s.%s}.", CATALOG, SCHEMA, "fileset1")); Fileset fileset2 = filesetCatalogNormalUser.loadFileset(NameIdentifier.of(SCHEMA, "fileset2")); assertEquals("fileset2", fileset2.name()); Fileset fileset3 = filesetCatalogNormalUser.loadFileset(NameIdentifier.of(SCHEMA, "fileset3")); @@ -249,12 +249,12 @@ public void testAlterFileset() { // normal user cannot alter fileset1 (no privilege) assertThrows( - String.format("Can not access metadata {%s.%s.%s}.", CATALOG, SCHEMA, "fileset1"), ForbiddenException.class, () -> { filesetCatalogNormalUser.alterFileset( NameIdentifier.of(SCHEMA, "fileset1"), FilesetChange.setProperty("key", "value")); - }); + }, + String.format("Can not access metadata {%s.%s.%s}.", CATALOG, SCHEMA, "fileset1")); // grant normal user write privilege on fileset1 gravitinoMetalake.grantPrivilegesToRole( role, @@ -280,11 +280,11 @@ public void testGetFileLocation() { // normal user cannot alter fileset1 (no privilege) assertThrows( - String.format("Can not access metadata {%s.%s.%s}.", CATALOG, SCHEMA, "fileset1"), ForbiddenException.class, () -> { filesetCatalogNormalUser.getFileLocation(NameIdentifier.of(SCHEMA, "fileset1"), "/test"); - }); + }, + String.format("Can not access metadata {%s.%s.%s}.", CATALOG, SCHEMA, "fileset1")); // grant normal user owner privilege on fileset4 gravitinoMetalake.setOwner( MetadataObjects.of( @@ -315,9 +315,9 @@ public void testListFiles() throws IOException { // normal user cannot alter fileset1 (no privilege) assertThrows( - String.format("Can not access metadata {%s.%s.%s}.", CATALOG, SCHEMA, "fileset1"), ForbiddenException.class, - () -> getFileInfos("fileset1", "/test", "", NORMAL_USER)); + () -> getFileInfos("fileset1", "/test", "", NORMAL_USER), + String.format("Can not access metadata {%s.%s.%s}.", CATALOG, SCHEMA, "fileset1")); // grant normal user owner privilege on fileset4 gravitinoMetalake.setOwner( MetadataObjects.of( @@ -348,11 +348,11 @@ public void testDropFileset() { Owner.Type.USER); // normal user cannot drop fileset1 assertThrows( - String.format("Can not access metadata {%s.%s.%s}.", CATALOG, SCHEMA, "fileset1"), ForbiddenException.class, () -> { filesetCatalogNormalUser.dropFileset(NameIdentifier.of(SCHEMA, "fileset1")); - }); + }, + String.format("Can not access metadata {%s.%s.%s}.", CATALOG, SCHEMA, "fileset1")); // normal user can drop fileset2 and fileset3 (they created them) filesetCatalogNormalUser.dropFileset(NameIdentifier.of(SCHEMA, "fileset2")); filesetCatalogNormalUser.dropFileset(NameIdentifier.of(SCHEMA, "fileset3")); diff --git a/clients/client-java/src/test/java/org/apache/gravitino/client/integration/test/authorization/FunctionAuthorizationIT.java b/clients/client-java/src/test/java/org/apache/gravitino/client/integration/test/authorization/FunctionAuthorizationIT.java index 6493e1278e0..b72abd4ae7f 100644 --- a/clients/client-java/src/test/java/org/apache/gravitino/client/integration/test/authorization/FunctionAuthorizationIT.java +++ b/clients/client-java/src/test/java/org/apache/gravitino/client/integration/test/authorization/FunctionAuthorizationIT.java @@ -17,8 +17,8 @@ package org.apache.gravitino.client.integration.test.authorization; -import static org.junit.Assert.assertThrows; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; diff --git a/clients/client-java/src/test/java/org/apache/gravitino/client/integration/test/authorization/GroupAuthorizationIT.java b/clients/client-java/src/test/java/org/apache/gravitino/client/integration/test/authorization/GroupAuthorizationIT.java index ce50d4f1247..f5f1f46d566 100644 --- a/clients/client-java/src/test/java/org/apache/gravitino/client/integration/test/authorization/GroupAuthorizationIT.java +++ b/clients/client-java/src/test/java/org/apache/gravitino/client/integration/test/authorization/GroupAuthorizationIT.java @@ -17,8 +17,8 @@ package org.apache.gravitino.client.integration.test.authorization; -import static org.junit.Assert.assertThrows; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; import com.google.common.collect.ImmutableList; import com.google.common.collect.Maps; @@ -55,11 +55,11 @@ public class GroupAuthorizationIT extends BaseRestApiAuthorizationIT { @Order(1) public void testCreateGroup() { assertThrows( - "Current user access metadata {testMetalake}", ForbiddenException.class, () -> { normalUserClient.loadMetalake(METALAKE).addGroup("group1"); - }); + }, + "Current user access metadata {testMetalake}"); client.loadMetalake(METALAKE).addGroup("group1"); client.loadMetalake(METALAKE).addGroup("group2"); } @@ -116,11 +116,11 @@ public void testBulkGroupInterfaces() throws Exception { @Order(3) public void testRemoveGroup() { assertThrows( - "Current user access metadata {testMetalake}", ForbiddenException.class, () -> { normalUserClient.loadMetalake(METALAKE).removeGroup("group1"); - }); + }, + "Current user access metadata {testMetalake}"); GravitinoMetalake gravitinoMetalake = client.loadMetalake(METALAKE); // owner can remove user gravitinoMetalake.removeGroup("group1"); diff --git a/clients/client-java/src/test/java/org/apache/gravitino/client/integration/test/authorization/HierarchicalSchemaAuthorizationIT.java b/clients/client-java/src/test/java/org/apache/gravitino/client/integration/test/authorization/HierarchicalSchemaAuthorizationIT.java index 7e64f66501a..fce33a279a6 100644 --- a/clients/client-java/src/test/java/org/apache/gravitino/client/integration/test/authorization/HierarchicalSchemaAuthorizationIT.java +++ b/clients/client-java/src/test/java/org/apache/gravitino/client/integration/test/authorization/HierarchicalSchemaAuthorizationIT.java @@ -17,9 +17,9 @@ package org.apache.gravitino.client.integration.test.authorization; -import static org.junit.Assert.assertThrows; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; import com.google.common.collect.ImmutableList; diff --git a/clients/client-java/src/test/java/org/apache/gravitino/client/integration/test/authorization/MetadataObjectRoleAuthorizationIT.java b/clients/client-java/src/test/java/org/apache/gravitino/client/integration/test/authorization/MetadataObjectRoleAuthorizationIT.java index 21000208e91..b90f7ffc318 100644 --- a/clients/client-java/src/test/java/org/apache/gravitino/client/integration/test/authorization/MetadataObjectRoleAuthorizationIT.java +++ b/clients/client-java/src/test/java/org/apache/gravitino/client/integration/test/authorization/MetadataObjectRoleAuthorizationIT.java @@ -17,7 +17,7 @@ package org.apache.gravitino.client.integration.test.authorization; -import static org.junit.Assert.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertArrayEquals; import com.google.common.collect.ImmutableList; import com.google.common.collect.Maps; diff --git a/clients/client-java/src/test/java/org/apache/gravitino/client/integration/test/authorization/MetalakeAuthorizationIT.java b/clients/client-java/src/test/java/org/apache/gravitino/client/integration/test/authorization/MetalakeAuthorizationIT.java index 70810c251ad..ae2c05e3e90 100644 --- a/clients/client-java/src/test/java/org/apache/gravitino/client/integration/test/authorization/MetalakeAuthorizationIT.java +++ b/clients/client-java/src/test/java/org/apache/gravitino/client/integration/test/authorization/MetalakeAuthorizationIT.java @@ -17,7 +17,7 @@ package org.apache.gravitino.client.integration.test.authorization; -import static org.junit.Assert.assertThrows; +import static org.junit.jupiter.api.Assertions.assertThrows; import java.util.Arrays; import java.util.Comparator; @@ -73,11 +73,11 @@ protected void putServiceAdmin() { @Order(1) public void testCreateMetalake() { assertThrows( - "Only service admins can create metalakes, current user can't create the metalake, you should configure it in the server configuration first", ForbiddenException.class, () -> { normalUserClient.createMetalake(testMetalake2, "", new HashMap<>()); - }); + }, + "Only service admins can create metalakes, current user can't create the metalake, you should configure it in the server configuration first"); serviceAdminClient.createMetalake(testMetalake2, "", new HashMap<>()); serviceAdminClient.createMetalake(testMetalake3, "", new HashMap<>()); } @@ -110,44 +110,44 @@ public void testLoadMetalake() { serviceAdminClient.loadMetalake(testMetalake3); normalUserClient.loadMetalake(METALAKE); assertThrows( - "Current user access metadata {testMetalake2}", ForbiddenException.class, () -> { normalUserClient.loadMetalake(testMetalake2); - }); + }, + "Current user access metadata {testMetalake2}"); assertThrows( - "Current user access metadata {testMetalake3}", ForbiddenException.class, () -> { normalUserClient.loadMetalake(testMetalake3); - }); + }, + "Current user access metadata {testMetalake3}"); serviceAdminButNotOwnerClient.loadMetalake(METALAKE); serviceAdminButNotOwnerClient.loadMetalake(testMetalake2); assertThrows( - "Current user access metadata {testMetalake3}", ForbiddenException.class, () -> { serviceAdminButNotOwnerClient.loadMetalake(testMetalake3); - }); + }, + "Current user access metadata {testMetalake3}"); } @Test @Order(4) public void testSetMetalake() { assertThrows( - "Current user access metadata {testMetalake2}", ForbiddenException.class, () -> { serviceAdminButNotOwnerClient.alterMetalake( testMetalake2, MetalakeChange.setProperty("key1", "value1")); - }); + }, + "Current user access metadata {testMetalake2}"); assertThrows( - "Current user access metadata {testMetalake2}", ForbiddenException.class, () -> { normalUserClient.alterMetalake( testMetalake2, MetalakeChange.setProperty("key1", "value1")); - }); + }, + "Current user access metadata {testMetalake2}"); serviceAdminClient.alterMetalake(testMetalake2, MetalakeChange.setProperty("key1", "value1")); } @@ -155,17 +155,17 @@ public void testSetMetalake() { @Order(5) public void testDropMetalake() { assertThrows( - "Current user access metadata {testMetalake2}", ForbiddenException.class, () -> { serviceAdminButNotOwnerClient.dropMetalake(testMetalake3, true); - }); + }, + "Current user access metadata {testMetalake2}"); assertThrows( - "Current user access metadata {testMetalake2}", ForbiddenException.class, () -> { normalUserClient.dropMetalake(testMetalake3, true); - }); + }, + "Current user access metadata {testMetalake2}"); serviceAdminClient.dropMetalake(testMetalake3, true); } diff --git a/clients/client-java/src/test/java/org/apache/gravitino/client/integration/test/authorization/ModelAuthorizationIT.java b/clients/client-java/src/test/java/org/apache/gravitino/client/integration/test/authorization/ModelAuthorizationIT.java index 7231c6efaba..72216d13b4c 100644 --- a/clients/client-java/src/test/java/org/apache/gravitino/client/integration/test/authorization/ModelAuthorizationIT.java +++ b/clients/client-java/src/test/java/org/apache/gravitino/client/integration/test/authorization/ModelAuthorizationIT.java @@ -17,9 +17,9 @@ package org.apache.gravitino.client.integration.test.authorization; -import static org.junit.Assert.assertThrows; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertLinesMatch; +import static org.junit.jupiter.api.Assertions.assertThrows; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; @@ -70,7 +70,6 @@ public void startIntegrationTest() throws Exception { .createSchema(SCHEMA, "test", new HashMap<>()); // try to load the schema as normal user, expect failure assertThrows( - "Can not access metadata {" + CATALOG + "." + SCHEMA + "}.", ForbiddenException.class, () -> { normalUserClient @@ -78,7 +77,8 @@ public void startIntegrationTest() throws Exception { .loadCatalog(CATALOG) .asSchemas() .loadSchema(SCHEMA); - }); + }, + "Can not access metadata {" + CATALOG + "." + SCHEMA + "}."); // grant tester privilege List securableObjects = new ArrayList<>(); GravitinoMetalake gravitinoMetalake = client.loadMetalake(METALAKE); @@ -91,11 +91,11 @@ public void startIntegrationTest() throws Exception { Catalog catalogLoadByNormalUser = normalUserClient.loadMetalake(METALAKE).loadCatalog(CATALOG); assertEquals(CATALOG, catalogLoadByNormalUser.name()); assertThrows( - "Can not access metadata {" + CATALOG + "." + SCHEMA + "}.", ForbiddenException.class, () -> { catalogLoadByNormalUser.asSchemas().loadSchema(SCHEMA); - }); + }, + "Can not access metadata {" + CATALOG + "." + SCHEMA + "}."); } @Test @@ -107,18 +107,18 @@ public void testCreateModel() { ModelCatalog normalUserCatalog = normalUserClient.loadMetalake(METALAKE).loadCatalog(CATALOG).asModelCatalog(); assertThrows( - "Can not access metadata {" + METALAKE + "," + CATALOG + "." + SCHEMA + "}.", ForbiddenException.class, () -> { normalUserCatalog.registerModel(NameIdentifier.of(SCHEMA, "model2"), "", new HashMap<>()); - }); + }, + "Can not access metadata {" + METALAKE + "," + CATALOG + "." + SCHEMA + "}."); assertThrows( - "Can not access metadata {" + METALAKE + "," + CATALOG + "." + SCHEMA + "}.", ForbiddenException.class, () -> { normalUserCatalog.listModels(Namespace.of(SCHEMA)); - }); + }, + "Can not access metadata {" + METALAKE + "," + CATALOG + "." + SCHEMA + "}."); GravitinoMetalake gravitinoMetalake = client.loadMetalake(METALAKE); // Test Case 1: Grant using NEW privilege name (REGISTER_MODEL) @@ -185,11 +185,11 @@ public void testLoadModel() { normalUserClient.loadMetalake(METALAKE).loadCatalog(CATALOG); ModelCatalog modelCatalog = catalogEntityLoadByNormalUser.asModelCatalog(); assertThrows( - "Can not access metadata {" + METALAKE + "," + CATALOG + "." + SCHEMA + "}.", ForbiddenException.class, () -> { modelCatalog.getModel(NameIdentifier.of(SCHEMA, "model1")); - }); + }, + "Can not access metadata {" + METALAKE + "," + CATALOG + "." + SCHEMA + "}."); // test grant use model GravitinoMetalake gravitinoMetalake = client.loadMetalake(METALAKE); gravitinoMetalake.grantPrivilegesToRole( @@ -212,12 +212,12 @@ public void testAlterModel() { normalUserClient.loadMetalake(METALAKE).loadCatalog(CATALOG); ModelCatalog modelCatalogLoadByNormalUser = catalogEntityLoadByNormalUser.asModelCatalog(); assertThrows( - "Can not access metadata {" + METALAKE + "," + CATALOG + "." + SCHEMA + "}.", ForbiddenException.class, () -> { modelCatalogLoadByNormalUser.alterModel( NameIdentifier.of(SCHEMA, "model1"), new ModelChange.RenameModel("model5")); - }); + }, + "Can not access metadata {" + METALAKE + "," + CATALOG + "." + SCHEMA + "}."); ModelCatalog modelCatalog = client.loadMetalake(METALAKE).loadCatalog(CATALOG).asModelCatalog(); modelCatalog.alterModel( NameIdentifier.of(SCHEMA, "model1"), new ModelChange.RenameModel("model4")); @@ -234,11 +234,11 @@ public void testDropModel() { normalUserClient.loadMetalake(METALAKE).loadCatalog(CATALOG); ModelCatalog modelCatalogLoadByNormalUser = catalogEntityLoadByNormalUser.asModelCatalog(); assertThrows( - "Can not access metadata {" + METALAKE + "," + CATALOG + "." + SCHEMA + "}.", ForbiddenException.class, () -> { modelCatalogLoadByNormalUser.deleteModel(NameIdentifier.of(SCHEMA, "model5")); - }); + }, + "Can not access metadata {" + METALAKE + "," + CATALOG + "." + SCHEMA + "}."); modelCatalog.deleteModel(NameIdentifier.of(SCHEMA, "model5")); } @@ -251,7 +251,6 @@ public void testLinkModel() { normalUserClient.loadMetalake(METALAKE).loadCatalog(CATALOG); ModelCatalog modelCatalogLoadByNormalUser = catalogEntityLoadByNormalUser.asModelCatalog(); assertThrows( - "Can not access metadata {" + METALAKE + "," + CATALOG + "." + SCHEMA + "model1" + "}.", ForbiddenException.class, () -> { modelCatalogLoadByNormalUser.linkModelVersion( @@ -260,7 +259,8 @@ public void testLinkModel() { new String[] {"alias2"}, "comment2", null); - }); + }, + "Can not access metadata {" + METALAKE + "," + CATALOG + "." + SCHEMA + "model1" + "}."); GravitinoMetalake gravitinoMetalake = client.loadMetalake(METALAKE); // Test Case 1: Grant using NEW privilege name (LINK_MODEL_VERSION) @@ -331,9 +331,9 @@ public void testListModelVersion() { int[] versions = modelCatalog.listModelVersions(NameIdentifier.of(SCHEMA, "model1")); assertEquals(2, versions.length); assertThrows( - "Can not access metadata {" + METALAKE + "," + CATALOG + "." + SCHEMA + "model1" + "}.", ForbiddenException.class, - () -> modelCatalogLoadByNormalUser.listModelVersions(NameIdentifier.of(SCHEMA, "model1"))); + () -> modelCatalogLoadByNormalUser.listModelVersions(NameIdentifier.of(SCHEMA, "model1")), + "Can not access metadata {" + METALAKE + "," + CATALOG + "." + SCHEMA + "model1" + "}."); gravitinoMetalake.grantPrivilegesToRole( role, MetadataObjects.of(ImmutableList.of(CATALOG, SCHEMA, "model1"), MetadataObject.Type.MODEL), @@ -357,11 +357,11 @@ public void testLoadModelVersion() { ModelVersion version = modelCatalog.getModelVersion(NameIdentifier.of(SCHEMA, "model1"), 1); assertEquals(1, version.version()); assertThrows( - "Can not access metadata {" + METALAKE + "," + CATALOG + "." + SCHEMA + "model1" + "}.", ForbiddenException.class, () -> { modelCatalogLoadByNormalUser.getModelVersion(NameIdentifier.of(SCHEMA, "model1"), 1); - }); + }, + "Can not access metadata {" + METALAKE + "," + CATALOG + "." + SCHEMA + "model1" + "}."); gravitinoMetalake.grantPrivilegesToRole( role, @@ -387,14 +387,14 @@ public void testAlterModelVersion() { NameIdentifier.of(SCHEMA, "model1"), 1, ModelVersionChange.setProperty("key", "value")); assertEquals("value", version.properties().get("key")); assertThrows( - "Can not access metadata {" + METALAKE + "," + CATALOG + "." + SCHEMA + "model1" + "}.", ForbiddenException.class, () -> { modelCatalogLoadByNormalUser.alterModelVersion( NameIdentifier.of(SCHEMA, "model1"), 1, ModelVersionChange.setProperty("key", "value")); - }); + }, + "Can not access metadata {" + METALAKE + "," + CATALOG + "." + SCHEMA + "model1" + "}."); } @Test @@ -405,11 +405,11 @@ public void testDropModelVersion() { normalUserClient.loadMetalake(METALAKE).loadCatalog(CATALOG); ModelCatalog modelCatalogLoadByNormalUser = catalogEntityLoadByNormalUser.asModelCatalog(); assertThrows( - "Can not access metadata {" + METALAKE + "," + CATALOG + "." + SCHEMA + "model1" + "}.", ForbiddenException.class, () -> { modelCatalogLoadByNormalUser.deleteModelVersion(NameIdentifier.of(SCHEMA, "model1"), 1); - }); + }, + "Can not access metadata {" + METALAKE + "," + CATALOG + "." + SCHEMA + "model1" + "}."); modelCatalog.deleteModelVersion(NameIdentifier.of(SCHEMA, "model1"), 1); int[] versions = modelCatalog.listModelVersions(NameIdentifier.of(SCHEMA, "model1")); assertEquals(1, versions.length); diff --git a/clients/client-java/src/test/java/org/apache/gravitino/client/integration/test/authorization/OwnerAuthorizationIT.java b/clients/client-java/src/test/java/org/apache/gravitino/client/integration/test/authorization/OwnerAuthorizationIT.java index d235e658fa4..a055cbce733 100644 --- a/clients/client-java/src/test/java/org/apache/gravitino/client/integration/test/authorization/OwnerAuthorizationIT.java +++ b/clients/client-java/src/test/java/org/apache/gravitino/client/integration/test/authorization/OwnerAuthorizationIT.java @@ -17,7 +17,7 @@ package org.apache.gravitino.client.integration.test.authorization; -import static org.junit.Assert.assertThrows; +import static org.junit.jupiter.api.Assertions.assertThrows; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; @@ -120,7 +120,6 @@ public void testSetTableOwnerByMetalakeOwner() { // normal user can not set owner GravitinoMetalake gravitinoMetalakeLoadByNormalUser = normalUserClient.loadMetalake(METALAKE); assertThrows( - "Current user can not set owner", ForbiddenException.class, () -> { gravitinoMetalakeLoadByNormalUser.setOwner( @@ -128,7 +127,8 @@ public void testSetTableOwnerByMetalakeOwner() { ImmutableList.of(CATALOG, SCHEMA, "table1"), MetadataObject.Type.TABLE), NORMAL_USER, Owner.Type.USER); - }); + }, + "Current user can not set owner"); } @Test @@ -146,7 +146,6 @@ public void testSetTableOwnerByCatalogOwner() { // normal user can not set owner GravitinoMetalake gravitinoMetalakeLoadByNormalUser = normalUserClient.loadMetalake(METALAKE); assertThrows( - "Current user can not set owner", ForbiddenException.class, () -> { gravitinoMetalakeLoadByNormalUser.setOwner( @@ -154,7 +153,8 @@ public void testSetTableOwnerByCatalogOwner() { ImmutableList.of(CATALOG, SCHEMA, "table1"), MetadataObject.Type.TABLE), NORMAL_USER, Owner.Type.USER); - }); + }, + "Current user can not set owner"); // set catalog owner to normal user gravitinoMetalake.setOwner( MetadataObjects.of(ImmutableList.of(CATALOG), MetadataObject.Type.CATALOG), @@ -192,7 +192,6 @@ public void testSetTableOwnerBySchemaOwner() { // normal user can not set owner GravitinoMetalake gravitinoMetalakeLoadByNormalUser = normalUserClient.loadMetalake(METALAKE); assertThrows( - "Current user can not set owner", ForbiddenException.class, () -> { gravitinoMetalakeLoadByNormalUser.setOwner( @@ -200,7 +199,8 @@ public void testSetTableOwnerBySchemaOwner() { ImmutableList.of(CATALOG, SCHEMA, "table1"), MetadataObject.Type.TABLE), NORMAL_USER, Owner.Type.USER); - }); + }, + "Current user can not set owner"); // set schema owner to normal user gravitinoMetalake.setOwner( MetadataObjects.of(ImmutableList.of(CATALOG, SCHEMA), MetadataObject.Type.SCHEMA), @@ -208,7 +208,6 @@ public void testSetTableOwnerBySchemaOwner() { Owner.Type.USER); // normal user can set owner assertThrows( - "Current user can not set owner", ForbiddenException.class, () -> { // NORMAL_USER has not USE_CATALOG @@ -217,7 +216,8 @@ public void testSetTableOwnerBySchemaOwner() { ImmutableList.of(CATALOG, SCHEMA, "table1"), MetadataObject.Type.TABLE), NORMAL_USER, Owner.Type.USER); - }); + }, + "Current user can not set owner"); gravitinoMetalake.grantPrivilegesToRole( role, MetadataObjects.of(ImmutableList.of(CATALOG), MetadataObject.Type.CATALOG), @@ -253,7 +253,6 @@ public void testSetTableOwnerByTableOwner() { // normal user can not set owner GravitinoMetalake gravitinoMetalakeLoadByNormalUser = normalUserClient.loadMetalake(METALAKE); assertThrows( - "Current user can not set owner", ForbiddenException.class, () -> { gravitinoMetalakeLoadByNormalUser.setOwner( @@ -261,14 +260,14 @@ public void testSetTableOwnerByTableOwner() { ImmutableList.of(CATALOG, SCHEMA, "table1"), MetadataObject.Type.TABLE), NORMAL_USER, Owner.Type.USER); - }); + }, + "Current user can not set owner"); // set schema owner to normal user gravitinoMetalake.setOwner( MetadataObjects.of(ImmutableList.of(CATALOG, SCHEMA, "table1"), MetadataObject.Type.TABLE), NORMAL_USER, Owner.Type.USER); assertThrows( - "Current user can not set owner", ForbiddenException.class, () -> { // NORMAL_USER has not USE_SCHEMA @@ -277,7 +276,8 @@ public void testSetTableOwnerByTableOwner() { ImmutableList.of(CATALOG, SCHEMA, "table1"), MetadataObject.Type.TABLE), NORMAL_USER, Owner.Type.USER); - }); + }, + "Current user can not set owner"); gravitinoMetalake.grantPrivilegesToRole( role, MetadataObjects.of(ImmutableList.of(CATALOG, SCHEMA), MetadataObject.Type.SCHEMA), @@ -305,14 +305,14 @@ public void testSetRoleOwner() { // normal user can not set owner GravitinoMetalake gravitinoMetalakeLoadByNormalUser = normalUserClient.loadMetalake(METALAKE); assertThrows( - "Current user can not set owner", ForbiddenException.class, () -> { gravitinoMetalakeLoadByNormalUser.setOwner( MetadataObjects.of(ImmutableList.of(tempRole), MetadataObject.Type.ROLE), tempUser, Owner.Type.USER); - }); + }, + "Current user can not set owner"); gravitinoMetalake.setOwner( MetadataObjects.of(ImmutableList.of(tempRole), MetadataObject.Type.ROLE), tempUser, @@ -333,12 +333,12 @@ public void testGetCatalogOwner() { MetadataObjects.of(ImmutableList.of(CATALOG), MetadataObject.Type.CATALOG), ImmutableSet.of(Privileges.UseCatalog.allow())); assertThrows( - "Current user can not get owner", ForbiddenException.class, () -> { gravitinoMetalakeLoadByNormalUser.getOwner( MetadataObjects.of(ImmutableList.of(CATALOG), MetadataObject.Type.CATALOG)); - }); + }, + "Current user can not get owner"); gravitinoMetalake.grantPrivilegesToRole( role, MetadataObjects.of(ImmutableList.of(CATALOG), MetadataObject.Type.CATALOG), @@ -359,12 +359,12 @@ public void testGetSchemaOwner() { MetadataObjects.of(ImmutableList.of(CATALOG, SCHEMA), MetadataObject.Type.SCHEMA), ImmutableSet.of(Privileges.UseSchema.allow())); assertThrows( - "Current user can not get owner", ForbiddenException.class, () -> { gravitinoMetalakeLoadByNormalUser.getOwner( MetadataObjects.of(ImmutableList.of(CATALOG, SCHEMA), MetadataObject.Type.SCHEMA)); - }); + }, + "Current user can not get owner"); gravitinoMetalake.grantPrivilegesToRole( role, MetadataObjects.of(ImmutableList.of(CATALOG, SCHEMA), MetadataObject.Type.SCHEMA), @@ -379,13 +379,13 @@ public void testGetTableOwner() { MetadataObjects.of(ImmutableList.of(CATALOG, SCHEMA, "table1"), MetadataObject.Type.TABLE)); GravitinoMetalake gravitinoMetalakeLoadByNormalUser = normalUserClient.loadMetalake(METALAKE); assertThrows( - "Current user can not get owner", ForbiddenException.class, () -> { gravitinoMetalakeLoadByNormalUser.getOwner( MetadataObjects.of( ImmutableList.of(CATALOG, SCHEMA, "table1"), MetadataObject.Type.TABLE)); - }); + }, + "Current user can not get owner"); } @Test @@ -411,12 +411,12 @@ public void testGetPolicyOwner() { gravitinoMetalake.getOwner( MetadataObjects.of(ImmutableList.of(policyName), MetadataObject.Type.POLICY)); assertThrows( - "Current user can not get owner", ForbiddenException.class, () -> { gravitinoMetalakeLoadByNormalUser.getOwner( MetadataObjects.of(ImmutableList.of(policyName), MetadataObject.Type.POLICY)); - }); + }, + "Current user can not get owner"); } @Test @@ -429,12 +429,12 @@ public void testGetTagOwner() { gravitinoMetalake.getOwner( MetadataObjects.of(ImmutableList.of(tagName), MetadataObject.Type.TAG)); assertThrows( - "Current user can not get owner", ForbiddenException.class, () -> { gravitinoMetalakeLoadByNormalUser.getOwner( MetadataObjects.of(ImmutableList.of(tagName), MetadataObject.Type.TAG)); - }); + }, + "Current user can not get owner"); } @Test @@ -459,24 +459,24 @@ public void testGetJobOwner() throws IOException { gravitinoMetalake.getOwner( MetadataObjects.of(ImmutableList.of("test_1"), MetadataObject.Type.JOB_TEMPLATE)); assertThrows( - "Current user can not get owner", ForbiddenException.class, () -> { gravitinoMetalakeLoadByNormalUser.getOwner( MetadataObjects.of(ImmutableList.of("test_1"), MetadataObject.Type.JOB_TEMPLATE)); - }); + }, + "Current user can not get owner"); JobHandle jobHandle = gravitinoMetalake.runJob( "test_1", ImmutableMap.of("arg1", "value1", "arg2", "success", "env_var", "value2")); gravitinoMetalake.getOwner( MetadataObjects.of(ImmutableList.of(jobHandle.jobId()), MetadataObject.Type.JOB)); assertThrows( - "Current user can not get owner", ForbiddenException.class, () -> { gravitinoMetalakeLoadByNormalUser.getOwner( MetadataObjects.of(ImmutableList.of(jobHandle.jobId()), MetadataObject.Type.JOB)); - }); + }, + "Current user can not get owner"); } @Test @@ -488,11 +488,11 @@ public void testGetRoleOwner() { MetadataObjects.of(ImmutableList.of(role), MetadataObject.Type.ROLE)); gravitinoMetalake.revokeRolesFromUser(ImmutableList.of(role), NORMAL_USER); assertThrows( - "Current user can not get owner", ForbiddenException.class, () -> { gravitinoMetalakeLoadByNormalUser.getOwner( MetadataObjects.of(ImmutableList.of(role), MetadataObject.Type.ROLE)); - }); + }, + "Current user can not get owner"); } } diff --git a/clients/client-java/src/test/java/org/apache/gravitino/client/integration/test/authorization/PartitionAuthorizationIT.java b/clients/client-java/src/test/java/org/apache/gravitino/client/integration/test/authorization/PartitionAuthorizationIT.java index b25fc1f0f4b..4418e3f2263 100644 --- a/clients/client-java/src/test/java/org/apache/gravitino/client/integration/test/authorization/PartitionAuthorizationIT.java +++ b/clients/client-java/src/test/java/org/apache/gravitino/client/integration/test/authorization/PartitionAuthorizationIT.java @@ -18,8 +18,8 @@ package org.apache.gravitino.client.integration.test.authorization; import static org.apache.gravitino.client.integration.test.StatisticIT.TABLE_COMMENT; -import static org.junit.Assert.assertThrows; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; @@ -87,7 +87,6 @@ public void startIntegrationTest() throws Exception { .createSchema(SCHEMA, "test", new HashMap<>()); // try to load the schema as normal user, expect failure assertThrows( - "Can not access metadata {" + CATALOG + "." + SCHEMA + "}.", RuntimeException.class, () -> { normalUserClient @@ -95,7 +94,8 @@ public void startIntegrationTest() throws Exception { .loadCatalog(CATALOG) .asSchemas() .loadSchema(SCHEMA); - }); + }, + "Can not access metadata {" + CATALOG + "." + SCHEMA + "}."); // grant tester privilege List securableObjects = new ArrayList<>(); GravitinoMetalake gravitinoMetalake = client.loadMetalake(METALAKE); @@ -129,12 +129,12 @@ public void startIntegrationTest() throws Exception { TableCatalog tableCatalogNormalUser = normalUserClient.loadMetalake(METALAKE).loadCatalog(CATALOG).asTableCatalog(); assertThrows( - "Can not access metadata {" + CATALOG + "." + SCHEMA + "}.", ForbiddenException.class, () -> { tableCatalogNormalUser.createTable( NameIdentifier.of(SCHEMA, "table2"), createColumns(), "test2", new HashMap<>()); - }); + }, + "Can not access metadata {" + CATALOG + "." + SCHEMA + "}."); } @AfterAll @@ -157,11 +157,11 @@ public void testCreatePartition() { normalUserClient.loadMetalake(METALAKE).loadCatalog(CATALOG).asTableCatalog(); SupportsPartitions supportsPartitions = getTable1(tableCatalogNormalUser).supportPartitions(); assertThrows( - "Can not access metadata {" + CATALOG + "." + SCHEMA + "}.", ForbiddenException.class, () -> { supportsPartitions.addPartition(genPartition()); - }); + }, + "Can not access metadata {" + CATALOG + "." + SCHEMA + "}."); SupportsPartitions supportsPartitionsByAdmin = table1.supportPartitions(); supportsPartitionsByAdmin.addPartition(genPartition()); } @@ -191,9 +191,9 @@ public void testListPartition() { .revokePrivilegesFromRole( role, metadataObject, ImmutableSet.of(Privileges.SelectTable.allow())); assertThrows( - "Can not access metadata {" + CATALOG + "." + SCHEMA + "}.", ForbiddenException.class, - supportsPartitions::listPartitions); + supportsPartitions::listPartitions, + "Can not access metadata {" + CATALOG + "." + SCHEMA + "}."); SupportsPartitions supportsPartitionsByAdmin = table1.supportPartitions(); supportsPartitionsByAdmin.listPartitions(); client @@ -218,11 +218,11 @@ public void testLoadPartition() { .revokePrivilegesFromRole( role, metadataObject, ImmutableSet.of(Privileges.SelectTable.allow())); assertThrows( - "Can not access metadata {" + CATALOG + "." + SCHEMA + "}.", ForbiddenException.class, () -> { supportsPartitions.getPartition("col1=1"); - }); + }, + "Can not access metadata {" + CATALOG + "." + SCHEMA + "}."); SupportsPartitions supportsPartitionsByAdmin = table1.supportPartitions(); supportsPartitionsByAdmin.getPartition("col1=1"); client @@ -242,11 +242,11 @@ public void testDropPartition() { SupportsPartitions supportsPartitions = tableCatalogNormalUser.loadTable(NameIdentifier.of(SCHEMA, "table1")).supportPartitions(); assertThrows( - "Can not access metadata {" + CATALOG + "." + SCHEMA + "}.", ForbiddenException.class, () -> { supportsPartitions.dropPartition("col1=1"); - }); + }, + "Can not access metadata {" + CATALOG + "." + SCHEMA + "}."); SupportsPartitions supportsPartitionsByAdmin = table1.supportPartitions(); supportsPartitionsByAdmin.dropPartition("col1=1"); } diff --git a/clients/client-java/src/test/java/org/apache/gravitino/client/integration/test/authorization/PermissionAuthorizationIT.java b/clients/client-java/src/test/java/org/apache/gravitino/client/integration/test/authorization/PermissionAuthorizationIT.java index 0bfab6c3317..2188b4e5683 100644 --- a/clients/client-java/src/test/java/org/apache/gravitino/client/integration/test/authorization/PermissionAuthorizationIT.java +++ b/clients/client-java/src/test/java/org/apache/gravitino/client/integration/test/authorization/PermissionAuthorizationIT.java @@ -17,7 +17,7 @@ package org.apache.gravitino.client.integration.test.authorization; -import static org.junit.Assert.assertThrows; +import static org.junit.jupiter.api.Assertions.assertThrows; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; @@ -52,47 +52,47 @@ public void testGrant() { gravitinoMetalake.revokeRolesFromUser(ImmutableList.of("role1"), NORMAL_USER); GravitinoMetalake gravitinoMetalakeLoadByNormalUser = normalUserClient.loadMetalake(METALAKE); assertThrows( - "Current user can not grant.", ForbiddenException.class, () -> { gravitinoMetalakeLoadByNormalUser.grantRolesToUser( ImmutableList.of("role1"), NORMAL_USER); - }); + }, + "Current user can not grant."); assertThrows( - "Current user can not grant.", ForbiddenException.class, () -> { gravitinoMetalakeLoadByNormalUser.grantRolesToGroup(ImmutableList.of("role2"), "group1"); - }); + }, + "Current user can not grant."); assertThrows( - "Current user can not grant.", ForbiddenException.class, () -> { gravitinoMetalakeLoadByNormalUser.grantRolesToGroup(ImmutableList.of("role2"), "group1"); - }); + }, + "Current user can not grant."); assertThrows( - "Current user can not grant.", ForbiddenException.class, () -> { gravitinoMetalakeLoadByNormalUser.revokePrivilegesFromRole( "role1", MetadataObjects.of(ImmutableList.of(METALAKE), MetadataObject.Type.METALAKE), ImmutableSet.of(Privileges.ManageUsers.allow())); - }); + }, + "Current user can not grant."); assertThrows( - "Current user can not grant.", ForbiddenException.class, () -> { gravitinoMetalakeLoadByNormalUser.revokeRolesFromGroup( ImmutableList.of("role2"), "group1"); - }); + }, + "Current user can not grant."); assertThrows( - "Current user can not grant.", ForbiddenException.class, () -> { gravitinoMetalakeLoadByNormalUser.revokeRolesFromUser( ImmutableList.of("role1"), NORMAL_USER); - }); + }, + "Current user can not grant."); gravitinoMetalake.grantRolesToUser(ImmutableList.of("role3"), NORMAL_USER); gravitinoMetalake.grantPrivilegesToRole( diff --git a/clients/client-java/src/test/java/org/apache/gravitino/client/integration/test/authorization/PolicyAuthorizationIT.java b/clients/client-java/src/test/java/org/apache/gravitino/client/integration/test/authorization/PolicyAuthorizationIT.java index 754b975cb71..af524385fe4 100644 --- a/clients/client-java/src/test/java/org/apache/gravitino/client/integration/test/authorization/PolicyAuthorizationIT.java +++ b/clients/client-java/src/test/java/org/apache/gravitino/client/integration/test/authorization/PolicyAuthorizationIT.java @@ -17,8 +17,8 @@ package org.apache.gravitino.client.integration.test.authorization; -import static org.junit.Assert.assertThrows; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; @@ -87,7 +87,6 @@ public void startIntegrationTest() throws Exception { .createSchema(SCHEMA, "test", new HashMap<>()); // try to load the schema as normal user, expect failure assertThrows( - "Can not access metadata.{" + CATALOG + "." + SCHEMA + "}.", RuntimeException.class, () -> { normalUserClient @@ -95,7 +94,8 @@ public void startIntegrationTest() throws Exception { .loadCatalog(CATALOG) .asSchemas() .loadSchema(SCHEMA); - }); + }, + "Can not access metadata.{" + CATALOG + "." + SCHEMA + "}."); // grant tester privilege List securableObjects = new ArrayList<>(); GravitinoMetalake gravitinoMetalake = client.loadMetalake(METALAKE); @@ -108,11 +108,11 @@ public void startIntegrationTest() throws Exception { Catalog catalogLoadByNormalUser = normalUserClient.loadMetalake(METALAKE).loadCatalog(CATALOG); assertEquals(CATALOG, catalogLoadByNormalUser.name()); assertThrows( - "Can not access metadata.{" + CATALOG + "." + SCHEMA + "}.", ForbiddenException.class, () -> { catalogLoadByNormalUser.asSchemas().loadSchema(SCHEMA); - }); + }, + "Can not access metadata.{" + CATALOG + "." + SCHEMA + "}."); TableCatalog tableCatalog = client.loadMetalake(METALAKE).loadCatalog(CATALOG).asTableCatalog(); tableCatalog.createTable( NameIdentifier.of(SCHEMA, "table1"), createColumns(), "test", new HashMap<>()); @@ -139,7 +139,6 @@ public void createPolicy() { "policy1", "custom", "policy1", true, PolicyContents.custom(null, supportedTypes, null)); GravitinoMetalake metalakeLoadByNormalUser = normalUserClient.loadMetalake(METALAKE); assertThrows( - "Can not access metadata.{" + CATALOG + "." + SCHEMA + "}.", ForbiddenException.class, () -> { metalakeLoadByNormalUser.createPolicy( @@ -148,7 +147,8 @@ public void createPolicy() { "policy2", true, PolicyContents.custom(null, supportedTypes, null)); - }); + }, + "Can not access metadata.{" + CATALOG + "." + SCHEMA + "}."); gravitinoMetalake.grantPrivilegesToRole( role, MetadataObjects.of(ImmutableList.of(METALAKE), MetadataObject.Type.METALAKE), @@ -178,18 +178,18 @@ public void testGetPolicy() { gravitinoMetalake.getPolicy("policy3"); GravitinoMetalake metalakeLoadByNormalUser = normalUserClient.loadMetalake(METALAKE); assertThrows( - "Can not access metadata.{" + CATALOG + "." + SCHEMA + "}.", ForbiddenException.class, () -> { metalakeLoadByNormalUser.getPolicy("policy1"); - }); + }, + "Can not access metadata.{" + CATALOG + "." + SCHEMA + "}."); metalakeLoadByNormalUser.getPolicy("policy2"); assertThrows( - "Can not access metadata.{" + CATALOG + "." + SCHEMA + "}.", ForbiddenException.class, () -> { metalakeLoadByNormalUser.getPolicy("policy3"); - }); + }, + "Can not access metadata.{" + CATALOG + "." + SCHEMA + "}."); gravitinoMetalake.grantPrivilegesToRole( role, @@ -207,11 +207,11 @@ public void testAlterPolicy() { GravitinoMetalake metalakeLoadByNormalUser = normalUserClient.loadMetalake(METALAKE); gravitinoMetalake.alterPolicy("policy1", PolicyChange.updateComment("222")); assertThrows( - "Can not access metadata.{" + CATALOG + "." + SCHEMA + "}.", ForbiddenException.class, () -> { metalakeLoadByNormalUser.alterPolicy("policy3", PolicyChange.updateComment("222")); - }); + }, + "Can not access metadata.{" + CATALOG + "." + SCHEMA + "}."); metalakeLoadByNormalUser.alterPolicy("policy2", PolicyChange.updateComment("222")); } @@ -288,11 +288,11 @@ public void testDropPolicy() { GravitinoMetalake gravitinoMetalake = client.loadMetalake(METALAKE); GravitinoMetalake metalakeLoadByNormalUser = normalUserClient.loadMetalake(METALAKE); assertThrows( - "Can not access metadata.", ForbiddenException.class, () -> { metalakeLoadByNormalUser.deletePolicy("policy1"); - }); + }, + "Can not access metadata."); gravitinoMetalake.deletePolicy("policy1"); metalakeLoadByNormalUser.deletePolicy("policy2"); } diff --git a/clients/client-java/src/test/java/org/apache/gravitino/client/integration/test/authorization/RoleAuthorizationIT.java b/clients/client-java/src/test/java/org/apache/gravitino/client/integration/test/authorization/RoleAuthorizationIT.java index b84a3e673d1..b3e54003836 100644 --- a/clients/client-java/src/test/java/org/apache/gravitino/client/integration/test/authorization/RoleAuthorizationIT.java +++ b/clients/client-java/src/test/java/org/apache/gravitino/client/integration/test/authorization/RoleAuthorizationIT.java @@ -17,9 +17,9 @@ package org.apache.gravitino.client.integration.test.authorization; -import static org.junit.Assert.assertArrayEquals; -import static org.junit.Assert.assertThrows; +import static org.junit.jupiter.api.Assertions.assertArrayEquals; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; @@ -63,13 +63,13 @@ public void testCreateRole() { client.loadMetalake(METALAKE).createRole("role3", new HashMap<>(), Collections.emptyList()); // normal user can not create role assertThrows( - "Current user can not create role.", ForbiddenException.class, () -> { normalUserClient .loadMetalake(METALAKE) .createRole("role4", new HashMap<>(), Collections.emptyList()); - }); + }, + "Current user can not create role."); client.loadMetalake(METALAKE).grantRolesToUser(ImmutableList.of("role1"), NORMAL_USER); client .loadMetalake(METALAKE) @@ -118,17 +118,17 @@ public void testGetRole() { normalUserClient.loadMetalake(METALAKE).getRole("role1"); // normal user can not get role assertThrows( - "Current user can not create role.", ForbiddenException.class, () -> { normalUserClient.loadMetalake(METALAKE).getRole("role2"); - }); + }, + "Current user can not create role."); assertThrows( - "Current user can not create role.", ForbiddenException.class, () -> { normalUserClient.loadMetalake(METALAKE).getRole("role3"); - }); + }, + "Current user can not create role."); } @Test @@ -136,36 +136,36 @@ public void testGetRole() { public void testDeleteRole() { // normal user can not delete role assertThrows( - "Current user can not create role.", ForbiddenException.class, () -> { normalUserClient.loadMetalake(METALAKE).deleteRole("role1"); - }); + }, + "Current user can not create role."); assertThrows( - "Current user can not create role.", ForbiddenException.class, () -> { normalUserClient.loadMetalake(METALAKE).deleteRole("role2"); - }); + }, + "Current user can not create role."); assertThrows( - "Current user can not create role.", ForbiddenException.class, () -> { normalUserClient.loadMetalake(METALAKE).deleteRole("role3"); - }); + }, + "Current user can not create role."); // owner can delete role client.loadMetalake(METALAKE).deleteRole("role1"); client.loadMetalake(METALAKE).deleteRole("role2"); client.loadMetalake(METALAKE).deleteRole("role3"); // normal user can not create role after delete role assertThrows( - "Current user can not create role.", ForbiddenException.class, () -> { normalUserClient .loadMetalake(METALAKE) .createRole("role2", new HashMap<>(), Collections.emptyList()); - }); + }, + "Current user can not create role."); } @Test diff --git a/clients/client-java/src/test/java/org/apache/gravitino/client/integration/test/authorization/SchemaAuthorizationIT.java b/clients/client-java/src/test/java/org/apache/gravitino/client/integration/test/authorization/SchemaAuthorizationIT.java index c92ef8e6bc1..5fef28f15ca 100644 --- a/clients/client-java/src/test/java/org/apache/gravitino/client/integration/test/authorization/SchemaAuthorizationIT.java +++ b/clients/client-java/src/test/java/org/apache/gravitino/client/integration/test/authorization/SchemaAuthorizationIT.java @@ -17,9 +17,9 @@ package org.apache.gravitino.client.integration.test.authorization; -import static org.junit.Assert.assertArrayEquals; -import static org.junit.Assert.assertThrows; +import static org.junit.jupiter.api.Assertions.assertArrayEquals; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; import com.google.common.collect.ImmutableList; import com.google.common.collect.Maps; @@ -75,11 +75,11 @@ public void startIntegrationTest() throws Exception { .loadMetalake(METALAKE) .createCatalog(CATALOG, Catalog.Type.RELATIONAL, "hive", "comment", properties); assertThrows( - "Can not access metadata {" + CATALOG + "}.", RuntimeException.class, () -> { normalUserClient.loadMetalake(METALAKE).loadCatalog(CATALOG); - }); + }, + "Can not access metadata {" + CATALOG + "}."); // grant tester load catalog privilege List securableObjects = new ArrayList<>(); @@ -104,11 +104,11 @@ public void testCreateSchema() { Catalog catalogEntityLoadByTester2 = normalUserClient.loadMetalake(METALAKE).loadCatalog(CATALOG); assertThrows( - "Can not access metadata {" + CATALOG + "}.", ForbiddenException.class, () -> { catalogEntityLoadByTester2.asSchemas().createSchema("schema2", "test2", new HashMap<>()); - }); + }, + "Can not access metadata {" + CATALOG + "}."); GravitinoMetalake gravitinoMetalake = client.loadMetalake(METALAKE); // test grant create schema privilege @@ -142,11 +142,11 @@ public void testLoadSchema() { String[] schemasLoadByTester2 = schemas.listSchemas(); assertArrayEquals(new String[] {"schema2", "schema3"}, schemasLoadByTester2); assertThrows( - String.format("Can not access metadata {%s.%s}.", CATALOG, "schema1"), ForbiddenException.class, () -> { catalogEntityLoadByTester2.asSchemas().loadSchema("schema1"); - }); + }, + String.format("Can not access metadata {%s.%s}.", CATALOG, "schema1")); // test grant use schema GravitinoMetalake gravitinoMetalake = client.loadMetalake(METALAKE); gravitinoMetalake.grantPrivilegesToRole( @@ -175,13 +175,13 @@ public void testAlterSchema() { Catalog catalogEntityLoadByTester2 = normalUserClient.loadMetalake(METALAKE).loadCatalog(CATALOG); assertThrows( - String.format("Can not access metadata {%s.%s}.", CATALOG, "schema1"), ForbiddenException.class, () -> { catalogEntityLoadByTester2 .asSchemas() .alterSchema("schema1", SchemaChange.setProperty("key", "value")); - }); + }, + String.format("Can not access metadata {%s.%s}.", CATALOG, "schema1")); // test set owner gravitinoMetalake.setOwner( MetadataObjects.of(ImmutableList.of(CATALOG, "schema1"), MetadataObject.Type.SCHEMA), @@ -207,11 +207,11 @@ public void testDropSchema() { Catalog catalogEntityLoadByTester2 = normalUserClient.loadMetalake(METALAKE).loadCatalog(CATALOG); assertThrows( - String.format("Can not access metadata {%s.%s}.", CATALOG, "schema1"), ForbiddenException.class, () -> { catalogEntityLoadByTester2.asSchemas().dropSchema("schema1", true); - }); + }, + String.format("Can not access metadata {%s.%s}.", CATALOG, "schema1")); catalogEntityLoadByTester2.asSchemas().dropSchema("schema2", true); Catalog catalogEntityLoadByTester1 = gravitinoMetalake.loadCatalog(CATALOG); catalogEntityLoadByTester1.asSchemas().dropSchema("schema1", true); @@ -223,11 +223,11 @@ public void testDropSchema() { catalogEntityLoadByTester1.asSchemas().createSchema("schema1", "test", new HashMap<>()); // test catalog owner assertThrows( - String.format("Can not access metadata {%s.%s}.", CATALOG, "schema1"), ForbiddenException.class, () -> { catalogEntityLoadByTester2.asSchemas().dropSchema("schema1", true); - }); + }, + String.format("Can not access metadata {%s.%s}.", CATALOG, "schema1")); gravitinoMetalake.setOwner( MetadataObjects.of(ImmutableList.of(CATALOG, "schema1"), MetadataObject.Type.SCHEMA), NORMAL_USER, diff --git a/clients/client-java/src/test/java/org/apache/gravitino/client/integration/test/authorization/StatisticAuthorizationIT.java b/clients/client-java/src/test/java/org/apache/gravitino/client/integration/test/authorization/StatisticAuthorizationIT.java index 6dd0433149f..5615076c5f6 100644 --- a/clients/client-java/src/test/java/org/apache/gravitino/client/integration/test/authorization/StatisticAuthorizationIT.java +++ b/clients/client-java/src/test/java/org/apache/gravitino/client/integration/test/authorization/StatisticAuthorizationIT.java @@ -16,7 +16,7 @@ */ package org.apache.gravitino.client.integration.test.authorization; -import static org.junit.Assert.assertThrows; +import static org.junit.jupiter.api.Assertions.assertThrows; import com.google.common.collect.ImmutableList; import com.google.common.collect.Lists; @@ -107,11 +107,11 @@ public void testListStatistics() { normalUserClient.loadMetalake(METALAKE).loadCatalog(CATALOG).asTableCatalog(); // normal user can't load table1 assertThrows( - String.format("Can not access metadata {%s.%s.%s}.", CATALOG, SCHEMA, "table1"), ForbiddenException.class, () -> { tableCatalogNormalUser.loadTable(NameIdentifier.of(SCHEMA, "table1")); - }); + }, + String.format("Can not access metadata {%s.%s.%s}.", CATALOG, SCHEMA, "table1")); // grant normal user privilege to use table1 GravitinoMetalake gravitinoMetalake = client.loadMetalake(METALAKE); @@ -143,37 +143,37 @@ public void testModifyStatistics() { SupportsPartitionStatistics supportsPartitionStatistics = table.supportsPartitionStatistics(); assertThrows( - String.format("Can not access metadata {%s.%s.%s}.", CATALOG, SCHEMA, "table1"), ForbiddenException.class, () -> { supportsStatistics.updateStatistics(statistics); - }); + }, + String.format("Can not access metadata {%s.%s.%s}.", CATALOG, SCHEMA, "table1")); List updates = Lists.newArrayList(PartitionStatisticsModification.update("p1", statistics)); assertThrows( - String.format("Can not access metadata {%s.%s.%s}.", CATALOG, SCHEMA, "table1"), ForbiddenException.class, () -> { supportsPartitionStatistics.updatePartitionStatistics(updates); - }); + }, + String.format("Can not access metadata {%s.%s.%s}.", CATALOG, SCHEMA, "table1")); assertThrows( - String.format("Can not access metadata {%s.%s.%s}.", CATALOG, SCHEMA, "table1"), ForbiddenException.class, () -> { supportsStatistics.dropStatistics(Lists.newArrayList("custom-k1")); - }); + }, + String.format("Can not access metadata {%s.%s.%s}.", CATALOG, SCHEMA, "table1")); List drops = Lists.newArrayList( PartitionStatisticsModification.drop("p1", Lists.newArrayList("custom-k1"))); assertThrows( - String.format("Can not access metadata {%s.%s.%s}.", CATALOG, SCHEMA, "table1"), ForbiddenException.class, () -> { supportsPartitionStatistics.dropPartitionStatistics(drops); - }); + }, + String.format("Can not access metadata {%s.%s.%s}.", CATALOG, SCHEMA, "table1")); // grant normal user owner privilege on table1 gravitinoMetalake.setOwner( diff --git a/clients/client-java/src/test/java/org/apache/gravitino/client/integration/test/authorization/TableAuthorizationIT.java b/clients/client-java/src/test/java/org/apache/gravitino/client/integration/test/authorization/TableAuthorizationIT.java index b244fad6eb0..ebc7a8d39cf 100644 --- a/clients/client-java/src/test/java/org/apache/gravitino/client/integration/test/authorization/TableAuthorizationIT.java +++ b/clients/client-java/src/test/java/org/apache/gravitino/client/integration/test/authorization/TableAuthorizationIT.java @@ -18,10 +18,10 @@ */ package org.apache.gravitino.client.integration.test.authorization; -import static org.junit.Assert.assertThrows; import static org.junit.jupiter.api.Assertions.assertArrayEquals; import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; @@ -88,7 +88,6 @@ public void startIntegrationTest() throws Exception { .createSchema(SCHEMA, "test", new HashMap<>()); // try to load the schema as normal user, expect failure assertThrows( - "Can not access metadata {" + CATALOG + "." + SCHEMA + "}.", RuntimeException.class, () -> { normalUserClient @@ -96,7 +95,8 @@ public void startIntegrationTest() throws Exception { .loadCatalog(CATALOG) .asSchemas() .loadSchema(SCHEMA); - }); + }, + "Can not access metadata {" + CATALOG + "." + SCHEMA + "}."); // grant tester privilege List securableObjects = new ArrayList<>(); GravitinoMetalake gravitinoMetalake = client.loadMetalake(METALAKE); @@ -109,11 +109,11 @@ public void startIntegrationTest() throws Exception { Catalog catalogLoadByNormalUser = normalUserClient.loadMetalake(METALAKE).loadCatalog(CATALOG); assertEquals(CATALOG, catalogLoadByNormalUser.name()); assertThrows( - "Can not access metadata {" + CATALOG + "." + SCHEMA + "}.", ForbiddenException.class, () -> { catalogLoadByNormalUser.asSchemas().loadSchema(SCHEMA); - }); + }, + "Can not access metadata {" + CATALOG + "." + SCHEMA + "}."); } @AfterAll @@ -152,18 +152,18 @@ public void testCreateTable() { TableCatalog tableCatalogNormalUser = normalUserClient.loadMetalake(METALAKE).loadCatalog(CATALOG).asTableCatalog(); assertThrows( - "Can not access metadata {" + CATALOG + "." + SCHEMA + "}.", ForbiddenException.class, () -> { tableCatalogNormalUser.createTable( NameIdentifier.of(SCHEMA, "table2"), createColumns(), "test2", new HashMap<>()); - }); + }, + "Can not access metadata {" + CATALOG + "." + SCHEMA + "}."); assertThrows( - "Can not access metadata {" + CATALOG + "." + SCHEMA + "}.", ForbiddenException.class, () -> { tableCatalogNormalUser.listTables(Namespace.of(SCHEMA)); - }); + }, + "Can not access metadata {" + CATALOG + "." + SCHEMA + "}."); // grant privileges GravitinoMetalake gravitinoMetalake = client.loadMetalake(METALAKE); gravitinoMetalake.grantPrivilegesToRole( @@ -189,12 +189,12 @@ public void testCreateTable() { TableCatalog catalogLoadByTmpUser = tempUserClient.loadMetalake(METALAKE).loadCatalog(CATALOG).asTableCatalog(); assertThrows( - "Can not access metadata {" + CATALOG + "." + SCHEMA + "}.", ForbiddenException.class, () -> { catalogLoadByTmpUser.createTable( NameIdentifier.of(SCHEMA, "table2"), createColumns(), "test2", new HashMap<>()); - }); + }, + "Can not access metadata {" + CATALOG + "." + SCHEMA + "}."); gravitinoMetalake.grantRolesToUser(ImmutableList.of(role), tempUser); catalogLoadByTmpUser.createTable( NameIdentifier.of(SCHEMA, "table4"), createColumns(), "test2", new HashMap<>()); @@ -233,11 +233,11 @@ public void testLoadTable() { normalUserClient.loadMetalake(METALAKE).loadCatalog(CATALOG).asTableCatalog(); // normal user can load table2 and table3, but not table1 assertThrows( - String.format("Can not access metadata {%s.%s.%s}.", CATALOG, SCHEMA, "table1"), ForbiddenException.class, () -> { tableCatalogNormalUser.loadTable(NameIdentifier.of(SCHEMA, "table1")); - }); + }, + String.format("Can not access metadata {%s.%s.%s}.", CATALOG, SCHEMA, "table1")); Table table2 = tableCatalogNormalUser.loadTable(NameIdentifier.of(SCHEMA, "table2")); assertEquals("table2", table2.name()); Table table3 = tableCatalogNormalUser.loadTable(NameIdentifier.of(SCHEMA, "table3")); @@ -252,23 +252,23 @@ public void testLoadTable() { tableCatalogNormalUser.loadTable(NameIdentifier.of(SCHEMA, "table1")); assertThrows( - String.format("Can not access metadata {%s.%s.%s}.", CATALOG, SCHEMA, "table1"), ForbiddenException.class, () -> { tableCatalogNormalUser.loadTable( NameIdentifier.of(SCHEMA, "table1"), Sets.newHashSet(Privilege.Name.MODIFY_TABLE)); - }); + }, + String.format("Can not access metadata {%s.%s.%s}.", CATALOG, SCHEMA, "table1")); gravitinoMetalake.grantPrivilegesToRole( role, MetadataObjects.of(ImmutableList.of(CATALOG, SCHEMA, "table1"), MetadataObject.Type.TABLE), ImmutableList.of(Privileges.SelectTable.deny())); assertThrows( - String.format("Can not access metadata {%s.%s.%s}.", CATALOG, SCHEMA, "table1"), ForbiddenException.class, () -> { tableCatalogNormalUser.loadTable(NameIdentifier.of(SCHEMA, "table1")); - }); + }, + String.format("Can not access metadata {%s.%s.%s}.", CATALOG, SCHEMA, "table1")); gravitinoMetalake.grantPrivilegesToRole( role, @@ -291,12 +291,12 @@ public void testAlterTable() { // normal user cannot alter table1 (no privilege) assertThrows( - String.format("Can not access metadata {%s.%s.%s}.", CATALOG, SCHEMA, "table1"), ForbiddenException.class, () -> { tableCatalogNormalUser.alterTable( NameIdentifier.of(SCHEMA, "table1"), TableChange.setProperty("key", "value")); - }); + }, + String.format("Can not access metadata {%s.%s.%s}.", CATALOG, SCHEMA, "table1")); // grant normal user owner privilege on table1 gravitinoMetalake.setOwner( MetadataObjects.of(ImmutableList.of(CATALOG, SCHEMA, "table1"), MetadataObject.Type.TABLE), @@ -319,11 +319,11 @@ public void testDropTable() { Owner.Type.USER); // normal user cannot drop table1 assertThrows( - String.format("Can not access metadata {%s.%s.%s}.", CATALOG, SCHEMA, "table1"), ForbiddenException.class, () -> { tableCatalogNormalUser.dropTable(NameIdentifier.of(SCHEMA, "table1")); - }); + }, + String.format("Can not access metadata {%s.%s.%s}.", CATALOG, SCHEMA, "table1")); // normal user can drop table2 and table3 (they created them) tableCatalogNormalUser.dropTable(NameIdentifier.of(SCHEMA, "table2")); tableCatalogNormalUser.dropTable(NameIdentifier.of(SCHEMA, "table3")); diff --git a/clients/client-java/src/test/java/org/apache/gravitino/client/integration/test/authorization/TagOperationsAuthorizationIT.java b/clients/client-java/src/test/java/org/apache/gravitino/client/integration/test/authorization/TagOperationsAuthorizationIT.java index 6c76d0fdd9e..5c4b1fd7769 100644 --- a/clients/client-java/src/test/java/org/apache/gravitino/client/integration/test/authorization/TagOperationsAuthorizationIT.java +++ b/clients/client-java/src/test/java/org/apache/gravitino/client/integration/test/authorization/TagOperationsAuthorizationIT.java @@ -17,9 +17,9 @@ package org.apache.gravitino.client.integration.test.authorization; -import static org.junit.Assert.assertArrayEquals; -import static org.junit.Assert.assertThrows; +import static org.junit.jupiter.api.Assertions.assertArrayEquals; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; @@ -86,7 +86,6 @@ public void startIntegrationTest() throws Exception { .createSchema(SCHEMA, "test", new HashMap<>()); // try to load the schema as normal user, expect failure assertThrows( - "Can not access metadata.{" + CATALOG + "." + SCHEMA + "}.", RuntimeException.class, () -> { normalUserClient @@ -94,7 +93,8 @@ public void startIntegrationTest() throws Exception { .loadCatalog(CATALOG) .asSchemas() .loadSchema(SCHEMA); - }); + }, + "Can not access metadata.{" + CATALOG + "." + SCHEMA + "}."); // grant tester privilege List securableObjects = new ArrayList<>(); GravitinoMetalake gravitinoMetalake = client.loadMetalake(METALAKE); @@ -108,11 +108,11 @@ public void startIntegrationTest() throws Exception { Catalog catalogLoadByNormalUser = normalUserClient.loadMetalake(METALAKE).loadCatalog(CATALOG); assertEquals(CATALOG, catalogLoadByNormalUser.name()); assertThrows( - "Can not access metadata.{" + CATALOG + "." + SCHEMA + "}.", ForbiddenException.class, () -> { catalogLoadByNormalUser.asSchemas().loadSchema(SCHEMA); - }); + }, + "Can not access metadata.{" + CATALOG + "." + SCHEMA + "}."); TableCatalog tableCatalog = client.loadMetalake(METALAKE).loadCatalog(CATALOG).asTableCatalog(); tableCatalog.createTable( NameIdentifier.of(SCHEMA, "table1"), createColumns(), "test", new HashMap<>()); @@ -132,11 +132,11 @@ public void createTag() { GravitinoMetalake gravitinoMetalake = client.loadMetalake(METALAKE); gravitinoMetalake.createTag("tag1", "tag1", Map.of()); assertThrows( - "Can not access metadata.", ForbiddenException.class, () -> { normalUserClient.loadMetalake(METALAKE).createTag("tag2", "tag2", Map.of()); - }); + }, + "Can not access metadata."); gravitinoMetalake.grantPrivilegesToRole( role, MetadataObjects.of(ImmutableList.of(METALAKE), MetadataObject.Type.METALAKE), @@ -150,11 +150,11 @@ public void createTag() { public void testAlterTag() { GravitinoMetalake gravitinoMetalakeLoadByNormalUser = normalUserClient.loadMetalake(METALAKE); assertThrows( - "Can not access metadata.", ForbiddenException.class, () -> { gravitinoMetalakeLoadByNormalUser.alterTag("tag1", TagChange.setProperty("k1", "v1")); - }); + }, + "Can not access metadata."); gravitinoMetalakeLoadByNormalUser.alterTag("tag2", TagChange.setProperty("k1", "v1")); } @@ -175,11 +175,11 @@ public void testGetTag() { GravitinoMetalake gravitinoMetalakeLoadByNormalUser = normalUserClient.loadMetalake(METALAKE); gravitinoMetalakeLoadByNormalUser.getTag("tag2"); assertThrows( - "Can not access metadata.", ForbiddenException.class, () -> { gravitinoMetalakeLoadByNormalUser.getTag("tag1"); - }); + }, + "Can not access metadata."); GravitinoMetalake gravitinoMetalake = client.loadMetalake(METALAKE); gravitinoMetalake.getTag("tag1"); gravitinoMetalake.getTag("tag2"); @@ -193,7 +193,6 @@ public void testAssociateTag() { SupportsTags supportsTags = gravitinoMetalake.loadCatalog(CATALOG).supportsTags(); supportsTags.associateTags(new String[] {"tag1"}, null); assertThrows( - "Can not access metadata.", ForbiddenException.class, () -> { normalUserClient @@ -203,9 +202,9 @@ public void testAssociateTag() { .loadTable(NameIdentifier.of(SCHEMA, "table1")) .supportsTags() .associateTags(new String[] {"tag1"}, null); - }); + }, + "Can not access metadata."); assertThrows( - "Can not access metadata.", ForbiddenException.class, () -> { normalUserClient @@ -215,13 +214,13 @@ public void testAssociateTag() { .loadTable(NameIdentifier.of(SCHEMA, "table1")) .supportsTags() .associateTags(new String[] {"tag2"}, null); - }); + }, + "Can not access metadata."); gravitinoMetalake.grantPrivilegesToRole( role, MetadataObjects.of(ImmutableList.of(CATALOG, SCHEMA), MetadataObject.Type.SCHEMA), ImmutableList.of(Privileges.SelectTable.allow(), Privileges.UseSchema.allow())); assertThrows( - "Can not access metadata.", ForbiddenException.class, () -> { normalUserClient @@ -231,7 +230,8 @@ public void testAssociateTag() { .loadTable(NameIdentifier.of(SCHEMA, "table1")) .supportsTags() .associateTags(new String[] {"tag1"}, null); - }); + }, + "Can not access metadata."); normalUserClient .loadMetalake(METALAKE) .loadCatalog(CATALOG) @@ -370,7 +370,6 @@ public void testListObjForTag() { Arrays.sort(objs); Assertions.assertArrayEquals(new String[] {"table1"}, objs); assertThrows( - "Can not access metadata.", ForbiddenException.class, () -> { Arrays.stream( @@ -378,7 +377,8 @@ public void testListObjForTag() { .map(MetadataObject::name) .toList() .toArray(new String[0]); - }); + }, + "Can not access metadata."); objs = Arrays.stream( gravitinoMetalakeLoadByNormalUser.getTag("tag2").associatedObjects().objects()) @@ -498,11 +498,11 @@ public void testTagOperationsWithNonExistentMetalake() throws Exception { public void testDropTag() { GravitinoMetalake gravitinoMetalakeLoadByNormalUser = normalUserClient.loadMetalake(METALAKE); assertThrows( - "Can not access metadata.", ForbiddenException.class, () -> { gravitinoMetalakeLoadByNormalUser.deleteTag("tag1"); - }); + }, + "Can not access metadata."); gravitinoMetalakeLoadByNormalUser.deleteTag("tag2"); GravitinoMetalake gravitinoMetalake = client.loadMetalake(METALAKE); gravitinoMetalake.deleteTag("tag1"); diff --git a/clients/client-java/src/test/java/org/apache/gravitino/client/integration/test/authorization/TopicAuthorizationIT.java b/clients/client-java/src/test/java/org/apache/gravitino/client/integration/test/authorization/TopicAuthorizationIT.java index fb0a90036b9..19ae04f75ae 100644 --- a/clients/client-java/src/test/java/org/apache/gravitino/client/integration/test/authorization/TopicAuthorizationIT.java +++ b/clients/client-java/src/test/java/org/apache/gravitino/client/integration/test/authorization/TopicAuthorizationIT.java @@ -18,9 +18,9 @@ */ package org.apache.gravitino.client.integration.test.authorization; -import static org.junit.Assert.assertThrows; import static org.junit.jupiter.api.Assertions.assertArrayEquals; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; import com.google.common.collect.ImmutableList; import com.google.common.collect.Maps; @@ -79,7 +79,6 @@ public void startIntegrationTest() throws Exception { .createCatalog(CATALOG, Catalog.Type.MESSAGING, "kafka", "comment", properties); // try to load the schema as normal user, expect failure assertThrows( - "Can not access metadata {" + CATALOG + "." + SCHEMA + "}.", RuntimeException.class, () -> { normalUserClient @@ -87,7 +86,8 @@ public void startIntegrationTest() throws Exception { .loadCatalog(CATALOG) .asSchemas() .loadSchema(SCHEMA); - }); + }, + "Can not access metadata {" + CATALOG + "." + SCHEMA + "}."); // grant tester privilege List securableObjects = new ArrayList<>(); GravitinoMetalake gravitinoMetalake = client.loadMetalake(METALAKE); @@ -100,11 +100,11 @@ public void startIntegrationTest() throws Exception { Catalog catalogLoadByNormalUser = normalUserClient.loadMetalake(METALAKE).loadCatalog(CATALOG); assertEquals(CATALOG, catalogLoadByNormalUser.name()); assertThrows( - "Can not access metadata {" + CATALOG + "." + SCHEMA + "}.", ForbiddenException.class, () -> { catalogLoadByNormalUser.asSchemas().loadSchema(SCHEMA); - }); + }, + "Can not access metadata {" + CATALOG + "." + SCHEMA + "}."); } @Test @@ -117,19 +117,19 @@ public void testCreateTopic() { TopicCatalog topicCatalogNormalUser = normalUserClient.loadMetalake(METALAKE).loadCatalog(CATALOG).asTopicCatalog(); assertThrows( - "Can not access metadata {" + CATALOG + "." + SCHEMA + "}.", ForbiddenException.class, () -> { topicCatalogNormalUser.createTopic( NameIdentifier.of(SCHEMA, "topic2"), "test2", null, new HashMap<>()); - }); + }, + "Can not access metadata {" + CATALOG + "." + SCHEMA + "}."); assertThrows( - "Can not access metadata {" + CATALOG + "." + SCHEMA + "}.", ForbiddenException.class, () -> { topicCatalogNormalUser.listTopics(Namespace.of(SCHEMA)); - }); + }, + "Can not access metadata {" + CATALOG + "." + SCHEMA + "}."); // grant privileges GravitinoMetalake gravitinoMetalake = client.loadMetalake(METALAKE); @@ -185,11 +185,11 @@ public void testLoadTopic() { normalUserClient.loadMetalake(METALAKE).loadCatalog(CATALOG).asTopicCatalog(); // normal user can load topic2 and topic3, but not topic1 assertThrows( - String.format("Can not access metadata {%s.%s.%s}.", CATALOG, SCHEMA, "topic1"), ForbiddenException.class, () -> { topicCatalogNormalUser.loadTopic(NameIdentifier.of(SCHEMA, "topic1")); - }); + }, + String.format("Can not access metadata {%s.%s.%s}.", CATALOG, SCHEMA, "topic1")); Topic topic2 = topicCatalogNormalUser.loadTopic(NameIdentifier.of(SCHEMA, "topic2")); assertEquals("topic2", topic2.name()); Topic topic3 = topicCatalogNormalUser.loadTopic(NameIdentifier.of(SCHEMA, "topic3")); @@ -213,12 +213,12 @@ public void testAlterTopic() { // normal user cannot alter topic1 (no privilege) assertThrows( - String.format("Can not access metadata {%s.%s.%s}.", CATALOG, SCHEMA, "topic1"), ForbiddenException.class, () -> { topicCatalogNormalUser.alterTopic( NameIdentifier.of(SCHEMA, "topic1"), TopicChange.updateComment("new comment")); - }); + }, + String.format("Can not access metadata {%s.%s.%s}.", CATALOG, SCHEMA, "topic1")); // grant normal user owner privilege on topic1 gravitinoMetalake.setOwner( MetadataObjects.of(ImmutableList.of(CATALOG, SCHEMA, "topic1"), MetadataObject.Type.TOPIC), @@ -241,11 +241,11 @@ public void testDropTopic() { Owner.Type.USER); // normal user cannot drop topic1 assertThrows( - String.format("Can not access metadata {%s.%s.%s}.", CATALOG, SCHEMA, "topic1"), ForbiddenException.class, () -> { topicCatalogNormalUser.dropTopic(NameIdentifier.of(SCHEMA, "topic1")); - }); + }, + String.format("Can not access metadata {%s.%s.%s}.", CATALOG, SCHEMA, "topic1")); // normal user can drop topic2 and topic3 (they created them) topicCatalogNormalUser.dropTopic(NameIdentifier.of(SCHEMA, "topic2")); topicCatalogNormalUser.dropTopic(NameIdentifier.of(SCHEMA, "topic3")); diff --git a/clients/client-java/src/test/java/org/apache/gravitino/client/integration/test/authorization/UserAuthorizationIT.java b/clients/client-java/src/test/java/org/apache/gravitino/client/integration/test/authorization/UserAuthorizationIT.java index 57bbc54ef2e..1abb9e36dbf 100644 --- a/clients/client-java/src/test/java/org/apache/gravitino/client/integration/test/authorization/UserAuthorizationIT.java +++ b/clients/client-java/src/test/java/org/apache/gravitino/client/integration/test/authorization/UserAuthorizationIT.java @@ -17,8 +17,8 @@ package org.apache.gravitino.client.integration.test.authorization; -import static org.junit.Assert.assertThrows; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; import com.google.common.collect.ImmutableList; import com.google.common.collect.Maps; @@ -58,11 +58,11 @@ public class UserAuthorizationIT extends BaseRestApiAuthorizationIT { @Order(1) public void testCreateUser() { assertThrows( - "Current user can not access metadata {testMetalake}", ForbiddenException.class, () -> { normalUserClient.loadMetalake(METALAKE).addUser("user1"); - }); + }, + "Current user can not access metadata {testMetalake}"); client.loadMetalake(METALAKE).addUser("user1"); client.loadMetalake(METALAKE).addUser("user2"); } @@ -94,11 +94,11 @@ public void testLoadUser() { GravitinoAdminClient user1Client = getClientByUser("user1"); user1Client.loadMetalake(METALAKE).getUser("user1"); assertThrows( - "Current user can not get user.", ForbiddenException.class, () -> { user1Client.loadMetalake(METALAKE).getUser("user2"); - }); + }, + "Current user can not get user."); client.loadMetalake(METALAKE).getUser("user1"); client.loadMetalake(METALAKE).getUser("user2"); } @@ -156,17 +156,17 @@ public void testBulkUserInterfaces() throws Exception { public void testRemoveUser() { GravitinoAdminClient user1Client = getClientByUser("user1"); assertThrows( - "Current user can not get user", ForbiddenException.class, () -> { user1Client.loadMetalake(METALAKE).removeUser("user2"); - }); + }, + "Current user can not get user"); assertThrows( - "Current user can not get user.", ForbiddenException.class, () -> { user1Client.loadMetalake(METALAKE).removeUser("user1"); - }); + }, + "Current user can not get user."); GravitinoMetalake gravitinoMetalake = client.loadMetalake(METALAKE); // owner can remove user gravitinoMetalake.removeUser("user2"); diff --git a/clients/filesystem-hadoop3/src/test/java/org/apache/gravitino/filesystem/hadoop/integration/test/FileSystemGCSCredentialAuthorizationIT.java b/clients/filesystem-hadoop3/src/test/java/org/apache/gravitino/filesystem/hadoop/integration/test/FileSystemGCSCredentialAuthorizationIT.java index 5bf78888eed..526d6a83c2c 100644 --- a/clients/filesystem-hadoop3/src/test/java/org/apache/gravitino/filesystem/hadoop/integration/test/FileSystemGCSCredentialAuthorizationIT.java +++ b/clients/filesystem-hadoop3/src/test/java/org/apache/gravitino/filesystem/hadoop/integration/test/FileSystemGCSCredentialAuthorizationIT.java @@ -21,6 +21,7 @@ import com.google.common.collect.Maps; import java.util.Map; +import org.apache.commons.lang3.StringUtils; import org.apache.gravitino.catalog.hadoop.fs.FileSystemUtils; import org.apache.gravitino.credential.GCSTokenCredential; import org.apache.gravitino.gcs.fs.GCSFileSystemProvider; @@ -28,7 +29,6 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; import org.junit.jupiter.api.condition.EnabledIf; -import org.junit.platform.commons.util.StringUtils; @EnabledIf(value = "isGCPConfigured", disabledReason = "GCP is not configured") public class FileSystemGCSCredentialAuthorizationIT diff --git a/clients/filesystem-hadoop3/src/test/java/org/apache/gravitino/filesystem/hadoop/integration/test/FileSystemS3CredentialAuthorizationIT.java b/clients/filesystem-hadoop3/src/test/java/org/apache/gravitino/filesystem/hadoop/integration/test/FileSystemS3CredentialAuthorizationIT.java index 9ba793a693a..9902fdffee4 100644 --- a/clients/filesystem-hadoop3/src/test/java/org/apache/gravitino/filesystem/hadoop/integration/test/FileSystemS3CredentialAuthorizationIT.java +++ b/clients/filesystem-hadoop3/src/test/java/org/apache/gravitino/filesystem/hadoop/integration/test/FileSystemS3CredentialAuthorizationIT.java @@ -21,6 +21,7 @@ import com.google.common.collect.Maps; import java.util.Map; +import org.apache.commons.lang3.StringUtils; import org.apache.gravitino.catalog.hadoop.fs.FileSystemUtils; import org.apache.gravitino.credential.S3TokenCredential; import org.apache.gravitino.s3.fs.S3FileSystemProvider; @@ -28,7 +29,6 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; import org.junit.jupiter.api.condition.EnabledIf; -import org.junit.platform.commons.util.StringUtils; /** * GVFS integration test that verifies credential vending with authorization enabled on S3.
diff --git a/clients/filesystem-hadoop3/src/test/java/org/apache/gravitino/filesystem/hadoop/integration/test/GravitinoVirtualFileSystemABSCredentialIT.java b/clients/filesystem-hadoop3/src/test/java/org/apache/gravitino/filesystem/hadoop/integration/test/GravitinoVirtualFileSystemABSCredentialIT.java index 99f21812bb4..9fefd4bacbe 100644 --- a/clients/filesystem-hadoop3/src/test/java/org/apache/gravitino/filesystem/hadoop/integration/test/GravitinoVirtualFileSystemABSCredentialIT.java +++ b/clients/filesystem-hadoop3/src/test/java/org/apache/gravitino/filesystem/hadoop/integration/test/GravitinoVirtualFileSystemABSCredentialIT.java @@ -26,6 +26,7 @@ import java.io.IOException; import java.util.Collections; import java.util.Map; +import org.apache.commons.lang3.StringUtils; import org.apache.gravitino.Catalog; import org.apache.gravitino.abs.fs.AzureFileSystemProvider; import org.apache.gravitino.catalog.hadoop.fs.FileSystemUtils; @@ -38,7 +39,6 @@ import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.condition.EnabledIf; -import org.junit.platform.commons.util.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/clients/filesystem-hadoop3/src/test/java/org/apache/gravitino/filesystem/hadoop/integration/test/GravitinoVirtualFileSystemABSIT.java b/clients/filesystem-hadoop3/src/test/java/org/apache/gravitino/filesystem/hadoop/integration/test/GravitinoVirtualFileSystemABSIT.java index 62c1d29195b..2f4cff4e715 100644 --- a/clients/filesystem-hadoop3/src/test/java/org/apache/gravitino/filesystem/hadoop/integration/test/GravitinoVirtualFileSystemABSIT.java +++ b/clients/filesystem-hadoop3/src/test/java/org/apache/gravitino/filesystem/hadoop/integration/test/GravitinoVirtualFileSystemABSIT.java @@ -26,6 +26,7 @@ import java.io.IOException; import java.util.Collections; import java.util.Map; +import org.apache.commons.lang3.StringUtils; import org.apache.gravitino.Catalog; import org.apache.gravitino.abs.fs.AzureFileSystemProvider; import org.apache.gravitino.catalog.hadoop.fs.FileSystemUtils; @@ -37,7 +38,6 @@ import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.condition.EnabledIf; -import org.junit.platform.commons.util.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/clients/filesystem-hadoop3/src/test/java/org/apache/gravitino/filesystem/hadoop/integration/test/GravitinoVirtualFileSystemCOSCredentialIT.java b/clients/filesystem-hadoop3/src/test/java/org/apache/gravitino/filesystem/hadoop/integration/test/GravitinoVirtualFileSystemCOSCredentialIT.java index f2a1b7a8183..af5f7167af2 100644 --- a/clients/filesystem-hadoop3/src/test/java/org/apache/gravitino/filesystem/hadoop/integration/test/GravitinoVirtualFileSystemCOSCredentialIT.java +++ b/clients/filesystem-hadoop3/src/test/java/org/apache/gravitino/filesystem/hadoop/integration/test/GravitinoVirtualFileSystemCOSCredentialIT.java @@ -26,6 +26,7 @@ import java.io.IOException; import java.util.Collections; import java.util.Map; +import org.apache.commons.lang3.StringUtils; import org.apache.gravitino.Catalog; import org.apache.gravitino.NameIdentifier; import org.apache.gravitino.catalog.hadoop.fs.FileSystemUtils; @@ -43,7 +44,6 @@ import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.condition.EnabledIf; -import org.junit.platform.commons.util.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/clients/filesystem-hadoop3/src/test/java/org/apache/gravitino/filesystem/hadoop/integration/test/GravitinoVirtualFileSystemCOSIT.java b/clients/filesystem-hadoop3/src/test/java/org/apache/gravitino/filesystem/hadoop/integration/test/GravitinoVirtualFileSystemCOSIT.java index e8880244a01..08087c6a243 100644 --- a/clients/filesystem-hadoop3/src/test/java/org/apache/gravitino/filesystem/hadoop/integration/test/GravitinoVirtualFileSystemCOSIT.java +++ b/clients/filesystem-hadoop3/src/test/java/org/apache/gravitino/filesystem/hadoop/integration/test/GravitinoVirtualFileSystemCOSIT.java @@ -25,6 +25,7 @@ import java.io.IOException; import java.util.Collections; import java.util.Map; +import org.apache.commons.lang3.StringUtils; import org.apache.gravitino.Catalog; import org.apache.gravitino.catalog.hadoop.fs.FileSystemUtils; import org.apache.gravitino.cos.fs.COSFileSystemProvider; @@ -36,7 +37,6 @@ import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.condition.EnabledIf; -import org.junit.platform.commons.util.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/clients/filesystem-hadoop3/src/test/java/org/apache/gravitino/filesystem/hadoop/integration/test/GravitinoVirtualFileSystemOSSCredentialIT.java b/clients/filesystem-hadoop3/src/test/java/org/apache/gravitino/filesystem/hadoop/integration/test/GravitinoVirtualFileSystemOSSCredentialIT.java index 00d6fa8cb26..81cbac1efca 100644 --- a/clients/filesystem-hadoop3/src/test/java/org/apache/gravitino/filesystem/hadoop/integration/test/GravitinoVirtualFileSystemOSSCredentialIT.java +++ b/clients/filesystem-hadoop3/src/test/java/org/apache/gravitino/filesystem/hadoop/integration/test/GravitinoVirtualFileSystemOSSCredentialIT.java @@ -25,6 +25,7 @@ import java.io.IOException; import java.util.Collections; import java.util.Map; +import org.apache.commons.lang3.StringUtils; import org.apache.gravitino.Catalog; import org.apache.gravitino.catalog.hadoop.fs.FileSystemUtils; import org.apache.gravitino.credential.CredentialConstants; @@ -38,7 +39,6 @@ import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.condition.EnabledIf; -import org.junit.platform.commons.util.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/clients/filesystem-hadoop3/src/test/java/org/apache/gravitino/filesystem/hadoop/integration/test/GravitinoVirtualFileSystemOSSIT.java b/clients/filesystem-hadoop3/src/test/java/org/apache/gravitino/filesystem/hadoop/integration/test/GravitinoVirtualFileSystemOSSIT.java index eb76bfd641d..3dc4832c099 100644 --- a/clients/filesystem-hadoop3/src/test/java/org/apache/gravitino/filesystem/hadoop/integration/test/GravitinoVirtualFileSystemOSSIT.java +++ b/clients/filesystem-hadoop3/src/test/java/org/apache/gravitino/filesystem/hadoop/integration/test/GravitinoVirtualFileSystemOSSIT.java @@ -25,6 +25,7 @@ import java.io.IOException; import java.util.Collections; import java.util.Map; +import org.apache.commons.lang3.StringUtils; import org.apache.gravitino.Catalog; import org.apache.gravitino.catalog.hadoop.fs.FileSystemUtils; import org.apache.gravitino.integration.test.util.GravitinoITUtils; @@ -36,7 +37,6 @@ import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.condition.EnabledIf; -import org.junit.platform.commons.util.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/clients/filesystem-hadoop3/src/test/java/org/apache/gravitino/filesystem/hadoop/integration/test/GravitinoVirtualFileSystemS3CredentialIT.java b/clients/filesystem-hadoop3/src/test/java/org/apache/gravitino/filesystem/hadoop/integration/test/GravitinoVirtualFileSystemS3CredentialIT.java index e02eaa9d673..2d1cb55e3fd 100644 --- a/clients/filesystem-hadoop3/src/test/java/org/apache/gravitino/filesystem/hadoop/integration/test/GravitinoVirtualFileSystemS3CredentialIT.java +++ b/clients/filesystem-hadoop3/src/test/java/org/apache/gravitino/filesystem/hadoop/integration/test/GravitinoVirtualFileSystemS3CredentialIT.java @@ -25,6 +25,7 @@ import java.io.IOException; import java.util.Collections; import java.util.Map; +import org.apache.commons.lang3.StringUtils; import org.apache.gravitino.Catalog; import org.apache.gravitino.catalog.hadoop.fs.FileSystemUtils; import org.apache.gravitino.credential.CredentialConstants; @@ -38,7 +39,6 @@ import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.condition.EnabledIf; -import org.junit.platform.commons.util.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/config/checkstyle/checkstyle.xml b/config/checkstyle/checkstyle.xml new file mode 100644 index 00000000000..4f1453157f9 --- /dev/null +++ b/config/checkstyle/checkstyle.xml @@ -0,0 +1,35 @@ + + + + + + + + + + + + + + + diff --git a/core/build.gradle.kts b/core/build.gradle.kts index e53c185baed..f5a8fb70245 100644 --- a/core/build.gradle.kts +++ b/core/build.gradle.kts @@ -80,6 +80,8 @@ dependencies { testImplementation(libs.awaitility) testImplementation(libs.junit.jupiter.api) testImplementation(libs.junit.jupiter.params) + testImplementation(libs.junit.platform.commons) + testImplementation(libs.junit.platform.engine) testImplementation(libs.junit.platform.testkit) testImplementation(libs.mockito.core) testImplementation(libs.mockito.inline) diff --git a/core/src/test/java/org/apache/gravitino/storage/TestSQLScripts.java b/core/src/test/java/org/apache/gravitino/storage/TestSQLScripts.java index 813934acaa1..19e05d9d5d3 100644 --- a/core/src/test/java/org/apache/gravitino/storage/TestSQLScripts.java +++ b/core/src/test/java/org/apache/gravitino/storage/TestSQLScripts.java @@ -45,7 +45,6 @@ import org.apache.ibatis.session.SqlSession; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.TestTemplate; -import org.opentest4j.AssertionFailedError; @DatabaseFixture(resetStrategy = ResetStrategy.FRESH_DATABASE) public class TestSQLScripts extends TestJDBCBackend { @@ -295,8 +294,8 @@ public void testUpgradeToTwoZeroIsIdempotent() throws SQLException, IOException executeStatements( List.of("ALTER TABLE schema_meta DROP INDEX schema_meta_idx_mid"), "missing target index"); - AssertionFailedError failure = - Assertions.assertThrows(AssertionFailedError.class, () -> executeScript(upgrade)); + AssertionError failure = + Assertions.assertThrows(AssertionError.class, () -> executeScript(upgrade)); Assertions.assertInstanceOf(SQLException.class, failure.getCause()); Assertions.assertTrue( failure.getCause().getMessage().contains("'idx_mid'"), failure.getMessage()); @@ -412,9 +411,9 @@ public void testUpgradeToTwoZeroRejectsUnexpectedPrimaryKey() throws SQLExceptio List.of("ALTER TABLE table_version_info DROP INDEX uk_table_id_version_deleted_at"), "remove legacy index"); } - AssertionFailedError failure = + AssertionError failure = Assertions.assertThrows( - AssertionFailedError.class, + AssertionError.class, () -> executeScript(scriptDir.resolve("upgrade-1.3.0-to-2.0.0-mysql.sql").toFile())); Assertions.assertInstanceOf(SQLException.class, failure.getCause()); String expectedError = keepLegacyIndex ? "primary key" : "uk_table_id_version_deleted_at"; diff --git a/flink-connector/flink-common/src/test/java/org/apache/gravitino/flink/connector/store/TestGravitinoCatalogStore.java b/flink-connector/flink-common/src/test/java/org/apache/gravitino/flink/connector/store/TestGravitinoCatalogStore.java index 046422b2ce3..5b3b1864761 100644 --- a/flink-connector/flink-common/src/test/java/org/apache/gravitino/flink/connector/store/TestGravitinoCatalogStore.java +++ b/flink-connector/flink-common/src/test/java/org/apache/gravitino/flink/connector/store/TestGravitinoCatalogStore.java @@ -18,10 +18,10 @@ */ package org.apache.gravitino.flink.connector.store; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertSame; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertSame; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.fail; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; @@ -51,14 +51,14 @@ import org.apache.gravitino.secret.SecretProviderRegistry; import org.apache.gravitino.secret.SupportsSecrets; import org.apache.gravitino.secret.memory.InMemorySecretsProvider; -import org.junit.Before; -import org.junit.Test; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; public class TestGravitinoCatalogStore { private GravitinoCatalogManager gravitinoCatalogMockManager; private GravitinoCatalogStore gravitinoCatalogStore; - @Before + @BeforeEach public void setupCatalogStore() { gravitinoCatalogMockManager = mock(GravitinoCatalogManager.class); gravitinoCatalogStore = new GravitinoCatalogStore(gravitinoCatalogMockManager); @@ -97,8 +97,8 @@ public void testRemoveCatalog_whenCatalogNotExists_ignoreFlagFalse_shouldThrow() fail("Expected CatalogException to be thrown"); } catch (CatalogException e) { assertTrue( - "Expected failure message to contain 'Failed to remove the catalog:'", - e.getMessage().contains("Failed to remove the catalog:")); + e.getMessage().contains("Failed to remove the catalog:"), + "Expected failure message to contain 'Failed to remove the catalog:'"); } verify(gravitinoCatalogMockManager).dropCatalog(catalogName); } @@ -113,9 +113,9 @@ public void testRemoveCatalog_UnexpectedException_shouldThrow() { fail("Expected CatalogException to be thrown"); } catch (CatalogException e) { assertTrue( - "Expected failure message to contain 'Failed to remove the catalog:'", - e.getMessage().contains("Failed to remove the catalog:")); - assertTrue("Expected cause to be RuntimeException", e.getCause() instanceof RuntimeException); + e.getMessage().contains("Failed to remove the catalog:"), + "Expected failure message to contain 'Failed to remove the catalog:'"); + assertTrue(e.getCause() instanceof RuntimeException, "Expected cause to be RuntimeException"); } verify(gravitinoCatalogMockManager).dropCatalog(catalogName); } diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 0c90a42a6fc..c97090f6d03 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -119,6 +119,9 @@ h2db = "2.2.224" kafka = "3.4.0" curator = "2.12.0" awaitility = "4.2.1" +# Lowest AssertJ among the Trino test stacks; newer Trino versions resolve it upward. +assertj = "3.25.3" +checkstyle = "10.12.5" servlet = "3.1.0" jodd = "3.5.2" flink119 = "1.19.0" @@ -222,6 +225,8 @@ lombok = { group = "org.projectlombok", name = "lombok", version.ref = "lombok" junit-jupiter-api = { group = "org.junit.jupiter", name = "junit-jupiter-api", version.ref = "junit" } junit-jupiter-params = { group = "org.junit.jupiter", name = "junit-jupiter-params", version.ref = "junit" } junit-jupiter-engine = { group = "org.junit.jupiter", name = "junit-jupiter-engine", version.ref = "junit" } +junit-platform-commons = { group = "org.junit.platform", name = "junit-platform-commons", version.ref = "junit-platform" } +junit-platform-engine = { group = "org.junit.platform", name = "junit-platform-engine", version.ref = "junit-platform" } junit-platform-testkit = { group = "org.junit.platform", name = "junit-platform-testkit", version.ref = "junit-platform" } slf4j-api = { group = "org.slf4j", name = "slf4j-api", version.ref = "slf4j" } slf4j-simple = { group = "org.slf4j", name = "slf4j-simple", version.ref = "slf4j" } @@ -324,6 +329,7 @@ trino-client= { group = "io.trino", name = "trino-client", version.ref = "trino" sqlite-jdbc = { group = "org.xerial", name = "sqlite-jdbc", version.ref = "sqlite-jdbc" } commons-dbcp2 = { group = "org.apache.commons", name = "commons-dbcp2", version.ref = "commons-dbcp2" } testcontainers = { group = "org.testcontainers", name = "testcontainers", version.ref = "testcontainers" } +testcontainers-jdbc = { group = "org.testcontainers", name = "jdbc", version.ref = "testcontainers" } testcontainers-clickhouse = { group = "org.testcontainers", name = "clickhouse", version.ref = "testcontainers" } testcontainers-mysql = { group = "org.testcontainers", name = "mysql", version.ref = "testcontainers" } testcontainers-postgresql = { group = "org.testcontainers", name = "postgresql", version.ref = "testcontainers" } @@ -376,6 +382,7 @@ selenium = { group = "org.seleniumhq.selenium", name = "selenium-java", version. rauschig = { group = "org.rauschig", name = "jarchivelib", version.ref = "rauschig" } mybatis = { group = "org.mybatis", name = "mybatis", version.ref = "mybatis"} h2db = { group = "com.h2database", name = "h2", version.ref = "h2db"} +assertj-core = { group = "org.assertj", name = "assertj-core", version.ref = "assertj" } awaitility = { group = "org.awaitility", name = "awaitility", version.ref = "awaitility" } servlet = { group = "javax.servlet", name = "javax.servlet-api", version.ref = "servlet" } mail = { group = "javax.mail", name = "mail", version.ref = "mail" } diff --git a/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTOSSIT.java b/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTOSSIT.java index 28e35c2657f..30c82143ff9 100644 --- a/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTOSSIT.java +++ b/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTOSSIT.java @@ -21,6 +21,7 @@ import java.util.HashMap; import java.util.Map; +import org.apache.commons.lang3.StringUtils; import org.apache.gravitino.catalog.lakehouse.iceberg.IcebergConstants; import org.apache.gravitino.credential.CredentialConstants; import org.apache.gravitino.credential.OSSTokenCredential; @@ -29,7 +30,6 @@ import org.apache.gravitino.integration.test.util.ITUtils; import org.apache.gravitino.storage.OSSProperties; import org.junit.jupiter.api.condition.EnabledIfEnvironmentVariable; -import org.junit.platform.commons.util.StringUtils; @EnabledIfEnvironmentVariable(named = "GRAVITINO_TEST_CLOUD_IT", matches = "true") public class IcebergRESTOSSIT extends IcebergRESTJdbcCatalogIT { diff --git a/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTS3TokenAuthorizationIT.java b/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTS3TokenAuthorizationIT.java index 10fd8f15e43..f1149e2db9c 100644 --- a/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTS3TokenAuthorizationIT.java +++ b/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTS3TokenAuthorizationIT.java @@ -21,6 +21,7 @@ import java.util.HashMap; import java.util.Map; +import org.apache.commons.lang3.StringUtils; import org.apache.gravitino.catalog.lakehouse.iceberg.IcebergConstants; import org.apache.gravitino.credential.CredentialConstants; import org.apache.gravitino.credential.S3TokenCredential; @@ -29,7 +30,6 @@ import org.apache.gravitino.storage.S3Properties; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.condition.EnabledIfEnvironmentVariable; -import org.junit.platform.commons.util.StringUtils; @SuppressWarnings("FormatStringAnnotation") @EnabledIfEnvironmentVariable(named = "GRAVITINO_TEST_CLOUD_IT", matches = "true") diff --git a/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTS3TokenIT.java b/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTS3TokenIT.java index 743f3bbc482..998fa22d2c4 100644 --- a/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTS3TokenIT.java +++ b/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTS3TokenIT.java @@ -23,6 +23,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import org.apache.commons.lang3.StringUtils; import org.apache.gravitino.catalog.lakehouse.iceberg.IcebergConstants; import org.apache.gravitino.credential.CredentialConstants; import org.apache.gravitino.credential.S3TokenCredential; @@ -34,7 +35,6 @@ import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.condition.EnabledIfEnvironmentVariable; -import org.junit.platform.commons.util.StringUtils; @SuppressWarnings("FormatStringAnnotation") @EnabledIfEnvironmentVariable(named = "GRAVITINO_TEST_CLOUD_IT", matches = "true") diff --git a/iceberg/iceberg-rest-trino-it/build.gradle.kts b/iceberg/iceberg-rest-trino-it/build.gradle.kts index 670e7bbf2b0..f08cdfdb3a0 100644 --- a/iceberg/iceberg-rest-trino-it/build.gradle.kts +++ b/iceberg/iceberg-rest-trino-it/build.gradle.kts @@ -68,6 +68,7 @@ dependencies { testRuntimeOnly(libs.awaitility) testRuntimeOnly(libs.commons.lang3) testImplementation(libs.postgresql.driver) + testImplementation(libs.junit.jupiter.api) testImplementation(libs.testcontainers) testImplementation(libs.testcontainers.postgresql) diff --git a/integration-test-common/build.gradle.kts b/integration-test-common/build.gradle.kts index a14fed40cb5..614618dbc22 100644 --- a/integration-test-common/build.gradle.kts +++ b/integration-test-common/build.gradle.kts @@ -47,8 +47,10 @@ dependencies { testImplementation(libs.commons.io) testImplementation(libs.guava) testImplementation(libs.httpclient5) + testImplementation(libs.junit.jupiter.api) testImplementation(libs.mockito.core) testImplementation(libs.testcontainers) + testImplementation(libs.testcontainers.jdbc) testImplementation(libs.testcontainers.mysql) testImplementation(libs.testcontainers.postgresql) testImplementation(libs.ranger.intg) { diff --git a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/MiniGravitino.java b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/MiniGravitino.java index 841785548bd..96285455760 100644 --- a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/MiniGravitino.java +++ b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/MiniGravitino.java @@ -43,6 +43,7 @@ import java.util.concurrent.TimeUnit; import javax.annotation.Nullable; import org.apache.commons.io.FileUtils; +import org.apache.commons.lang3.StringUtils; import org.apache.gravitino.Config; import org.apache.gravitino.Configs; import org.apache.gravitino.auth.AuthenticatorType; @@ -57,7 +58,6 @@ import org.apache.gravitino.server.GravitinoServer; import org.apache.gravitino.server.ServerConfig; import org.apache.gravitino.server.web.JettyServerConfig; -import org.junit.platform.commons.util.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/lance/lance-rest-server/build.gradle.kts b/lance/lance-rest-server/build.gradle.kts index b528c633955..657cfb5b2a0 100644 --- a/lance/lance-rest-server/build.gradle.kts +++ b/lance/lance-rest-server/build.gradle.kts @@ -144,6 +144,7 @@ dependencies { testImplementation(libs.junit.jupiter.api) testImplementation(libs.junit.jupiter.params) + testImplementation(libs.mockito.core) testImplementation(libs.mockito.inline) testImplementation(libs.mysql.driver) testImplementation(libs.postgresql.driver) diff --git a/plugins/idp-basic/build.gradle.kts b/plugins/idp-basic/build.gradle.kts index 626a78d8092..12255f68108 100644 --- a/plugins/idp-basic/build.gradle.kts +++ b/plugins/idp-basic/build.gradle.kts @@ -58,6 +58,7 @@ dependencies { } testImplementation(libs.junit.jupiter.api) testImplementation(libs.junit.jupiter.params) + testImplementation(libs.mockito.core) testImplementation(libs.mockito.inline) testImplementation(libs.mysql.driver) testImplementation(libs.postgresql.driver) diff --git a/server-common/build.gradle.kts b/server-common/build.gradle.kts index aa48f710603..a90624bd048 100644 --- a/server-common/build.gradle.kts +++ b/server-common/build.gradle.kts @@ -58,6 +58,7 @@ dependencies { testImplementation(libs.commons.io) testImplementation(libs.junit.jupiter.api) testImplementation(libs.junit.jupiter.params) + testImplementation(libs.mockito.core) testImplementation(libs.mockito.inline) testImplementation(libs.commons.io) testImplementation(libs.minikdc) { diff --git a/server/build.gradle.kts b/server/build.gradle.kts index 895c64a5118..fe6b4289e2e 100644 --- a/server/build.gradle.kts +++ b/server/build.gradle.kts @@ -64,6 +64,7 @@ dependencies { testImplementation(libs.junit.jupiter.api) testImplementation(libs.junit.jupiter.params) + testImplementation(libs.mockito.core) testImplementation(libs.mockito.inline) testImplementation(libs.ognl) diff --git a/spark-connector/spark-common/src/test/java/org/apache/gravitino/spark/connector/integration/test/authorization/SparkAuthorizationIT.java b/spark-connector/spark-common/src/test/java/org/apache/gravitino/spark/connector/integration/test/authorization/SparkAuthorizationIT.java index 8e7e0b6d0a7..4a85a596362 100644 --- a/spark-connector/spark-common/src/test/java/org/apache/gravitino/spark/connector/integration/test/authorization/SparkAuthorizationIT.java +++ b/spark-connector/spark-common/src/test/java/org/apache/gravitino/spark/connector/integration/test/authorization/SparkAuthorizationIT.java @@ -18,7 +18,7 @@ package org.apache.gravitino.spark.connector.integration.test.authorization; import static org.apache.gravitino.integration.test.util.TestDatabaseName.MYSQL_CATALOG_MYSQL_IT; -import static org.junit.Assert.assertThrows; +import static org.junit.jupiter.api.Assertions.assertThrows; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; @@ -222,11 +222,11 @@ public void testListTable() { ImmutableList.of(JDBC_CATALOG, JDBC_DATABASE), MetadataObject.Type.SCHEMA), ImmutableList.of(Privileges.CreateTable.deny())); assertThrows( - "Can not access metadata {" + JDBC_CATALOG + "." + JDBC_DATABASE + "}.", ForbiddenException.class, () -> { normalUserSparkSession.sql("CREATE TABLE table_d(a STRING)"); - }); + }, + "Can not access metadata {" + JDBC_CATALOG + "." + JDBC_DATABASE + "}."); List tables = normalUserSparkSession.sql("show tables").collectAsList(); assertEqualsRows( ImmutableList.of( @@ -252,22 +252,22 @@ public void testAlterLoad() { RowFactory.create("jdbcDatabase", "table_c", false)), tables); assertThrows( - "Can not access metadata {" + JDBC_CATALOG + "." + JDBC_DATABASE + "}.", ForbiddenException.class, () -> { normalUserSparkSession.sql("ALTER TABLE table_c ADD COLUMNS (b STRING);"); - }); + }, + "Can not access metadata {" + JDBC_CATALOG + "." + JDBC_DATABASE + "}."); } @Test @Order(4) public void testDropTable() { assertThrows( - "Can not access metadata {" + JDBC_CATALOG + "." + JDBC_DATABASE + "}.", ForbiddenException.class, () -> { normalUserSparkSession.sql("DROP TABLE table_c"); - }); + }, + "Can not access metadata {" + JDBC_CATALOG + "." + JDBC_DATABASE + "}."); normalUserSparkSession.sql("DROP TABLE table_a"); } diff --git a/spark-connector/spark-common/src/test/java/org/apache/gravitino/spark/connector/integration/test/sql/SparkTestConfig.java b/spark-connector/spark-common/src/test/java/org/apache/gravitino/spark/connector/integration/test/sql/SparkTestConfig.java index d73986b18c6..48fb4d6f1b7 100644 --- a/spark-connector/spark-common/src/test/java/org/apache/gravitino/spark/connector/integration/test/sql/SparkTestConfig.java +++ b/spark-connector/spark-common/src/test/java/org/apache/gravitino/spark/connector/integration/test/sql/SparkTestConfig.java @@ -22,11 +22,11 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.List; +import org.apache.commons.lang3.StringUtils; import org.apache.gravitino.Config; import org.apache.gravitino.config.ConfigBuilder; import org.apache.gravitino.config.ConfigConstants; import org.apache.gravitino.config.ConfigEntry; -import org.junit.platform.commons.util.StringUtils; public class SparkTestConfig extends Config { private static final String DEFAULT_BASE_DIR = diff --git a/spark-connector/v3.5/spark/build.gradle.kts b/spark-connector/v3.5/spark/build.gradle.kts index 704a7a31776..0d57caa570e 100644 --- a/spark-connector/v3.5/spark/build.gradle.kts +++ b/spark-connector/v3.5/spark/build.gradle.kts @@ -211,6 +211,7 @@ dependencies { exclude("org.eclipse.jetty.orbit", "javax.servlet") exclude("org.slf4j") } + testImplementation(libs.commons.lang3) testImplementation(libs.junit.jupiter.api) testImplementation(libs.junit.jupiter.params) testImplementation(libs.lance.namespace.apache.client) diff --git a/spark-connector/v4.0/spark/build.gradle.kts b/spark-connector/v4.0/spark/build.gradle.kts index 8a9364add38..00e77a67d2d 100644 --- a/spark-connector/v4.0/spark/build.gradle.kts +++ b/spark-connector/v4.0/spark/build.gradle.kts @@ -241,6 +241,7 @@ dependencies { exclude("org.eclipse.jetty.orbit", "javax.servlet") exclude("org.slf4j") } + testImplementation(libs.commons.lang3) testImplementation(libs.junit.jupiter.api) testImplementation(libs.junit.jupiter.params) testImplementation(libs.lance.namespace.apache.client) diff --git a/trino-connector/trino-connector-440-445/build.gradle.kts b/trino-connector/trino-connector-440-445/build.gradle.kts index af4f5158eaf..42865aa4c0a 100644 --- a/trino-connector/trino-connector-440-445/build.gradle.kts +++ b/trino-connector/trino-connector-440-445/build.gradle.kts @@ -62,7 +62,9 @@ dependencies { compileOnly("io.trino:trino-spi:$trinoVersion") { exclude("org.apache.logging.log4j") } + testImplementation(libs.assertj.core) testImplementation(libs.awaitility) + testImplementation(libs.junit.jupiter.api) testImplementation(libs.mockito.core) testImplementation(libs.mysql.driver) testImplementation("io.trino:trino-memory:$trinoVersion") { diff --git a/trino-connector/trino-connector-446-451/build.gradle.kts b/trino-connector/trino-connector-446-451/build.gradle.kts index e4e75eda71b..13063774c50 100644 --- a/trino-connector/trino-connector-446-451/build.gradle.kts +++ b/trino-connector/trino-connector-446-451/build.gradle.kts @@ -62,7 +62,9 @@ dependencies { compileOnly("io.trino:trino-spi:$trinoVersion") { exclude("org.apache.logging.log4j") } + testImplementation(libs.assertj.core) testImplementation(libs.awaitility) + testImplementation(libs.junit.jupiter.api) testImplementation(libs.mockito.core) testImplementation(libs.mysql.driver) testImplementation("io.trino:trino-memory:$trinoVersion") { diff --git a/trino-connector/trino-connector-452-468/build.gradle.kts b/trino-connector/trino-connector-452-468/build.gradle.kts index 767903355fc..391437c8e8c 100644 --- a/trino-connector/trino-connector-452-468/build.gradle.kts +++ b/trino-connector/trino-connector-452-468/build.gradle.kts @@ -62,7 +62,9 @@ dependencies { compileOnly("io.trino:trino-spi:$trinoVersion") { exclude("org.apache.logging.log4j") } + testImplementation(libs.assertj.core) testImplementation(libs.awaitility) + testImplementation(libs.junit.jupiter.api) testImplementation(libs.mockito.core) testImplementation(libs.mysql.driver) testImplementation("io.trino:trino-memory:$trinoVersion") { diff --git a/trino-connector/trino-connector-469-472/build.gradle.kts b/trino-connector/trino-connector-469-472/build.gradle.kts index e8ca4e29943..4714132eab1 100644 --- a/trino-connector/trino-connector-469-472/build.gradle.kts +++ b/trino-connector/trino-connector-469-472/build.gradle.kts @@ -67,7 +67,9 @@ dependencies { compileOnly("io.trino:trino-spi:$trinoVersion") { exclude("org.apache.logging.log4j") } + testImplementation(libs.assertj.core) testImplementation(libs.awaitility) + testImplementation(libs.junit.jupiter.api) testImplementation(libs.mockito.core) testImplementation(libs.mysql.driver) testImplementation("io.trino:trino-memory:$trinoVersion") { diff --git a/trino-connector/trino-connector-473-478/build.gradle.kts b/trino-connector/trino-connector-473-478/build.gradle.kts index f41b2aba877..735307d963e 100644 --- a/trino-connector/trino-connector-473-478/build.gradle.kts +++ b/trino-connector/trino-connector-473-478/build.gradle.kts @@ -62,6 +62,7 @@ dependencies { compileOnly("io.trino:trino-spi:$trinoVersion") { exclude("org.apache.logging.log4j") } + testImplementation(libs.assertj.core) testImplementation(libs.awaitility) testImplementation(libs.mockito.core) testImplementation(libs.mysql.driver) diff --git a/trino-connector/trino-connector/build.gradle.kts b/trino-connector/trino-connector/build.gradle.kts index a43e1a0faac..f1ad7864d78 100644 --- a/trino-connector/trino-connector/build.gradle.kts +++ b/trino-connector/trino-connector/build.gradle.kts @@ -58,7 +58,9 @@ dependencies { compileOnly("io.trino:trino-spi:$trinoVersion") { exclude("org.apache.logging.log4j") } + testImplementation(libs.assertj.core) testImplementation(libs.awaitility) + testImplementation(libs.junit.jupiter.api) testImplementation(libs.mockito.core) testImplementation(libs.mysql.driver) testImplementation("io.trino:trino-memory:$trinoVersion") { From 2037142ac5dc2e5d15813c909f0687ca580c1d92 Mon Sep 17 00:00:00 2001 From: Nevin Zheng Date: Wed, 21 Oct 2026 08:53:38 -0700 Subject: [PATCH 02/14] [#14105] improvement(build): Upgrade JUnit to 6.1.3 --- build.gradle.kts | 29 +++++++++++++++---- .../relational/BackendTestExtension.java | 2 +- gradle/libs.versions.toml | 11 +++---- .../iceberg-rest-trino-it/build.gradle.kts | 7 ++--- integration-test-common/build.gradle.kts | 1 - .../test/util/CloseContainerExtension.java | 6 ++-- plugins/k8s-job-executor/build.gradle.kts | 1 + .../trino-connector-473-478/build.gradle.kts | 2 +- 8 files changed, 38 insertions(+), 21 deletions(-) diff --git a/build.gradle.kts b/build.gradle.kts index 46fa0bded6d..c722594f85f 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -584,6 +584,8 @@ fun excludePackagesForSparkConnector(project: Project) { val commonsBeanutilsVersion: String = libs.versions.commons.beanutils.get() val checkstyleVersion: String = libs.versions.checkstyle.get() +val junitBom = libs.junit.bom +val junitPlatformLauncher = libs.junit.platform.launcher subprojects { // Gravitino Python client project didn't need to apply the java plugin @@ -711,13 +713,25 @@ subprojects { options.release.set(8) } - // Tests still need Java 17 to compile against dependencies that only publish Java 17 variants. - tasks.named("compileTestJava") { - options.release.set(17) - } + // Tests and test fixtures still need Java 17 to compile against dependencies that only + // publish Java 17 variants, such as JUnit 6. + tasks.matching { it.name in setOf("compileTestJava", "compileTestFixturesJava") } + .withType() + .configureEach { + options.release.set(17) + } val targetJvmVersionAttribute = TargetJvmVersion.TARGET_JVM_VERSION_ATTRIBUTE - configurations.matching { it.name in setOf("testCompileClasspath", "testRuntimeClasspath") } + configurations.matching { + it.name in setOf( + "testCompileClasspath", + "testRuntimeClasspath", + "testFixturesCompileClasspath", + "testFixturesRuntimeClasspath", + "testFixturesApiElements", + "testFixturesRuntimeElements" + ) + } .configureEach { attributes.attribute(targetJvmVersionAttribute, 17) } @@ -744,6 +758,11 @@ subprojects { apply(plugin = "net.ltgt.errorprone") dependencies { errorprone("com.google.errorprone:error_prone_core:2.10.0") + // Align every JUnit artifact (including transitive ones) on one version. JUnit 6 also needs + // the matching launcher on the test runtime classpath, otherwise Gradle falls back to its + // bundled 1.x launcher, which cannot drive a 6.x engine. + testImplementation(platform(junitBom)) + testRuntimeOnly(junitPlatformLauncher) } tasks.withType().configureEach { diff --git a/core/src/test/java/org/apache/gravitino/storage/relational/BackendTestExtension.java b/core/src/test/java/org/apache/gravitino/storage/relational/BackendTestExtension.java index 0ff86858234..39b7dcfb64e 100644 --- a/core/src/test/java/org/apache/gravitino/storage/relational/BackendTestExtension.java +++ b/core/src/test/java/org/apache/gravitino/storage/relational/BackendTestExtension.java @@ -246,7 +246,7 @@ private ResetStrategy resetStrategy(ExtensionContext context) { * Closes with the root store, in reverse insertion order, so before the containers {@code * CloseContainerExtension} registered earlier. */ - static class LiveBackend implements ExtensionContext.Store.CloseableResource { + static class LiveBackend implements AutoCloseable { private final String backendType; private final RelationalBackend backend; private final Config config; diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index c97090f6d03..408eb44dcf2 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -24,8 +24,7 @@ azure-storage-file-datalake = "12.20.0" reactor-netty-http = "1.2.1" reactor-netty-core = "1.2.1" # JUnit Jupiter 5.x ships with JUnit Platform 1.x of the same minor; bump them together. -junit = "5.8.1" -junit-platform = "1.8.1" +junit = "6.1.3" jackson = "2.15.2" snakeyaml = "2.0" gson = "2.11.0" @@ -225,9 +224,11 @@ lombok = { group = "org.projectlombok", name = "lombok", version.ref = "lombok" junit-jupiter-api = { group = "org.junit.jupiter", name = "junit-jupiter-api", version.ref = "junit" } junit-jupiter-params = { group = "org.junit.jupiter", name = "junit-jupiter-params", version.ref = "junit" } junit-jupiter-engine = { group = "org.junit.jupiter", name = "junit-jupiter-engine", version.ref = "junit" } -junit-platform-commons = { group = "org.junit.platform", name = "junit-platform-commons", version.ref = "junit-platform" } -junit-platform-engine = { group = "org.junit.platform", name = "junit-platform-engine", version.ref = "junit-platform" } -junit-platform-testkit = { group = "org.junit.platform", name = "junit-platform-testkit", version.ref = "junit-platform" } +junit-platform-commons = { group = "org.junit.platform", name = "junit-platform-commons", version.ref = "junit" } +junit-platform-engine = { group = "org.junit.platform", name = "junit-platform-engine", version.ref = "junit" } +junit-platform-launcher = { group = "org.junit.platform", name = "junit-platform-launcher", version.ref = "junit" } +junit-platform-testkit = { group = "org.junit.platform", name = "junit-platform-testkit", version.ref = "junit" } +junit-bom = { group = "org.junit", name = "junit-bom", version.ref = "junit" } slf4j-api = { group = "org.slf4j", name = "slf4j-api", version.ref = "slf4j" } slf4j-simple = { group = "org.slf4j", name = "slf4j-simple", version.ref = "slf4j" } slf4j-jdk14 = { group = "org.slf4j", name = "slf4j-jdk14", version.ref = "slf4j" } diff --git a/iceberg/iceberg-rest-trino-it/build.gradle.kts b/iceberg/iceberg-rest-trino-it/build.gradle.kts index f08cdfdb3a0..876b82a39f4 100644 --- a/iceberg/iceberg-rest-trino-it/build.gradle.kts +++ b/iceberg/iceberg-rest-trino-it/build.gradle.kts @@ -81,12 +81,9 @@ dependencies { testRuntimeOnly(libs.jwt.impl) testImplementation(libs.guava) testImplementation(libs.slf4j.api) - // Trino 478's test stack forces JUnit to 6.0.0; pin the BOM to match so the platform launcher, - // engine and commons jars stay on the same version (a mismatch makes Gradle's TagFilter crash - // with NoSuchMethodError on CollectionUtils.toUnmodifiableList). - testImplementation(platform("org.junit:junit-bom:6.0.0")) + // The root build aligns JUnit through the BOM and adds the matching launcher; Trino 478's own + // 6.0.x JUnit dependencies resolve up to that version. testImplementation("org.junit.jupiter:junit-jupiter") - testRuntimeOnly("org.junit.platform:junit-platform-launcher") } tasks.withType().configureEach { diff --git a/integration-test-common/build.gradle.kts b/integration-test-common/build.gradle.kts index 614618dbc22..e20e6c7e1f2 100644 --- a/integration-test-common/build.gradle.kts +++ b/integration-test-common/build.gradle.kts @@ -67,7 +67,6 @@ dependencies { testImplementation(libs.hadoop3.client.api) testImplementation(libs.hadoop3.client.runtime) - testImplementation(platform("org.junit:junit-bom:5.9.1")) testImplementation("org.junit.jupiter:junit-jupiter") } diff --git a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/util/CloseContainerExtension.java b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/util/CloseContainerExtension.java index 28ebc741d72..1e1b2aa41d4 100644 --- a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/util/CloseContainerExtension.java +++ b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/util/CloseContainerExtension.java @@ -32,14 +32,14 @@ public class CloseContainerExtension implements BeforeAllCallback { @Override public void beforeAll(ExtensionContext extensionContext) { - // Store.getOrComputeIfAbsent is atomic, so concurrent classes register the closer once. + // Store.computeIfAbsent is atomic, so concurrent classes register the closer once. extensionContext .getRoot() .getStore(ExtensionContext.Namespace.GLOBAL) - .getOrComputeIfAbsent(CloseableContainer.class); + .computeIfAbsent(CloseableContainer.class); } - private static class CloseableContainer implements ExtensionContext.Store.CloseableResource { + private static class CloseableContainer implements AutoCloseable { private static final Logger LOGGER = LoggerFactory.getLogger(CloseableContainer.class); private static final ContainerSuite CONTAINER_SUITE = ContainerSuite.getInstance(); diff --git a/plugins/k8s-job-executor/build.gradle.kts b/plugins/k8s-job-executor/build.gradle.kts index ec066212184..e164bbfd180 100644 --- a/plugins/k8s-job-executor/build.gradle.kts +++ b/plugins/k8s-job-executor/build.gradle.kts @@ -70,6 +70,7 @@ dependencies { shadedJarTestRuntimeOnly(project(":server")) shadedJarTestRuntimeOnly(files(tasks.named("shadowJar").flatMap { it.archiveFile })) shadedJarTestRuntimeOnly(libs.junit.jupiter.engine) + shadedJarTestRuntimeOnly(libs.junit.platform.launcher) } tasks.withType(ShadowJar::class.java) { diff --git a/trino-connector/trino-connector-473-478/build.gradle.kts b/trino-connector/trino-connector-473-478/build.gradle.kts index 735307d963e..32aad20c1f0 100644 --- a/trino-connector/trino-connector-473-478/build.gradle.kts +++ b/trino-connector/trino-connector-473-478/build.gradle.kts @@ -75,7 +75,7 @@ dependencies { exclude("org.junit.jupiter") exclude("org.junit.platform") } - testImplementation(enforcedPlatform("org.junit:junit-bom:5.11.4")) + testImplementation(enforcedPlatform(libs.junit.bom)) testImplementation("org.junit.jupiter:junit-jupiter-api") testRuntimeOnly("org.junit.jupiter:junit-jupiter-engine") } From 58a009f96784fb15edcfa6a0dee18020f378afe0 Mon Sep 17 00:00:00 2001 From: Nevin Zheng Date: Wed, 21 Oct 2026 09:13:44 -0700 Subject: [PATCH 03/14] [#13958] improvement(build): Upgrade Testcontainers to 2.0.5 --- .../integration/test/CatalogClickHouseIT.java | 13 +++++----- .../operations/TestClickHouseEngineIT.java | 15 ++++++------ .../TestClickHouseTableOperations.java | 15 ++++++------ .../integration/test/CatalogOceanBaseIT.java | 9 +++---- .../TestOceanBaseTableOperations.java | 19 ++++++++------- .../jdbc/converter/SqliteTypeConverter.java | 4 ++-- .../operation/TestJdbcDatabaseOperations.java | 4 ++-- .../catalog/mysql/TestMysqlDataSource.java | 6 ++--- .../integration/test/CatalogMysqlIT.java | 11 +++++---- .../TestMysqlDatabaseOperations.java | 2 +- .../operation/TestMysqlTableOperations.java | 24 ++++++++++--------- .../postgresql/TestPostgreSqlDataSource.java | 4 ++-- .../integration/test/CatalogPostgreSqlIT.java | 5 ++-- .../postgresql/operation/TestPostgreSql.java | 4 ++-- .../TestPostgreSqlTableOperations.java | 7 +++--- flink-connector/flink-common/build.gradle.kts | 1 - flink-connector/v1.19/flink/build.gradle.kts | 1 - flink-connector/v1.20/flink/build.gradle.kts | 1 - flink-connector/v2.1/flink/build.gradle.kts | 1 - flink-connector/v2.2/flink/build.gradle.kts | 1 - gradle/libs.versions.toml | 17 +++++++------ .../test/container/DorisContainer.java | 3 +-- .../test/container/MySQLContainer.java | 9 +++---- .../test/container/PostgreSQLContainer.java | 2 +- .../integration/test/util/BaseIT.java | 2 +- maintenance/jobs/build.gradle.kts | 1 - .../TestIcebergUpdateStatsJobWithSpark.java | 14 +++++------ maintenance/optimizer/build.gradle.kts | 1 - ...enericJdbcMetricsRepositoryBehaviorIT.java | 6 ++--- ...tgreSqlGenericJdbcMetricsRepositoryIT.java | 6 ++--- 30 files changed, 105 insertions(+), 103 deletions(-) diff --git a/catalogs-contrib/catalog-jdbc-clickhouse/src/test/java/org/apache/gravitino/catalog/clickhouse/integration/test/CatalogClickHouseIT.java b/catalogs-contrib/catalog-jdbc-clickhouse/src/test/java/org/apache/gravitino/catalog/clickhouse/integration/test/CatalogClickHouseIT.java index 8d7444acb8d..801b0759674 100644 --- a/catalogs-contrib/catalog-jdbc-clickhouse/src/test/java/org/apache/gravitino/catalog/clickhouse/integration/test/CatalogClickHouseIT.java +++ b/catalogs-contrib/catalog-jdbc-clickhouse/src/test/java/org/apache/gravitino/catalog/clickhouse/integration/test/CatalogClickHouseIT.java @@ -51,6 +51,7 @@ import java.util.Set; import java.util.stream.Collectors; import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.Strings; import org.apache.gravitino.Catalog; import org.apache.gravitino.CatalogChange; import org.apache.gravitino.NameIdentifier; @@ -562,8 +563,7 @@ void testLoadExpressionIndexDoesNotFabricateColumnIndex() { String.format("SHOW CREATE TABLE `%s`.`%s`", schemaName, sourceTableName)); String normalizedSourceCreateSql = sourceCreateSql.replace("`", "").replaceAll("\\s+", ""); Assertions.assertTrue( - StringUtils.containsIgnoreCase( - normalizedSourceCreateSql, "INDEXidx_lowerlower(name)TYPEminmax"), + Strings.CI.contains(normalizedSourceCreateSql, "INDEXidx_lowerlower(name)TYPEminmax"), "Source table should retain its expression index: " + sourceCreateSql); TableCatalog tableCatalog = catalog.asTableCatalog(); @@ -595,10 +595,10 @@ void testLoadExpressionIndexDoesNotFabricateColumnIndex() { String normalizedRecreatedCreateSql = recreatedCreateSql.replace("`", "").replaceAll("\\s+", ""); Assertions.assertTrue( - StringUtils.containsIgnoreCase(normalizedRecreatedCreateSql, "INDEXidx_namenametypeMINMAX"), + Strings.CI.contains(normalizedRecreatedCreateSql, "INDEXidx_namenametypeMINMAX"), "Recreated table should retain the simple index: " + recreatedCreateSql); Assertions.assertFalse( - StringUtils.containsIgnoreCase(normalizedRecreatedCreateSql, "idx_lower"), + Strings.CI.contains(normalizedRecreatedCreateSql, "idx_lower"), "Recreated table must not contain a fabricated replacement index: " + recreatedCreateSql); } @@ -700,8 +700,7 @@ void testLoadAndCreateWithStartOfDatePartitionTransforms() { String.format("SHOW CREATE TABLE `%s`.`%s`", schemaName, createdTableName)); String normalizedShowCreateSql = StringUtils.deleteWhitespace(showCreateSql).replace("`", ""); Assertions.assertTrue( - StringUtils.containsIgnoreCase( - normalizedShowCreateSql, "PARTITIONBY" + expectedPartitionKey), + Strings.CI.contains(normalizedShowCreateSql, "PARTITIONBY" + expectedPartitionKey), "CREATE should emit the ClickHouse partition function: " + showCreateSql); } } @@ -1096,7 +1095,7 @@ void testLoadLegacyAnnoyIndexMetadata() throws Exception { Assertions.assertEquals("annoy", nativeType); Assertions.assertTrue( - StringUtils.startsWithIgnoreCase(nativeTypeFull, "annoy"), + Strings.CI.startsWith(nativeTypeFull, "annoy"), "Expected ClickHouse to preserve the Annoy type expression, but got: " + nativeTypeFull); Assertions.assertTrue(nativeGranularity > 0); diff --git a/catalogs-contrib/catalog-jdbc-clickhouse/src/test/java/org/apache/gravitino/catalog/clickhouse/operations/TestClickHouseEngineIT.java b/catalogs-contrib/catalog-jdbc-clickhouse/src/test/java/org/apache/gravitino/catalog/clickhouse/operations/TestClickHouseEngineIT.java index 18b6ea9335d..e1af276862a 100644 --- a/catalogs-contrib/catalog-jdbc-clickhouse/src/test/java/org/apache/gravitino/catalog/clickhouse/operations/TestClickHouseEngineIT.java +++ b/catalogs-contrib/catalog-jdbc-clickhouse/src/test/java/org/apache/gravitino/catalog/clickhouse/operations/TestClickHouseEngineIT.java @@ -122,7 +122,7 @@ public void setup() throws Exception { new ClickHouseColumnDefaultValueConverter(), Collections.emptyMap()); - schemaName = "engine_it_" + RandomStringUtils.randomAlphanumeric(6).toLowerCase(); + schemaName = "engine_it_" + RandomStringUtils.insecure().nextAlphanumeric(6).toLowerCase(); databaseOperations.create(schemaName, null, Collections.emptyMap()); } @@ -139,7 +139,7 @@ public void cleanup() { @Test void testTinyLogEngineCreateLoadAndPersistAfterRestart() throws Exception { - String tableName = "tinylog_" + RandomStringUtils.randomAlphanumeric(6).toLowerCase(); + String tableName = "tinylog_" + RandomStringUtils.insecure().nextAlphanumeric(6).toLowerCase(); createSimpleTable(schemaName, tableName, ENGINE.TINYLOG); JdbcTable before = tableOperations.load(schemaName, tableName); @@ -160,7 +160,8 @@ void testTinyLogEngineCreateLoadAndPersistAfterRestart() throws Exception { @Test void testStripeLogEngineCreateLoadAndPersistAfterRestart() throws Exception { - String tableName = "stripelog_" + RandomStringUtils.randomAlphanumeric(6).toLowerCase(); + String tableName = + "stripelog_" + RandomStringUtils.insecure().nextAlphanumeric(6).toLowerCase(); createSimpleTable(schemaName, tableName, ENGINE.STRIPELOG); JdbcTable before = tableOperations.load(schemaName, tableName); @@ -181,7 +182,7 @@ void testStripeLogEngineCreateLoadAndPersistAfterRestart() throws Exception { @Test void testLogEngineCreateLoadAndPersistAfterRestart() throws Exception { - String tableName = "log_" + RandomStringUtils.randomAlphanumeric(6).toLowerCase(); + String tableName = "log_" + RandomStringUtils.insecure().nextAlphanumeric(6).toLowerCase(); createSimpleTable(schemaName, tableName, ENGINE.LOG); JdbcTable before = tableOperations.load(schemaName, tableName); @@ -204,7 +205,7 @@ void testLogEngineCreateLoadAndPersistAfterRestart() throws Exception { @Test void testMemoryEngineIsVolatileAfterRestart() throws Exception { - String tableName = "memory_" + RandomStringUtils.randomAlphanumeric(6).toLowerCase(); + String tableName = "memory_" + RandomStringUtils.insecure().nextAlphanumeric(6).toLowerCase(); createSimpleTable(schemaName, tableName, ENGINE.MEMORY); JdbcTable before = tableOperations.load(schemaName, tableName); @@ -233,7 +234,7 @@ void testMemoryEngineIsVolatileAfterRestart() throws Exception { @Test void testNullEngineCreateLoadAndPersistAfterRestart() throws Exception { - String tableName = "null_" + RandomStringUtils.randomAlphanumeric(6).toLowerCase(); + String tableName = "null_" + RandomStringUtils.insecure().nextAlphanumeric(6).toLowerCase(); createSimpleTable(schemaName, tableName, ENGINE.NULL); JdbcTable before = tableOperations.load(schemaName, tableName); @@ -251,7 +252,7 @@ void testNullEngineCreateLoadAndPersistAfterRestart() throws Exception { @Test void testSetEngineCreateLoadAndPersistAfterRestart() throws Exception { - String tableName = "set_" + RandomStringUtils.randomAlphanumeric(6).toLowerCase(); + String tableName = "set_" + RandomStringUtils.insecure().nextAlphanumeric(6).toLowerCase(); createSimpleTable(schemaName, tableName, ENGINE.SET); JdbcTable before = tableOperations.load(schemaName, tableName); diff --git a/catalogs-contrib/catalog-jdbc-clickhouse/src/test/java/org/apache/gravitino/catalog/clickhouse/operations/TestClickHouseTableOperations.java b/catalogs-contrib/catalog-jdbc-clickhouse/src/test/java/org/apache/gravitino/catalog/clickhouse/operations/TestClickHouseTableOperations.java index 031ca47d699..5f41ac1c00a 100644 --- a/catalogs-contrib/catalog-jdbc-clickhouse/src/test/java/org/apache/gravitino/catalog/clickhouse/operations/TestClickHouseTableOperations.java +++ b/catalogs-contrib/catalog-jdbc-clickhouse/src/test/java/org/apache/gravitino/catalog/clickhouse/operations/TestClickHouseTableOperations.java @@ -35,6 +35,7 @@ import java.util.Map; import org.apache.commons.lang3.RandomStringUtils; import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.Strings; import org.apache.gravitino.catalog.clickhouse.ClickHouseConstants.TableConstants; import org.apache.gravitino.catalog.clickhouse.ClickHouseTablePropertiesMetadata; import org.apache.gravitino.catalog.clickhouse.ClickHouseUtils; @@ -71,7 +72,7 @@ public class TestClickHouseTableOperations extends TestClickHouse { @Test public void testCreateAndAlterTable() { - String tableName = RandomStringUtils.randomAlphabetic(16) + "_op_table"; + String tableName = RandomStringUtils.insecure().nextAlphabetic(16) + "_op_table"; String tableComment = "test_comment"; List columns = new ArrayList<>(); columns.add( @@ -172,7 +173,7 @@ public void testCreateAndAlterTable() { UnsupportedOperationException.class, () -> TABLE_OPERATIONS.alterTable(TEST_DB_NAME.toString(), newName, setProperty)); Assertions.assertTrue( - StringUtils.contains( + Strings.CS.contains( gravitinoRuntimeException.getMessage(), "Only ClickHouse table properties with the 'settings.' prefix can be altered")); @@ -215,7 +216,7 @@ public void testCreateAndAlterTable() { @Test public void testAlterTable() { - String tableName = RandomStringUtils.randomAlphabetic(16) + "_al_table"; + String tableName = RandomStringUtils.insecure().nextAlphabetic(16) + "_al_table"; String tableComment = "test_comment"; List columns = new ArrayList<>(); JdbcColumn col_1 = @@ -440,7 +441,7 @@ public void testAlterTableUpdateColumnDefaultValue() { @Test public void testCreateAndLoadTable() { - String tableName = RandomStringUtils.randomAlphabetic(16) + "_cl_table"; + String tableName = RandomStringUtils.insecure().nextAlphabetic(16) + "_cl_table"; String tableComment = "test_comment"; List columns = new ArrayList<>(); columns.add( @@ -497,7 +498,7 @@ public void testCreateAndLoadTable() { @Test public void testLoadTableWithMaterializedAndAliasColumns() throws Exception { - String tableName = RandomStringUtils.randomAlphabetic(16) + "_default_kind"; + String tableName = RandomStringUtils.insecure().nextAlphabetic(16) + "_default_kind"; // Create table with MATERIALIZED and ALIAS columns via raw SQL. // Using DriverManager.getConnection directly because Gravitino's create() API @@ -559,7 +560,7 @@ private static Column findColumn(Column[] columns, String name) { @Test public void testTypeConversionAgainstCluster() { - String tableName = RandomStringUtils.randomAlphabetic(16) + "_type_conv"; + String tableName = RandomStringUtils.insecure().nextAlphabetic(16) + "_type_conv"; List columns = new ArrayList<>(); columns.add( @@ -723,7 +724,7 @@ public void testTypeConversionAgainstCluster() { @Test public void testClearCommentsWithNull() { - String tableName = RandomStringUtils.randomAlphabetic(16) + "_clear_comment"; + String tableName = RandomStringUtils.insecure().nextAlphabetic(16) + "_clear_comment"; JdbcColumn column = JdbcColumn.builder() .withName("col_1") diff --git a/catalogs-contrib/catalog-jdbc-oceanbase/src/test/java/org/apache/gravitino/catalog/oceanbase/integration/test/CatalogOceanBaseIT.java b/catalogs-contrib/catalog-jdbc-oceanbase/src/test/java/org/apache/gravitino/catalog/oceanbase/integration/test/CatalogOceanBaseIT.java index ac997a4d60b..f2e5cb46bd4 100644 --- a/catalogs-contrib/catalog-jdbc-oceanbase/src/test/java/org/apache/gravitino/catalog/oceanbase/integration/test/CatalogOceanBaseIT.java +++ b/catalogs-contrib/catalog-jdbc-oceanbase/src/test/java/org/apache/gravitino/catalog/oceanbase/integration/test/CatalogOceanBaseIT.java @@ -35,6 +35,7 @@ import java.util.stream.Collectors; import org.apache.commons.lang3.ArrayUtils; import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.Strings; import org.apache.gravitino.Catalog; import org.apache.gravitino.NameIdentifier; import org.apache.gravitino.Namespace; @@ -938,7 +939,7 @@ void testCreateTableIndex() { indexes2); }); Assertions.assertTrue( - StringUtils.contains( + Strings.CS.contains( illegalArgumentException.getMessage(), "Index does not support complex fields in this Catalog")); @@ -958,7 +959,7 @@ void testCreateTableIndex() { indexes3); }); Assertions.assertTrue( - StringUtils.contains( + Strings.CS.contains( illegalArgumentException.getMessage(), "Index does not support complex fields in this Catalog")); @@ -1118,7 +1119,7 @@ public void testAutoIncrement() { new SortOrder[0], Indexes.EMPTY_INDEXES)); Assertions.assertTrue( - StringUtils.contains( + Strings.CS.contains( runtimeException.getMessage(), "Incorrect table definition; there can be only one auto column and it must be defined as a key")); @@ -1142,7 +1143,7 @@ public void testAutoIncrement() { sortOrder, index2)); Assertions.assertTrue( - StringUtils.contains( + Strings.CS.contains( runtimeException.getMessage(), "Only one column can be auto-incremented. There are multiple auto-increment columns in your table: [col_1,col_6]")); } diff --git a/catalogs-contrib/catalog-jdbc-oceanbase/src/test/java/org/apache/gravitino/catalog/oceanbase/operation/TestOceanBaseTableOperations.java b/catalogs-contrib/catalog-jdbc-oceanbase/src/test/java/org/apache/gravitino/catalog/oceanbase/operation/TestOceanBaseTableOperations.java index 3a07734d6c7..f88d89b1463 100644 --- a/catalogs-contrib/catalog-jdbc-oceanbase/src/test/java/org/apache/gravitino/catalog/oceanbase/operation/TestOceanBaseTableOperations.java +++ b/catalogs-contrib/catalog-jdbc-oceanbase/src/test/java/org/apache/gravitino/catalog/oceanbase/operation/TestOceanBaseTableOperations.java @@ -28,6 +28,7 @@ import java.util.stream.Collectors; import org.apache.commons.lang3.RandomStringUtils; import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.Strings; import org.apache.gravitino.catalog.jdbc.JdbcColumn; import org.apache.gravitino.catalog.jdbc.JdbcTable; import org.apache.gravitino.rel.Column; @@ -148,7 +149,7 @@ void testClearCommentsWithNull() { @Test public void testOperationTable() { - String tableName = RandomStringUtils.randomAlphabetic(16).toLowerCase() + "_op_table"; + String tableName = RandomStringUtils.insecure().nextAlphabetic(16).toLowerCase() + "_op_table"; String tableComment = "test_comment"; List columns = new ArrayList<>(); columns.add( @@ -269,7 +270,8 @@ public void testOperationTable() { @Test public void testAlterTableUpdateColumnTypeKeepsDefaultValue() { - String tableName = RandomStringUtils.randomAlphabetic(16).toLowerCase() + "_type_default_table"; + String tableName = + RandomStringUtils.insecure().nextAlphabetic(16).toLowerCase() + "_type_default_table"; JdbcColumn col_1 = JdbcColumn.builder().withName("col_1").withType(INT).withNullable(false).build(); JdbcColumn col_2 = @@ -302,7 +304,7 @@ public void testAlterTableUpdateColumnTypeKeepsDefaultValue() { @Test public void testAlterTable() { - String tableName = RandomStringUtils.randomAlphabetic(16).toLowerCase() + "_al_table"; + String tableName = RandomStringUtils.insecure().nextAlphabetic(16).toLowerCase() + "_al_table"; String tableComment = "test_comment"; List columns = new ArrayList<>(); JdbcColumn col_1 = @@ -686,7 +688,7 @@ public void testUpdateColumnDefaultValueKeepsAutoIncrement() { @Test public void testCreateAndLoadTable() { - String tableName = RandomStringUtils.randomAlphabetic(16).toLowerCase() + "_cl_table"; + String tableName = RandomStringUtils.insecure().nextAlphabetic(16).toLowerCase() + "_cl_table"; String tableComment = "test_comment"; List columns = new ArrayList<>(); columns.add( @@ -1050,7 +1052,7 @@ public void testAutoIncrement() { Distributions.NONE, Indexes.EMPTY_INDEXES)); Assertions.assertTrue( - StringUtils.contains( + Strings.CS.contains( exception.getMessage(), "Incorrect table definition; there can be only one auto column and it must be defined as a key")); @@ -1086,7 +1088,7 @@ public void testAutoIncrement() { Distributions.NONE, primaryIndex)); Assertions.assertTrue( - StringUtils.contains( + Strings.CS.contains( exception.getMessage(), "Only one column can be auto-incremented. There are multiple auto-increment columns in your table: [col_1,col_4]")); } @@ -1155,7 +1157,7 @@ public void testOperationIndexDefinition() { sql = OceanBaseTableOperations.addIndexDefinition(successIndex); Assertions.assertEquals("ADD PRIMARY KEY (`col_1`, `col_2`)", sql); - String tableName = RandomStringUtils.randomAlphabetic(16).toLowerCase() + "_oi_table"; + String tableName = RandomStringUtils.insecure().nextAlphabetic(16).toLowerCase() + "_oi_table"; String tableComment = "test_comment"; List columns = new ArrayList<>(); columns.add( @@ -1301,7 +1303,8 @@ public boolean isMySQLDriverVersionSupported(String driverVersion) { @Test public void testAlterTableSkipsEmptySql() { - String tableName = RandomStringUtils.randomAlphabetic(16).toLowerCase() + "_skip_empty_sql"; + String tableName = + RandomStringUtils.insecure().nextAlphabetic(16).toLowerCase() + "_skip_empty_sql"; List columns = new ArrayList<>(); columns.add( JdbcColumn.builder() diff --git a/catalogs/catalog-jdbc-common/src/test/java/org/apache/gravitino/catalog/jdbc/converter/SqliteTypeConverter.java b/catalogs/catalog-jdbc-common/src/test/java/org/apache/gravitino/catalog/jdbc/converter/SqliteTypeConverter.java index 3c165f478d8..db653bc1c2d 100644 --- a/catalogs/catalog-jdbc-common/src/test/java/org/apache/gravitino/catalog/jdbc/converter/SqliteTypeConverter.java +++ b/catalogs/catalog-jdbc-common/src/test/java/org/apache/gravitino/catalog/jdbc/converter/SqliteTypeConverter.java @@ -21,7 +21,7 @@ import java.util.Collection; import java.util.HashMap; import java.util.Map; -import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.Strings; import org.apache.gravitino.rel.types.Type; import org.apache.gravitino.rel.types.Types; @@ -39,7 +39,7 @@ public class SqliteTypeConverter extends JdbcTypeConverter { @Override public Type toGravitino(JdbcTypeBean type) { return GRAVITINO_TO_SQLITE_MAPPING.entrySet().stream() - .filter(entry -> StringUtils.equalsIgnoreCase(type.getTypeName(), entry.getValue())) + .filter(entry -> Strings.CI.equals(type.getTypeName(), entry.getValue())) .map(Map.Entry::getKey) .findFirst() .orElse(Types.ExternalType.of(type.getTypeName())); diff --git a/catalogs/catalog-jdbc-common/src/test/java/org/apache/gravitino/catalog/jdbc/operation/TestJdbcDatabaseOperations.java b/catalogs/catalog-jdbc-common/src/test/java/org/apache/gravitino/catalog/jdbc/operation/TestJdbcDatabaseOperations.java index e5088077f03..5cdb10ce0b3 100644 --- a/catalogs/catalog-jdbc-common/src/test/java/org/apache/gravitino/catalog/jdbc/operation/TestJdbcDatabaseOperations.java +++ b/catalogs/catalog-jdbc-common/src/test/java/org/apache/gravitino/catalog/jdbc/operation/TestJdbcDatabaseOperations.java @@ -29,7 +29,7 @@ import java.util.Set; import javax.sql.DataSource; import org.apache.commons.io.FileUtils; -import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.Strings; import org.apache.gravitino.catalog.jdbc.JdbcSchema; import org.apache.gravitino.catalog.jdbc.config.JdbcConfig; import org.apache.gravitino.catalog.jdbc.converter.JdbcExceptionConverter; @@ -142,7 +142,7 @@ public void testOperationDatabase() { SchemaAlreadyExistsException.class, () -> JDBC_DATABASE_OPERATIONS.create(database1, null, null)); Assertions.assertTrue( - StringUtils.contains(illegalArgumentException.getMessage(), "already exists")); + Strings.CS.contains(illegalArgumentException.getMessage(), "already exists")); Assertions.assertDoesNotThrow(() -> JDBC_DATABASE_OPERATIONS.create(database2, null, null)); // list database diff --git a/catalogs/catalog-jdbc-mysql/src/test/java/org/apache/gravitino/catalog/mysql/TestMysqlDataSource.java b/catalogs/catalog-jdbc-mysql/src/test/java/org/apache/gravitino/catalog/mysql/TestMysqlDataSource.java index 3a06099404a..f7f708e8413 100644 --- a/catalogs/catalog-jdbc-mysql/src/test/java/org/apache/gravitino/catalog/mysql/TestMysqlDataSource.java +++ b/catalogs/catalog-jdbc-mysql/src/test/java/org/apache/gravitino/catalog/mysql/TestMysqlDataSource.java @@ -35,14 +35,14 @@ import org.junit.jupiter.api.Tag; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.ValueSource; -import org.testcontainers.containers.MySQLContainer; +import org.testcontainers.mysql.MySQLContainer; /** Exercises catalog switching and connection validation against a real MySQL server. */ @Tag("gravitino-docker-test") public class TestMysqlDataSource { private static final String TARGET_DATABASE = "pool_target"; - private static final MySQLContainer MYSQL = - new MySQLContainer<>("mysql:8.0").withDatabaseName("pool_bootstrap"); + private static final MySQLContainer MYSQL = + new MySQLContainer("mysql:8.0").withDatabaseName("pool_bootstrap"); /** Starts MySQL and creates a target database distinct from the default database. */ @BeforeAll diff --git a/catalogs/catalog-jdbc-mysql/src/test/java/org/apache/gravitino/catalog/mysql/integration/test/CatalogMysqlIT.java b/catalogs/catalog-jdbc-mysql/src/test/java/org/apache/gravitino/catalog/mysql/integration/test/CatalogMysqlIT.java index 8b920576e1f..3681a2a41f1 100644 --- a/catalogs/catalog-jdbc-mysql/src/test/java/org/apache/gravitino/catalog/mysql/integration/test/CatalogMysqlIT.java +++ b/catalogs/catalog-jdbc-mysql/src/test/java/org/apache/gravitino/catalog/mysql/integration/test/CatalogMysqlIT.java @@ -35,6 +35,7 @@ import java.util.Set; import java.util.stream.Collectors; import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.Strings; import org.apache.gravitino.Catalog; import org.apache.gravitino.CatalogChange; import org.apache.gravitino.NameIdentifier; @@ -1266,7 +1267,7 @@ void testCreateTableIndex() { indexes2); }); Assertions.assertTrue( - StringUtils.contains( + Strings.CS.contains( illegalArgumentException.getMessage(), "Index does not support complex fields in this Catalog")); @@ -1286,7 +1287,7 @@ void testCreateTableIndex() { indexes3); }); Assertions.assertTrue( - StringUtils.contains( + Strings.CS.contains( illegalArgumentException.getMessage(), "Index does not support complex fields in this Catalog")); @@ -1552,7 +1553,7 @@ public void testAutoIncrement() { new SortOrder[0], Indexes.EMPTY_INDEXES)); Assertions.assertTrue( - StringUtils.contains( + Strings.CS.contains( runtimeException.getMessage(), "Incorrect table definition; there can be only one auto column and it must be defined as a key")); @@ -1576,7 +1577,7 @@ public void testAutoIncrement() { sortOrder, index2)); Assertions.assertTrue( - StringUtils.contains( + Strings.CS.contains( runtimeException.getMessage(), "Only one column can be auto-incremented. There are multiple auto-increment columns in your table: [col_1,col_6]")); } @@ -2261,7 +2262,7 @@ void testAddColumnAutoIncrement() { false, true))); Assertions.assertTrue( - StringUtils.contains( + Strings.CS.contains( runtimeException.getMessage(), "Incorrect table definition; there can be only one auto column and it must be defined as a key")); diff --git a/catalogs/catalog-jdbc-mysql/src/test/java/org/apache/gravitino/catalog/mysql/operation/TestMysqlDatabaseOperations.java b/catalogs/catalog-jdbc-mysql/src/test/java/org/apache/gravitino/catalog/mysql/operation/TestMysqlDatabaseOperations.java index 1ad704016da..89b98aa80f2 100644 --- a/catalogs/catalog-jdbc-mysql/src/test/java/org/apache/gravitino/catalog/mysql/operation/TestMysqlDatabaseOperations.java +++ b/catalogs/catalog-jdbc-mysql/src/test/java/org/apache/gravitino/catalog/mysql/operation/TestMysqlDatabaseOperations.java @@ -62,7 +62,7 @@ void testDropTableWithSpecificName() { databaseName = RandomNameUtils.genRandomName("ct_db") + "--------end"; DATABASE_OPERATIONS.create(databaseName, null, properties); - String tableName = RandomStringUtils.randomAlphabetic(16) + "_op_table"; + String tableName = RandomStringUtils.insecure().nextAlphabetic(16) + "_op_table"; String tableComment = "test_comment"; List columns = new ArrayList<>(); columns.add( diff --git a/catalogs/catalog-jdbc-mysql/src/test/java/org/apache/gravitino/catalog/mysql/operation/TestMysqlTableOperations.java b/catalogs/catalog-jdbc-mysql/src/test/java/org/apache/gravitino/catalog/mysql/operation/TestMysqlTableOperations.java index d76eedc8c4f..14f257b57de 100644 --- a/catalogs/catalog-jdbc-mysql/src/test/java/org/apache/gravitino/catalog/mysql/operation/TestMysqlTableOperations.java +++ b/catalogs/catalog-jdbc-mysql/src/test/java/org/apache/gravitino/catalog/mysql/operation/TestMysqlTableOperations.java @@ -34,6 +34,7 @@ import javax.sql.DataSource; import org.apache.commons.lang3.RandomStringUtils; import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.Strings; import org.apache.gravitino.catalog.jdbc.JdbcColumn; import org.apache.gravitino.catalog.jdbc.JdbcTable; import org.apache.gravitino.catalog.jdbc.utils.DataSourceUtils; @@ -63,7 +64,7 @@ public class TestMysqlTableOperations extends TestMysql { @Test public void testOperationTable() { - String tableName = RandomStringUtils.randomAlphabetic(16) + "_op_table"; + String tableName = RandomStringUtils.insecure().nextAlphabetic(16) + "_op_table"; String tableComment = "test_comment"; List columns = new ArrayList<>(); columns.add( @@ -201,7 +202,7 @@ public void testOperationTable() { @Test public void testAlterTableUpdateColumnTypeKeepsDefaultValue() { - String tableName = RandomStringUtils.randomAlphabetic(16) + "_type_default_table"; + String tableName = RandomStringUtils.insecure().nextAlphabetic(16) + "_type_default_table"; JdbcColumn col_1 = JdbcColumn.builder().withName("col_1").withType(INT).withNullable(false).build(); JdbcColumn col_2 = @@ -234,7 +235,7 @@ public void testAlterTableUpdateColumnTypeKeepsDefaultValue() { @Test public void testUpdateColumnPositionWithSpecialReferenceColumn() { - String tableName = RandomStringUtils.randomAlphabetic(16) + "_position_table"; + String tableName = RandomStringUtils.insecure().nextAlphabetic(16) + "_position_table"; JdbcColumn[] columns = new JdbcColumn[] { JdbcColumn.builder().withName("id").withType(INT).withNullable(false).build(), @@ -275,7 +276,7 @@ public void testUpdateColumnPositionWithSpecialReferenceColumn() { @Test public void testAlterTable() { - String tableName = RandomStringUtils.randomAlphabetic(16) + "_al_table"; + String tableName = RandomStringUtils.insecure().nextAlphabetic(16) + "_al_table"; String tableComment = "test_comment"; List columns = new ArrayList<>(); JdbcColumn col_1 = @@ -660,7 +661,7 @@ public void testUpdateColumnDefaultValueKeepsAutoIncrement() { @Test public void testCreateAndLoadTable() { - String tableName = RandomStringUtils.randomAlphabetic(16) + "_cl_table"; + String tableName = RandomStringUtils.insecure().nextAlphabetic(16) + "_cl_table"; String tableComment = "test_comment"; List columns = new ArrayList<>(); columns.add( @@ -721,7 +722,7 @@ public void testCreateAndLoadTable() { @Test public void testClearCommentsWithNull() { - String tableName = RandomStringUtils.randomAlphabetic(16) + "_clear_comment"; + String tableName = RandomStringUtils.insecure().nextAlphabetic(16) + "_clear_comment"; JdbcColumn column = JdbcColumn.builder().withName("col_1").withType(INT).withComment("col comment").build(); try { @@ -1098,7 +1099,7 @@ public void testAutoIncrement() { Distributions.NONE, Indexes.EMPTY_INDEXES)); Assertions.assertTrue( - StringUtils.contains( + Strings.CS.contains( exception.getMessage(), "Incorrect table definition; there can be only one auto column and it must be defined as a key")); @@ -1134,7 +1135,7 @@ public void testAutoIncrement() { Distributions.NONE, primaryIndex)); Assertions.assertTrue( - StringUtils.contains( + Strings.CS.contains( exception.getMessage(), "Only one column can be auto-incremented. There are multiple auto-increment columns in your table: [col_1,col_4]")); } @@ -1203,7 +1204,7 @@ public void testOperationIndexDefinition() { sql = MysqlTableOperations.addIndexDefinition(successIndex); Assertions.assertEquals("ADD PRIMARY KEY (`col_1`, `col_2`)", sql); - String tableName = RandomStringUtils.randomAlphabetic(16) + "_op_table"; + String tableName = RandomStringUtils.insecure().nextAlphabetic(16) + "_op_table"; String tableComment = "test_comment"; List columns = new ArrayList<>(); columns.add( @@ -1294,7 +1295,8 @@ public void testLoadDatetimePrecisionWithoutDriverSupport() throws Exception { // MySQL Connector/J before 8.0.16 reports a COLUMN_SIZE the fractional seconds precision // cannot be derived from, which calculateDatetimePrecision signals by returning null. The // precision must then be recovered from information_schema instead of being left unset. - String tableName = RandomStringUtils.randomAlphabetic(16) + "_datetime_precision_table"; + String tableName = + RandomStringUtils.insecure().nextAlphabetic(16) + "_datetime_precision_table"; DataSource dataSource = DataSourceUtils.createDataSource(getMySQLCatalogProperties()); try { try (Connection connection = dataSource.getConnection()) { @@ -1351,7 +1353,7 @@ public void testLoadColumnsWithLossyTypeDeclarations() throws Exception { // These types either drop their parameters (VARBINARY/ENUM/SET) or collapse distinct widths // into the same Gravitino type (BIT/BINARY) when only JDBC's TYPE_NAME/COLUMN_SIZE metadata // is used. - String tableName = RandomStringUtils.randomAlphabetic(16) + "_lossy_type_table"; + String tableName = RandomStringUtils.insecure().nextAlphabetic(16) + "_lossy_type_table"; DataSource dataSource = DataSourceUtils.createDataSource(getMySQLCatalogProperties()); try { try (Connection connection = dataSource.getConnection()) { diff --git a/catalogs/catalog-jdbc-postgresql/src/test/java/org/apache/gravitino/catalog/postgresql/TestPostgreSqlDataSource.java b/catalogs/catalog-jdbc-postgresql/src/test/java/org/apache/gravitino/catalog/postgresql/TestPostgreSqlDataSource.java index cd3c896d7ea..4b6d5a5b6ae 100644 --- a/catalogs/catalog-jdbc-postgresql/src/test/java/org/apache/gravitino/catalog/postgresql/TestPostgreSqlDataSource.java +++ b/catalogs/catalog-jdbc-postgresql/src/test/java/org/apache/gravitino/catalog/postgresql/TestPostgreSqlDataSource.java @@ -33,12 +33,12 @@ import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; -import org.testcontainers.containers.PostgreSQLContainer; +import org.testcontainers.postgresql.PostgreSQLContainer; /** Exercises default validation against a real PostgreSQL server. */ @Tag("gravitino-docker-test") public class TestPostgreSqlDataSource { - private static final PostgreSQLContainer POSTGRES = new PostgreSQLContainer<>("postgres:13"); + private static final PostgreSQLContainer POSTGRES = new PostgreSQLContainer("postgres:13"); /** Starts PostgreSQL and creates a schema for connection state changes. */ @BeforeAll diff --git a/catalogs/catalog-jdbc-postgresql/src/test/java/org/apache/gravitino/catalog/postgresql/integration/test/CatalogPostgreSqlIT.java b/catalogs/catalog-jdbc-postgresql/src/test/java/org/apache/gravitino/catalog/postgresql/integration/test/CatalogPostgreSqlIT.java index e29616f4ece..d015de07591 100644 --- a/catalogs/catalog-jdbc-postgresql/src/test/java/org/apache/gravitino/catalog/postgresql/integration/test/CatalogPostgreSqlIT.java +++ b/catalogs/catalog-jdbc-postgresql/src/test/java/org/apache/gravitino/catalog/postgresql/integration/test/CatalogPostgreSqlIT.java @@ -39,6 +39,7 @@ import java.util.stream.Collectors; import org.apache.commons.lang3.ArrayUtils; import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.Strings; import org.apache.gravitino.Catalog; import org.apache.gravitino.NameIdentifier; import org.apache.gravitino.Namespace; @@ -828,7 +829,7 @@ void testCreateIndexTable() { primaryIndex); }); Assertions.assertTrue( - StringUtils.contains( + Strings.CS.contains( illegalArgumentException.getMessage(), "Index does not support complex fields in PostgreSQL")); @@ -849,7 +850,7 @@ void testCreateIndexTable() { primaryIndex2); }); Assertions.assertTrue( - StringUtils.contains( + Strings.CS.contains( illegalArgumentException.getMessage(), "Index does not support complex fields in PostgreSQL")); diff --git a/catalogs/catalog-jdbc-postgresql/src/test/java/org/apache/gravitino/catalog/postgresql/operation/TestPostgreSql.java b/catalogs/catalog-jdbc-postgresql/src/test/java/org/apache/gravitino/catalog/postgresql/operation/TestPostgreSql.java index eda7851b3ef..47098f37989 100644 --- a/catalogs/catalog-jdbc-postgresql/src/test/java/org/apache/gravitino/catalog/postgresql/operation/TestPostgreSql.java +++ b/catalogs/catalog-jdbc-postgresql/src/test/java/org/apache/gravitino/catalog/postgresql/operation/TestPostgreSql.java @@ -29,7 +29,7 @@ import org.apache.gravitino.catalog.postgresql.converter.PostgreSqlExceptionConverter; import org.apache.gravitino.catalog.postgresql.converter.PostgreSqlTypeConverter; import org.junit.jupiter.api.BeforeAll; -import org.testcontainers.containers.PostgreSQLContainer; +import org.testcontainers.postgresql.PostgreSQLContainer; public class TestPostgreSql extends TestJdbc { @@ -38,7 +38,7 @@ public class TestPostgreSql extends TestJdbc { @BeforeAll public static void startup() throws Exception { CONTAINER = - new PostgreSQLContainer<>(DEFAULT_POSTGRES_IMAGE) + new PostgreSQLContainer(DEFAULT_POSTGRES_IMAGE) .withDatabaseName(TEST_DB_NAME) .withUsername("root") .withPassword("root"); diff --git a/catalogs/catalog-jdbc-postgresql/src/test/java/org/apache/gravitino/catalog/postgresql/operation/TestPostgreSqlTableOperations.java b/catalogs/catalog-jdbc-postgresql/src/test/java/org/apache/gravitino/catalog/postgresql/operation/TestPostgreSqlTableOperations.java index 13382aa471c..c31d5a736c7 100644 --- a/catalogs/catalog-jdbc-postgresql/src/test/java/org/apache/gravitino/catalog/postgresql/operation/TestPostgreSqlTableOperations.java +++ b/catalogs/catalog-jdbc-postgresql/src/test/java/org/apache/gravitino/catalog/postgresql/operation/TestPostgreSqlTableOperations.java @@ -28,6 +28,7 @@ import java.util.Map; import javax.sql.DataSource; import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.Strings; import org.apache.gravitino.catalog.jdbc.JdbcColumn; import org.apache.gravitino.catalog.jdbc.JdbcTable; import org.apache.gravitino.catalog.jdbc.config.JdbcConfig; @@ -593,7 +594,7 @@ public void testCreateAutoIncrementTable() { }); Assertions.assertTrue( - StringUtils.contains(illegalArgumentException.getMessage(), "Unsupported auto-increment")); + Strings.CS.contains(illegalArgumentException.getMessage(), "Unsupported auto-increment")); } @Test @@ -678,7 +679,7 @@ public void testCreateIndexTable() { primaryIndex); }); Assertions.assertTrue( - StringUtils.contains( + Strings.CS.contains( gravitinoRuntimeException.getMessage(), "column \"no_exist_1\" named in key does not exist")); } @@ -730,7 +731,7 @@ public void testAppendIndexesSql() { PostgreSqlTableOperations.appendIndexesSql( new Index[] {primary, unique1, unique2}, stringBuilder)); Assertions.assertTrue( - StringUtils.contains( + Strings.CS.contains( illegalArgumentException.getMessage(), "Index does not support complex fields in PostgreSQL")); } diff --git a/flink-connector/flink-common/build.gradle.kts b/flink-connector/flink-common/build.gradle.kts index d8c4151d825..e982b3dde50 100644 --- a/flink-connector/flink-common/build.gradle.kts +++ b/flink-connector/flink-common/build.gradle.kts @@ -98,7 +98,6 @@ dependencies { testImplementation(libs.mysql.driver) testImplementation(libs.sqlite.jdbc) testImplementation(libs.testcontainers) - testImplementation(libs.testcontainers.junit.jupiter) testImplementation(libs.testcontainers.mysql) testImplementation(libs.metrics.core) testImplementation(libs.flinkjdbc119) diff --git a/flink-connector/v1.19/flink/build.gradle.kts b/flink-connector/v1.19/flink/build.gradle.kts index a024fd083c7..fa0efbad426 100644 --- a/flink-connector/v1.19/flink/build.gradle.kts +++ b/flink-connector/v1.19/flink/build.gradle.kts @@ -127,7 +127,6 @@ dependencies { testImplementation(libs.postgresql.driver) testImplementation(libs.sqlite.jdbc) testImplementation(libs.testcontainers) - testImplementation(libs.testcontainers.junit.jupiter) testImplementation(libs.testcontainers.mysql) testImplementation("org.apache.iceberg:iceberg-core:$icebergVersion") diff --git a/flink-connector/v1.20/flink/build.gradle.kts b/flink-connector/v1.20/flink/build.gradle.kts index 0b6aec2a4b6..63bb789bb9c 100644 --- a/flink-connector/v1.20/flink/build.gradle.kts +++ b/flink-connector/v1.20/flink/build.gradle.kts @@ -127,7 +127,6 @@ dependencies { testImplementation(libs.postgresql.driver) testImplementation(libs.sqlite.jdbc) testImplementation(libs.testcontainers) - testImplementation(libs.testcontainers.junit.jupiter) testImplementation(libs.testcontainers.mysql) testImplementation("org.apache.iceberg:iceberg-core:$icebergVersion") diff --git a/flink-connector/v2.1/flink/build.gradle.kts b/flink-connector/v2.1/flink/build.gradle.kts index e8758b70860..76aed9b52e1 100644 --- a/flink-connector/v2.1/flink/build.gradle.kts +++ b/flink-connector/v2.1/flink/build.gradle.kts @@ -146,7 +146,6 @@ dependencies { testImplementation(libs.postgresql.driver) testImplementation(libs.sqlite.jdbc) testImplementation(libs.testcontainers) - testImplementation(libs.testcontainers.junit.jupiter) testImplementation(libs.testcontainers.mysql) testImplementation("org.apache.iceberg:iceberg-core:$icebergVersion") diff --git a/flink-connector/v2.2/flink/build.gradle.kts b/flink-connector/v2.2/flink/build.gradle.kts index e0a3bae1627..2a107e8696d 100644 --- a/flink-connector/v2.2/flink/build.gradle.kts +++ b/flink-connector/v2.2/flink/build.gradle.kts @@ -146,7 +146,6 @@ dependencies { testImplementation(libs.postgresql.driver) testImplementation(libs.sqlite.jdbc) testImplementation(libs.testcontainers) - testImplementation(libs.testcontainers.junit.jupiter) testImplementation(libs.testcontainers.mysql) testImplementation("org.apache.iceberg:iceberg-core:$icebergVersion") diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 408eb44dcf2..1d0c3167f30 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -94,10 +94,10 @@ trino-iceberg-it = "478" scala-collection-compat = "2.7.0" scala-java-compat = "1.0.2" sqlite-jdbc = "3.42.0.0" -testcontainers = "1.21.4" -# docker-java version aligned with testcontainers 1.21.4; forced in iceberg-rest-trino-it to +testcontainers = "2.0.5" +# docker-java version aligned with testcontainers 2.0.5; forced in iceberg-rest-trino-it to # override the newer versions Trino's test stack would otherwise pull in. -docker-java = "3.4.2" +docker-java = "3.7.1" jwt = "0.11.1" nimbus-jose-jwt = "10.9" jline = "3.21.0" @@ -330,12 +330,11 @@ trino-client= { group = "io.trino", name = "trino-client", version.ref = "trino" sqlite-jdbc = { group = "org.xerial", name = "sqlite-jdbc", version.ref = "sqlite-jdbc" } commons-dbcp2 = { group = "org.apache.commons", name = "commons-dbcp2", version.ref = "commons-dbcp2" } testcontainers = { group = "org.testcontainers", name = "testcontainers", version.ref = "testcontainers" } -testcontainers-jdbc = { group = "org.testcontainers", name = "jdbc", version.ref = "testcontainers" } -testcontainers-clickhouse = { group = "org.testcontainers", name = "clickhouse", version.ref = "testcontainers" } -testcontainers-mysql = { group = "org.testcontainers", name = "mysql", version.ref = "testcontainers" } -testcontainers-postgresql = { group = "org.testcontainers", name = "postgresql", version.ref = "testcontainers" } -testcontainers-junit-jupiter = { group = "org.testcontainers", name = "junit-jupiter", version.ref = "testcontainers" } -testcontainers-localstack = { group = "org.testcontainers", name = "localstack", version.ref = "testcontainers" } +testcontainers-jdbc = { group = "org.testcontainers", name = "testcontainers-jdbc", version.ref = "testcontainers" } +testcontainers-clickhouse = { group = "org.testcontainers", name = "testcontainers-clickhouse", version.ref = "testcontainers" } +testcontainers-mysql = { group = "org.testcontainers", name = "testcontainers-mysql", version.ref = "testcontainers" } +testcontainers-postgresql = { group = "org.testcontainers", name = "testcontainers-postgresql", version.ref = "testcontainers" } +testcontainers-localstack = { group = "org.testcontainers", name = "testcontainers-localstack", version.ref = "testcontainers" } trino-jdbc = { group = "io.trino", name = "trino-jdbc", version.ref = "trino" } jwt-api = { group = "io.jsonwebtoken", name = "jjwt-api", version.ref = "jwt"} jwt-impl = { group = "io.jsonwebtoken", name = "jjwt-impl", version.ref = "jwt"} diff --git a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/DorisContainer.java b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/DorisContainer.java index 658d273f8ce..c32f8b1e79d 100644 --- a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/DorisContainer.java +++ b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/DorisContainer.java @@ -128,8 +128,7 @@ protected DorisContainer( .withExposedService(FE_SERVICE, FE_HTTP_PORT) .withScaledService(BE_SERVICE, BE_SCALE) .withStartupTimeout(Duration.ofSeconds(STARTUP_TIMEOUT_SECONDS)) - .withTailChildContainers(true) - .withLocalCompose(true); + .withTailChildContainers(true); // Expose ports for each BE instance (1-indexed) for (int i = 1; i <= BE_SCALE; i++) { diff --git a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/MySQLContainer.java b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/MySQLContainer.java index b71a092f971..445f4be6c5f 100644 --- a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/MySQLContainer.java +++ b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/MySQLContainer.java @@ -134,18 +134,19 @@ public String getDriverClassName(TestDatabaseName testDatabaseName) throws SQLEx return DriverManager.getDriver(getJdbcUrl(testDatabaseName)).getClass().getName(); } - private static org.testcontainers.containers.MySQLContainer module( + private static org.testcontainers.mysql.MySQLContainer module( String image, Durability durability) { // The image may be a mirror or a custom tag; the module only needs a MySQL server. - return new org.testcontainers.containers.MySQLContainer<>( + return new org.testcontainers.mysql.MySQLContainer( DockerImageName.parse(image).asCompatibleSubstituteFor("mysql")) .withUsername(USER_NAME) .withPassword(PASSWORD) // The module always creates a database. Its default, "test", is also a schema name catalog // tests create, so name MySQL's own system database instead. .withDatabaseName("mysql") - // Our own config replaces the one Testcontainers 1.x bundles, so the server settings do not - // depend on the Testcontainers version; it carries the disposable settings too. + // The 2.x module mounts no default config, so the server runs on the image defaults plus + // ours, exactly as with 1.x, where this override replaced the bundled one. It carries the + // disposable settings too. .withConfigurationOverride( durability == Durability.DISPOSABLE ? DISPOSABLE_CONFIG : DEFAULT_CONFIG) // The module's readiness check queries the database and ignores BaseContainer's startup diff --git a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/PostgreSQLContainer.java b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/PostgreSQLContainer.java index 087561fe9c7..f510f04c351 100644 --- a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/PostgreSQLContainer.java +++ b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/PostgreSQLContainer.java @@ -59,7 +59,7 @@ protected PostgreSQLContainer( Optional network) { super( // The image may be a mirror or a custom tag; the module only needs a PostgreSQL server. - new org.testcontainers.containers.PostgreSQLContainer<>( + new org.testcontainers.postgresql.PostgreSQLContainer( DockerImageName.parse(image).asCompatibleSubstituteFor("postgres")) .withUsername(USER_NAME) .withPassword(PASSWORD) diff --git a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/util/BaseIT.java b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/util/BaseIT.java index bdbeaba0057..1db518b1369 100644 --- a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/util/BaseIT.java +++ b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/util/BaseIT.java @@ -248,7 +248,7 @@ public String startAndInitPGBackend() { POSTGRESQL_CONTAINER = containerSuite.getPostgreSQLContainer(); String pgUrlWithoutSchema = POSTGRESQL_CONTAINER.getJdbcUrl(META_DATA); - String randomSchemaName = RandomStringUtils.random(10, true, false); + String randomSchemaName = RandomStringUtils.insecure().next(10, true, false); // Connect to the PostgreSQL docker and create a schema String currentExecuteSql = ""; try (Connection connection = diff --git a/maintenance/jobs/build.gradle.kts b/maintenance/jobs/build.gradle.kts index ae5055bda8c..76583b16364 100644 --- a/maintenance/jobs/build.gradle.kts +++ b/maintenance/jobs/build.gradle.kts @@ -67,7 +67,6 @@ dependencies { testImplementation(libs.junit.jupiter.api) testImplementation(libs.awaitility) testImplementation(libs.testcontainers) - testImplementation(libs.testcontainers.junit.jupiter) testImplementation(libs.testcontainers.mysql) testRuntimeOnly(libs.mysql.driver) testImplementation("org.apache.iceberg:iceberg-spark-runtime-${sparkMajorVersion}_$scalaVersion:$icebergVersion") { diff --git a/maintenance/jobs/src/test/java/org/apache/gravitino/maintenance/jobs/iceberg/TestIcebergUpdateStatsJobWithSpark.java b/maintenance/jobs/src/test/java/org/apache/gravitino/maintenance/jobs/iceberg/TestIcebergUpdateStatsJobWithSpark.java index 9a85d482d46..82270c9b23d 100644 --- a/maintenance/jobs/src/test/java/org/apache/gravitino/maintenance/jobs/iceberg/TestIcebergUpdateStatsJobWithSpark.java +++ b/maintenance/jobs/src/test/java/org/apache/gravitino/maintenance/jobs/iceberg/TestIcebergUpdateStatsJobWithSpark.java @@ -72,7 +72,7 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.api.condition.EnabledIfEnvironmentVariable; import org.junit.jupiter.api.io.TempDir; -import org.testcontainers.containers.MySQLContainer; +import org.testcontainers.mysql.MySQLContainer; /** Integration tests for IcebergUpdateStatsAndMetricsJob with a real Spark+Iceberg runtime. */ public class TestIcebergUpdateStatsJobWithSpark { @@ -180,7 +180,7 @@ public void testRunBuiltInUpdateStatsJobViaServerForAllModes() throws Exception SparkSession restSpark = null; try { restSpark = createRestSparkSession(); - try (MySQLContainer mysql = new MySQLContainer<>("mysql:8.0.33"); + try (MySQLContainer mysql = new MySQLContainer("mysql:8.0.33"); GravitinoAdminClient adminClient = GravitinoAdminClient.builder(SERVER_URI).build()) { mysql.start(); initializeMySqlMetricsSchema(mysql); @@ -423,7 +423,7 @@ public void testUpdateNonPartitionedTableStatisticsAndMetrics() { @Test @Tag("gravitino-docker-test") public void testUpdatePartitionedTableMetricsStoredInMySql() throws Exception { - try (MySQLContainer mysql = new MySQLContainer<>("mysql:8.0.33")) { + try (MySQLContainer mysql = new MySQLContainer("mysql:8.0.33")) { mysql.start(); Map conf = new HashMap<>(); @@ -465,7 +465,7 @@ public void testUpdatePartitionedTableMetricsStoredInMySql() throws Exception { } } - private static void initializeMySqlMetricsSchema(MySQLContainer mysql) throws Exception { + private static void initializeMySqlMetricsSchema(MySQLContainer mysql) throws Exception { Path schemaPath = findRepoRoot() .resolve("scripts") @@ -629,7 +629,7 @@ private static List getPartitionMetrics( DataScope.forPartition(tableIdentifier, partitionPath), now - 1800, now + 1800); } - private static Map buildJdbcMetricsConfigs(MySQLContainer mysql) { + private static Map buildJdbcMetricsConfigs(MySQLContainer mysql) { Map conf = new HashMap<>(); conf.put("gravitino.optimizer.jdbcMetrics.jdbcUrl", mysql.getJdbcUrl()); conf.put("gravitino.optimizer.jdbcMetrics.jdbcUser", mysql.getUsername()); @@ -646,7 +646,7 @@ private static String buildUpdaterOptionsForStatsUpdaterOnly() { + "\",\"statistics_updater\":\"gravitino-statistics-updater\"}"; } - private static String buildUpdaterOptionsForMetricsUpdaterOnly(MySQLContainer mysql) { + private static String buildUpdaterOptionsForMetricsUpdaterOnly(MySQLContainer mysql) { return "{\"metrics_updater\":\"gravitino-metrics-updater\"," + "\"gravitino.optimizer.jdbcMetrics.jdbcUrl\":\"" + mysql.getJdbcUrl() @@ -662,7 +662,7 @@ private static String buildUpdaterOptionsForMetricsUpdaterOnly(MySQLContainer + "\"}"; } - private static String buildUpdaterOptionsForAllMode(MySQLContainer mysql) { + private static String buildUpdaterOptionsForAllMode(MySQLContainer mysql) { return "{\"gravitino_uri\":\"" + SERVER_URI + "\",\"metalake\":\"" diff --git a/maintenance/optimizer/build.gradle.kts b/maintenance/optimizer/build.gradle.kts index aa6b482204f..414626a7a89 100644 --- a/maintenance/optimizer/build.gradle.kts +++ b/maintenance/optimizer/build.gradle.kts @@ -87,7 +87,6 @@ dependencies { exclude(group = "org.slf4j", module = "slf4j-log4j12") } testImplementation(libs.testcontainers) - testImplementation(libs.testcontainers.junit.jupiter) testImplementation(libs.testcontainers.mysql) testImplementation(libs.testcontainers.postgresql) testImplementation(project(":integration-test-common", "testArtifacts")) diff --git a/maintenance/optimizer/src/test/java/org/apache/gravitino/maintenance/optimizer/integration/test/storage/TestMySqlGenericJdbcMetricsRepositoryBehaviorIT.java b/maintenance/optimizer/src/test/java/org/apache/gravitino/maintenance/optimizer/integration/test/storage/TestMySqlGenericJdbcMetricsRepositoryBehaviorIT.java index da64f064c3d..facdd3ed23a 100644 --- a/maintenance/optimizer/src/test/java/org/apache/gravitino/maintenance/optimizer/integration/test/storage/TestMySqlGenericJdbcMetricsRepositoryBehaviorIT.java +++ b/maintenance/optimizer/src/test/java/org/apache/gravitino/maintenance/optimizer/integration/test/storage/TestMySqlGenericJdbcMetricsRepositoryBehaviorIT.java @@ -25,17 +25,17 @@ import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.TestInstance; -import org.testcontainers.containers.MySQLContainer; +import org.testcontainers.mysql.MySQLContainer; @Tag("gravitino-docker-test") @TestInstance(TestInstance.Lifecycle.PER_CLASS) class TestMySqlGenericJdbcMetricsRepositoryBehaviorIT extends BaseGenericJdbcMetricsRepositoryTest { - private MySQLContainer mysql; + private MySQLContainer mysql; @BeforeAll void setUp() { - mysql = new MySQLContainer<>("mysql:8.0.33"); + mysql = new MySQLContainer("mysql:8.0.33"); mysql.start(); JdbcMetricsRepositoryITUtils.initializeSchema( mysql.getJdbcUrl(), mysql.getUsername(), mysql.getPassword(), "mysql"); diff --git a/maintenance/optimizer/src/test/java/org/apache/gravitino/maintenance/optimizer/integration/test/storage/TestPostgreSqlGenericJdbcMetricsRepositoryIT.java b/maintenance/optimizer/src/test/java/org/apache/gravitino/maintenance/optimizer/integration/test/storage/TestPostgreSqlGenericJdbcMetricsRepositoryIT.java index 8f29479be05..6761e7e0070 100644 --- a/maintenance/optimizer/src/test/java/org/apache/gravitino/maintenance/optimizer/integration/test/storage/TestPostgreSqlGenericJdbcMetricsRepositoryIT.java +++ b/maintenance/optimizer/src/test/java/org/apache/gravitino/maintenance/optimizer/integration/test/storage/TestPostgreSqlGenericJdbcMetricsRepositoryIT.java @@ -25,17 +25,17 @@ import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.TestInstance; -import org.testcontainers.containers.PostgreSQLContainer; +import org.testcontainers.postgresql.PostgreSQLContainer; @Tag("gravitino-docker-test") @TestInstance(TestInstance.Lifecycle.PER_CLASS) class TestPostgreSqlGenericJdbcMetricsRepositoryIT extends BaseGenericJdbcMetricsRepositoryTest { - private PostgreSQLContainer postgres; + private PostgreSQLContainer postgres; @BeforeAll void setUp() { - postgres = new PostgreSQLContainer<>("postgres:13"); + postgres = new PostgreSQLContainer("postgres:13"); postgres.start(); JdbcMetricsRepositoryITUtils.initializeSchema( postgres.getJdbcUrl(), postgres.getUsername(), postgres.getPassword(), "postgresql"); From 34d493e854bffa829476e4647b89a9eb428cc4fb Mon Sep 17 00:00:00 2001 From: Nevin Zheng Date: Wed, 21 Oct 2026 09:03:00 -0700 Subject: [PATCH 04/14] [#14105] improvement(test): Let BackendTestExtension use JUnit's built-in store cleanup --- .../relational/BackendTestExtension.java | 73 ++++++++++--------- 1 file changed, 39 insertions(+), 34 deletions(-) diff --git a/core/src/test/java/org/apache/gravitino/storage/relational/BackendTestExtension.java b/core/src/test/java/org/apache/gravitino/storage/relational/BackendTestExtension.java index 39b7dcfb64e..ab787ac9aed 100644 --- a/core/src/test/java/org/apache/gravitino/storage/relational/BackendTestExtension.java +++ b/core/src/test/java/org/apache/gravitino/storage/relational/BackendTestExtension.java @@ -47,8 +47,6 @@ import org.apache.gravitino.integration.test.util.BaseIT; import org.apache.gravitino.storage.RandomIdGenerator; import org.apache.gravitino.storage.relational.service.EntityIdService; -import org.junit.jupiter.api.extension.AfterAllCallback; -import org.junit.jupiter.api.extension.AfterEachCallback; import org.junit.jupiter.api.extension.BeforeEachCallback; import org.junit.jupiter.api.extension.Extension; import org.junit.jupiter.api.extension.ExtensionContext; @@ -69,21 +67,23 @@ *

{@code SqlSessionFactoryHelper} is process-global, so only one backend can be live in a JVM. * The extension keeps it in a single slot and reuses it for the tests of one class, emptying it * before each test per the test's {@link DatabaseFixture}. It builds a new backend when the test - * class or backend type changes and for every {@link ResetStrategy#FRESH_DATABASE} test, and closes - * the class's backend when the class finishes. + * class or backend type changes and for every {@link ResetStrategy#FRESH_DATABASE} test. + * + *

JUnit closes each backend with the store that holds it: a shared backend lives in the test + * class's store and closes when the class finishes, a fresh one in the test's store and closes when + * the test finishes. The extension closes one by hand only to replace it or to discard it after a + * failed reset. * *

The slot is read and replaced without locking, so tests must run one at a time: the core * database lanes run in one JVM with JUnit parallel execution off. */ -public class BackendTestExtension - implements TestTemplateInvocationContextProvider, AfterAllCallback { +public class BackendTestExtension implements TestTemplateInvocationContextProvider { private static final Logger LOG = LoggerFactory.getLogger(BackendTestExtension.class); private static final String DOCKER_TEST_FLAG = "dockerTest"; private static final ExtensionContext.Namespace NAMESPACE = ExtensionContext.Namespace.create(BackendTestExtension.class); - // Root store: one slot serves every class, and JUnit closes a leftover when the run ends. private static final String CURRENT_BACKEND = "CURRENT_BACKEND"; private final BackendFactory backendFactory; @@ -96,20 +96,29 @@ public BackendTestExtension() { this.backendFactory = backendFactory; } - @Override - public void afterAll(ExtensionContext context) throws Exception { - ExtensionContext.Store store = context.getRoot().getStore(NAMESPACE); - LiveBackend current = store.get(CURRENT_BACKEND, LiveBackend.class); - if (current != null && current.testClass == context.getRequiredTestClass()) { - closeCurrent(store); + // Store.get also reads the stores of enclosing contexts, but remove() only clears the context's + // own store, so walk up to the store that holds the backend. Removing it first leaves the slot + // empty even when close() fails. + private static void closeCurrent(ExtensionContext classContext) throws IOException { + for (ExtensionContext owner = classContext; + owner != null; + owner = owner.getParent().orElse(null)) { + LiveBackend current = owner.getStore(NAMESPACE).remove(CURRENT_BACKEND, LiveBackend.class); + if (current != null) { + current.close(); + return; + } } } - private static void closeCurrent(ExtensionContext.Store store) throws IOException { - LiveBackend current = store.remove(CURRENT_BACKEND, LiveBackend.class); - if (current != null) { - current.close(); + // Test template invocations and template methods are method contexts; the backend belongs to + // the class. + private static ExtensionContext classContext(ExtensionContext context) { + ExtensionContext current = context; + while (current.getTestMethod().isPresent()) { + current = current.getParent().orElseThrow(); } + return current; } @Override @@ -166,7 +175,7 @@ interface BackendFactory { LiveBackend create(String backendType, Class testClass) throws Exception; } - private class BackendSetupCallback implements BeforeEachCallback, AfterEachCallback { + private class BackendSetupCallback implements BeforeEachCallback { private final String backendType; private BackendSetupCallback(String backendType) { @@ -175,28 +184,31 @@ private BackendSetupCallback(String backendType) { @Override public void beforeEach(ExtensionContext context) throws Exception { - ExtensionContext.Store store = context.getRoot().getStore(NAMESPACE); + ExtensionContext classContext = classContext(context); Class testClass = context.getRequiredTestClass(); ResetStrategy resetStrategy = resetStrategy(context); boolean fresh = resetStrategy == ResetStrategy.FRESH_DATABASE; - LiveBackend current = store.get(CURRENT_BACKEND, LiveBackend.class); + // A fresh backend lives in the previous test's store, which JUnit has closed by now. + LiveBackend current = + classContext.getStore(NAMESPACE).get(CURRENT_BACKEND, LiveBackend.class); if (current != null && (fresh || current.testClass != testClass || !current.backendType.equals(backendType))) { // Only one backend can be live (see the class Javadoc), so the old one goes first. - closeCurrent(store); + closeCurrent(classContext); current = null; } if (current == null) { current = backendFactory.create(backendType, testClass); - store.put(CURRENT_BACKEND, current); + // JUnit closes the backend with the test (fresh) or the class (shared) that owns it. + (fresh ? context : classContext).getStore(NAMESPACE).put(CURRENT_BACKEND, current); } // Other core tests overwrite these globals, so point them back before every test. current.bindGravitinoEnv(); if (!fresh) { // Runs before the test's own @BeforeEach, so its setup always starts from empty tables. - resetOrDiscard(store, current, resetStrategy); + resetOrDiscard(classContext, current, resetStrategy); } Object testInstance = context.getRequiredTestInstance(); @@ -206,23 +218,16 @@ public void beforeEach(ExtensionContext context) throws Exception { } } - @Override - public void afterEach(ExtensionContext context) throws Exception { - if (resetStrategy(context) == ResetStrategy.FRESH_DATABASE) { - closeCurrent(context.getRoot().getStore(NAMESPACE)); - } - } - // A reset that fails (e.g. on a lock an earlier test left) would fail every later test the same // way, so drop the backend and let the next test build a clean one. private void resetOrDiscard( - ExtensionContext.Store store, LiveBackend current, ResetStrategy resetStrategy) + ExtensionContext classContext, LiveBackend current, ResetStrategy resetStrategy) throws Exception { try { current.reset(resetStrategy); } catch (Exception e) { try { - closeCurrent(store); + closeCurrent(classContext); } catch (IOException closeFailure) { e.addSuppressed(closeFailure); } @@ -243,8 +248,8 @@ private ResetStrategy resetStrategy(ExtensionContext context) { } /** - * Closes with the root store, in reverse insertion order, so before the containers {@code - * CloseContainerExtension} registered earlier. + * Closes with the class or test store that holds it, so always before the root store closes the + * containers {@code CloseContainerExtension} registered. */ static class LiveBackend implements AutoCloseable { private final String backendType; From 1509507fdc2b54fb1fabb204142e46d69cb598de Mon Sep 17 00:00:00 2001 From: Nevin Zheng Date: Mon, 19 Oct 2026 09:30:31 -0700 Subject: [PATCH 05/14] [#14000] docs(test): Add the test container framework design Design doc for #13900 and a rumdl config for design-docs/testing. --- design-docs/testing/.rumdl.toml | 64 + .../test-container-framework-design.md | 1474 +++++++++++++++++ 2 files changed, 1538 insertions(+) create mode 100644 design-docs/testing/.rumdl.toml create mode 100644 design-docs/testing/containers/test-container-framework-design.md diff --git a/design-docs/testing/.rumdl.toml b/design-docs/testing/.rumdl.toml new file mode 100644 index 00000000000..94792ff798d --- /dev/null +++ b/design-docs/testing/.rumdl.toml @@ -0,0 +1,64 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +# Markdown lint and format settings for the testing design docs. +# Based on rumdl's own config (github.com/rvben/rumdl/.rumdl.toml). +# Check: rumdl check -c design-docs/testing/.rumdl.toml design-docs/testing +# Format: rumdl fmt -c design-docs/testing/.rumdl.toml design-docs/testing + +[global] +# Cache in build/ next to this file; build/ is git-ignored, so RAT and jj skip it. +cache-dir = "build/rumdl_cache" +# MD041: the Apache license header is an HTML comment before the title. +disable = ["MD041"] + +# Wrap prose at 100 columns; `rumdl fmt` reflows paragraphs. +[MD013] +line-length = 100 +code-blocks = true +tables = false +headings = true +reflow = true + +# Align table columns; a table wider than 100 columns falls back to compact. +[MD060] +enabled = true +style = "aligned" +max-width = 100 + +[MD055] +style = "leading_and_trailing" + +[MD029] +style = "ordered" + +[MD046] +style = "fenced" + +[MD048] +style = "backtick" + +[MD049] +style = "asterisk" + +[MD050] +style = "asterisk" + +[MD051] +anchor-style = "github" diff --git a/design-docs/testing/containers/test-container-framework-design.md b/design-docs/testing/containers/test-container-framework-design.md new file mode 100644 index 00000000000..df3733d647e --- /dev/null +++ b/design-docs/testing/containers/test-container-framework-design.md @@ -0,0 +1,1474 @@ + + +# SPIP: Test Container Framework for Apache Gravitino + +| Field | Value | +| --- | --- | +| Status | Draft for discussion; S0–S7 built and green locally, CI running on the draft PRs | +| Author | Nevin Zheng | +| Issue | [#13900](https://github.com/apache/gravitino/issues/13900) (epic; children #14000–#14003, #14049) | +| Stack | #14103 → #14083–#14085 → #14005 → #14006 → #14052 → #14076 → #14007–#14012 → #14013 → #14079 → #14077 → #14078 → #14086–#14091 → #14094 | +| Scope | `integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/` and the integration tests that use it | + +## Highlights + +Gravitino's Docker-backed integration tests get one fixture framework for every container. On one +local Mac the container-heavy modules finish 32–79 % faster, no container outlives a killed test +JVM, and 14 defects on `main` are fixed along the way. The table lists each key result: its +target, the value on `main`, the value with the stack, and where the evidence is. + +| # | Key result | Target | `main` | With the stack | Status | Evidence | +| --- | --- | --- | --- | --- | --- | --- | +| 1 | No leaked containers | 0 left after a killed test JVM | Leaks (Ryuk off) | 0; gone within 15 s | Met | #14006, Appendix F | +| 2 | Faster local integration tests | Container-heavy modules faster, no module slower | idp-basic 488 s; catalog-hive 578 s | idp-basic 102 s (−79 %); catalog-hive 395 s (−32 %), then 380 s to 235 s with #14052; no measured module slower | Met locally | Results at a glance, Appendix H | +| 3 | One way to get a container | 0 direct `ContainerSuite` calls | 478 in 117 test files | 0 | Met | #14077 | +| 4 | One readiness pattern | 1 | 9 hand-written styles | 1 (`WaitStrategy`) | Met | #14086, Appendix C | +| 5 | No commands run inside containers | 0 on test paths | 65 | 0; the last two (`kadmin.local` in one test) move into the container's own start | Met | #14076, #14091, #14094, Appendix D | +| 6 | Every container start measured | One timing line per start | None | Every start, failed starts included | Met | #14006, Appendix H | +| 7 | Runs isolated from each other | 0 host-wide fixed names | 3 | 0 | Met | #14078, Appendix F | +| 8 | Defects on `main` fixed | Each one found gets a fix | — | 14 found and fixed | Met | Appendix M | +| 9 | Apache CI time | No module slower on CI | To be measured | To be measured | Open | Final timing campaign (Q7) | + +Why each result matters, and where it is explained: + +1. **No leaked containers.** A killed or crashed test JVM used to leave its containers running, + holding ports, memory and disk until someone pruned them; on a shared CI host or a laptop that + breaks the next run. Ryuk removes them (Appendix F; finding M-4 in Appendix M). +2. **Faster local integration tests.** Contributors wait for Docker tests before every push; + shorter runs mean more of them get run locally instead of being left to CI. The gain comes + from fewer starts and shorter waits, not from skipping tests (Appendices B, C, D, E; Results + at a glance). +3. **One way to get a container.** 478 hand-written calls each chose when to start and close a + shared container, which is how containers got closed mid-run under other tests. One fixture + per test makes that impossible and makes tests read the same (Appendix A; finding M-5). +4. **One readiness pattern.** Nine hand-written poll loops had nine budgets and bugs (a first poll + that always waited, a retry that never did). One Testcontainers `WaitStrategy` is reviewed + once and fixed once (Appendix C; findings M-2, M-8). +5. **No commands run inside containers.** Each `hive -e` or `hadoop fs` boots a JVM inside the + container, costing seconds per call, and its errors were easy to ignore. In-process clients are + faster and fail loudly (Appendix D; findings M-9, M-10). +6. **Every container start measured.** Nothing above was visible before; a slow start or a + restart loop was guessed, not seen. One `[CONTAINER-TIMING]` line per start makes a regression + show up in the log of the PR that causes it (Appendix H). +7. **Runs isolated from each other.** Fixed names let only one run per Docker host; a second run + broke the first. Per-run names let two runs share a host (Appendix F; finding M-14). +8. **Defects on `main` fixed.** Each is a real bug in the current library, found by measuring or + by migrating; together they are the case for S2 on its own (Appendix M). +9. **Apache CI time.** CI is what every PR waits on. Local numbers are labelled local until the CI + job times are measured on `main` and on the stack (Q7, Appendix H). + +## Summary + +We set out to make Gravitino's Docker-backed integration tests cheaper, leak-free and uniform. +On one local Mac (5-round medians, same tests and pass counts), idp-basic went from 488 s to +102 s (−79 %), hive-metastore-common from 394 s to 175 s (−56 %) and catalog-hive from 578 s to +395 s (−32 %); a further step took catalog-hive from 380 s to 235 s (−38 %, 3-round medians) by +running Hive SQL over JDBC instead of `hive -e` inside the container. Measuring and reading the +library found 14 defects on `main`, all fixed in the stack, among them containers that outlived +a killed JVM and a test setup that had been failing silently (Appendix M). Evidence: the +[results comment on #13900](https://github.com/apache/gravitino/issues/13900#issuecomment-6293741946), +Appendix H and the per-strategy appendices B–G. Apache CI totals are to be measured. + +### Results at a glance + +One row per module: its time on `main`, its time with the stack, the change, and what drove it. +Local Mac, per-module test task wall time, five-round medians, same tests and pass counts on both +sides (method and environment in Appendix H). The stack column is the stack through #14012; the +final campaign re-measures every row on the full stack (#14091) and adds Apache CI job times. + +| Module | Tests | `main` | Stack | Change | Main contributors | +| --- | --- | --- | --- | --- | --- | +| idp-basic | 221 | 488 s | 102 s | −79 % | MySQL and PostgreSQL start once per run instead of after every test (#14006; 423 s to 96 s on its own, Appendix B) | +| hive-metastore-common | 52 | 394 s | 175 s | −56 % | Hive keyed by version, so no suite-wide close to switch Hive (#14006; Hive starts 6 to 4); no wait before the first readiness check (#14006, Appendix C) | +| flink-1.20 | 439 | 426 s | 263 s | −38 % | No suite-wide closes (#14006; 355 s to 298 s on its own); faster Hive readiness (#14006) | +| catalog-kafka | 25 | 34 s | 22 s | −35 % | To be attributed in the final campaign | +| catalog-hive | 308 | 578 s | 395 s | −32 % | Fewer Hive starts (6 to 5) and faster readiness (#14006); then Hive SQL over JDBC instead of `hive -e` takes it from 380 s to 235 s (#14052, three rounds, Appendix D) | +| catalog-fileset | 436 | 253 s | 237 s | −6 % | Test-bound; its Kerberos ITs gain another 10 % from in-process clients (#14076, Appendix D) | +| spark-3.5 | 581 | 482 s | 452 s | −6 % | Test-bound: most of its time is the tests, not containers | + +**What changes.** Every integration test gets one way to ask for a container: a fixture +annotation such as `@HiveFixture(version = HIVE3, auth = KERBEROS)`, served by one JUnit +extension that starts each configuration once per run, injects it and closes it at run end. Six +optimizations hang off that one control point, and every start logs what it cost. The 478 direct +`ContainerSuite` calls in 117 test files go to 0, nine hand-written readiness loops become one +Testcontainers `WaitStrategy`, and six services run on official Testcontainers modules. Test +code only; no user-facing change. + +## Goals + +What the project and its contributors get if the proposal is adopted. Each goal has a measure +and the evidence behind it; Q7 tracks them as exams. + +| Goal | Measure | Evidence | +| --- | --- | --- | +| **Faster, cheaper tests.** Fewer container starts, no fixed waits, no JVM boots inside containers, no durability work tests never read back | Per-module test task wall time; container starts per run | Local Mac medians: −32 % to −79 % on container-heavy modules, −6 % on test-bound ones (Appendix H); #14052 and #14076 per-module numbers (Appendix D); CI: to be measured | +| **Reliability.** No leaked containers, no stopped container handed to a later test, no silently failing setup | Leaks after a killed JVM; defects on `main` found and fixed | Ryuk kill test: containers gone by 15 s (Appendix F); 14 findings, each with its PR (Appendix M) | +| **Modern infrastructure.** JUnit 6, Testcontainers 2, official modules, Testcontainers wait strategies | Library versions; official modules in use; readiness styles | JUnit 6.1.3, Testcontainers 2.0.5 (Appendix L); modules 2 → 6, readiness styles 9 → 1 (Appendix G) | +| **Industry practice.** Shared containers through one registry keyed by configuration, as Testcontainers recommends; fixtures as dependency injection | Pattern matches upstream guidance and the merged `core` precedent | [testcontainers-java#1441](https://github.com/testcontainers/testcontainers-java/issues/1441); `BackendTestExtension` in #13518 (Q2) | +| **Readable, uniform test code.** One pattern for every service: `@Fixture`, `Spec`, `waitStrategy()` | Direct `ContainerSuite` call sites; image sources; exec sites on test paths | 478 → 0; 4 → 1; 65 → 2 (Q7) | +| **Visibility.** Every start and every test task logs its cost, so a regression is seen, not guessed | `[CONTAINER-TIMING]` and `[TEST-TIMING]` lines in every module | Appendix H method; CI upload of the lines is a later step | + +## The ask + +1. Review S2 now (#14006, #14052, #14076): lifecycle fixes, timing lines, Ryuk on, Hive without + its CLI. None of it needs the framework, and it carries most of the measured gain. +2. Answer the decisions below, each on its own, before S3 opens for review. A counter-proposal + on any of them costs a paragraph here, not a rewrite later. +3. Merge S0 (JUnit 6, Testcontainers 2) as the test toolchain floor; each PR stands alone. + +### Decisions for maintainers + +Each decision can be answered alone. The recommendation is what the stack implements; the +alternatives are in Appendix I. + +| ID | Question | Recommendation | Alternatives | A yes unlocks | +| --- | --- | --- | --- | --- | +| D-1 | Adopt the fixture API in Appendix A: `@Fixture`, `Spec`, `ContainerExtension`, `@FixtureContainer`; the `scope`, `startup`, `durability` attributes; Specs with `dependencies()` | Yes; it is the `BackendTestExtension` model from #13518 applied to every container | `@Testcontainers`/`@Container` (no sharing across classes); keep `ContainerSuite` and fix bugs only | S3 (#14007–#14012), then S5 and S7 | +| D-2 | Ryuk on by default, unprivileged, with `TESTCONTAINERS_RYUK_DISABLED=true` as the opt-out | Yes; a killed JVM's containers are gone in 15 s (Appendix F) | Keep Ryuk off and rely on CI prune; our own reaper by label | The Ryuk part of #14006; the CI stale-container sweep later | +| D-3 | Raise the test toolchain floor first: JUnit 6.1.3 and Testcontainers 2.0.5 as S0 | Yes; four mechanical, test-only PRs that merge and revert alone (Appendix L) | Upgrade after the stack; fold the Testcontainers bump into #14087 | S0 (#14103, #14083–#14085); every framework PR targets the final API once | +| D-4 | Merge segment by segment, bottom-up, with S2 needing no framework approval | Yes; S2 is bug fixes, timing and CLI removal, measured on its own | Wait for the whole design before any merge; two parallel review lanes | S2 review now (#14006, #14052, #14076) | +| D-5 | Disposable data by default, with `durability` per fixture and `-PdurableTestContainers` for debugging | Yes; a test that needs durable data says so where it declares the container (Appendix E) | One global JVM property; durable by default | S4 (#14013, #14079) and the `DURABLE` scope rules in #14089 | +| D-6 | The Trino cluster is Java (`TrinoCluster`) and owns its own Hive, MySQL and PostgreSQL; the Compose files and launch scripts go | Yes; same container classes, readiness as probes, logs still copied for CI, `trino_integration_test.sh` keeps its entry point | `ComposeContainer` over the existing Compose files with fixed names | #14012 and #14090 | +| D-7 | Ranger keeps its policies in the shared MySQL instead of the MariaDB inside its image | Yes; one database decision per Spec, the MySQL disposable recipe applies, as Apache Ranger's own Docker setup does | Keep the in-image MariaDB | The Ranger dependency edge in #14011 and its recipe in #14013 | + +**How to respond.** Comment on #14005 with the decision ID and yes, no, or a question; a no +with a counter-proposal is enough to change the text. A yes on D-4 alone lets S2 review start; +D-1 gates S3 and later; D-3 gates S0. A review comment on any PR body works the same way; the +body names the segment and the appendix it proves. + +--- + +## Q1. What are we trying to do? + +**Goal.** Give every Gravitino integration test one way to get a Docker container, and make the +containers behind it cheap, measured and leak-free. + +**Impact.** On one local Mac, modules that start many containers finish 32–79 % faster; a +hard-killed test JVM leaves no container running; every start logs what it cost (Summary, Q7). + +**How.** + +1. **One control point.** A test declares a fixture and receives a running container. One JUnit + extension owns its lifetime. This replaces the `ContainerSuite` singleton, which grew one + service at a time. +2. **Six optimizations on that point.** Start fewer containers, wait only as long as readiness + takes, use in-process clients instead of CLIs inside containers, skip durability work the + tests never read back, isolate runs by name, and build on official Testcontainers modules + (Appendices B–G). +3. **Measured.** Every start logs one `[CONTAINER-TIMING]` line with pull, start and readiness + time, so a reviewer or a CI job sees how many starts a run made and what each cost. + +**What a test author sees.** Today (`CatalogKafkaIT` on `main`): + +```java +private static final ContainerSuite CONTAINER_SUITE = ContainerSuite.getInstance(); + +@BeforeAll +public void startUp() { + kafkaBootstrapServers = String.format("%s:%d", + CONTAINER_SUITE.getKafkaContainer().getContainerIpAddress(), DEFAULT_BROKER_PORT); +``` + +With the stack (#14007): + +```java +@KafkaFixture +public class CatalogKafkaIT extends BaseIT { + + @BeforeAll + public void startUp(KafkaContainer kafka) { + kafkaBootstrapServers = String.format("%s:%d", + kafka.getContainerIpAddress(), DEFAULT_BROKER_PORT); +``` + +The flow behind the two versions. Today every test reaches a static singleton; with the stack +every test declares a Spec and the extension serves it. + +```mermaid +flowchart LR + subgraph today["Today: ContainerSuite"] + t1["Test class"] -->|"startHiveContainer()"| suite["ContainerSuite
static singleton"] + t2["Another test"] -->|"containerSuite.close()"| suite + suite -->|"one field per type"| hive1["HiveContainer"] + suite -->|"polls after start()"| poll["checkContainerStatus()"] + end + subgraph stack["With the stack: fixtures"] + f1["@HiveFixture(version = HIVE3)"] --> ext["ContainerExtension"] + f2["Fixtures.get(spec)"] --> ext + ext -->|"one container per Spec"| shared["SharedContainers
(package-private)"] + shared --> hive2["HiveContainer
waitStrategy()"] + ext -.->|"closes at run end"| shared + end +``` + +Legend: solid arrows are calls; the dotted arrow is the close at run end. + +--- + +## Q2. What problem is this solving, and what are the current limits? + +The container library grew one service at a time, so each service solved lifecycle, readiness +and data setup in its own way, and nothing measured the cost. The pieces on `main` before this +work: + +| Part | What it does | +| ------------------------- | -------------------------------------------------------------- | +| `BaseContainer` | Wraps a `GenericContainer`; sets ports, env, files, port wait | +| 15 container classes | Hive, MySQL, PostgreSQL, Doris, Kafka, Trino, Ranger, and more | +| `ContainerSuite` | Static singleton; one field per type; starts each on first use | +| `CloseContainerExtension` | Closes the suite once per run through the JUnit root store | +| `checkContainerStatus` | Each class polls its service after `start()` | +| Images | Constants, Gradle env vars, an enum and a compose file | + +117 test files make 478 direct `ContainerSuite` calls (`startContainer`, `getContainer`) +and reach the container by bridge IP (`getContainerIpAddress()`, 92 files). Ryuk is disabled in +`build.gradle.kts`, so cleanup relies on `close()` and on `docker container prune` in CI. + +The limits (Q7 sizes each on `main` and with the stack): + +| Area | Limit | +| -------------- | ------------------------------------------------------------------------------------ | +| Lifecycle | Six test classes close the whole suite mid-run; idp-basic closes it after every test | +| Lifecycle | Hive is not keyed by version, so Hive2 tests must close the suite for Hive3 | +| Lifecycle | `close()` resets only some fields; a later start gets a stopped container | +| Lifecycle | A start whose readiness check times out leaves the container running | +| Readiness | Every poll waits one interval before the first try | +| Readiness | Nine services, nine hand-written poll loops; PostgreSQL's retry never waits | +| Readiness | Probes spawn a JVM inside the container every second | +| Process spawns | Tests and probes run `hive -e`, `hadoop fs`, `mysql`, `awslocal` inside containers | +| Start path | In CI every start call, not every start, runs `docker container prune`, `df`, `free` | +| Isolation | Fixed names (`gravitino-ci-network`, `trino-ci-*`, `/tmp/obdata`): one run per host | +| Images | Images come from four places; StarRocks uses a floating tag | +| Cleanup | Ryuk is off, so containers of a hard-killed JVM keep running | +| Measurement | No log line says how long a container took to start | + +The last row matters most: nothing above was visible, so nothing was fixed. + +### Precedent + +The core database work ([#13515](https://github.com/apache/gravitino/issues/13515)) fixed the +same waste for MySQL and PostgreSQL in `core`: + +- [#13517](https://github.com/apache/gravitino/pull/13517): one CI lane per database backend. +- [#13518](https://github.com/apache/gravitino/pull/13518): `BackendTestExtension` keeps one + backend per test class in the JUnit root store and resets rows instead of restarting. CI time: + MySQL 31m09s to 9m41s, PostgreSQL 20m24s to 7m28s, H2 7m29s to 5m07s. +- [#13899](https://github.com/apache/gravitino/pull/13899) (merged): test database data in memory + without fsync; MySQL and PostgreSQL on the official Testcontainers modules. On a local Mac + (OrbStack): coreMySQLTest 448 s to 223 s, corePostgreSQLTest 432 s to 313 s. On GitHub Linux + CI it was flat for H2 and MySQL and about 10 % faster for PostgreSQL; #13518 is the big CI win. + +This proposal applies two of those ideas (share per run, skip durability) to every container +type and adds the ones only non-database services need. Row reset stays a `core` technique. + +--- + +## Q3. What is new in the approach, and why will it work? + +New is one control point: every container comes from a `Spec`, every optimization hangs off the +Spec rather than off tests, and every start is measured. + +- **One control point.** A Spec is a record that names the service and its variants + (`HiveSpec(HIVE3, KERBEROS, HDFS)`). A test declares one with a fixture annotation + (`@HiveFixture`); code that cannot carry an annotation asks for the same Spec with + `Fixtures.get(spec)`. One `ContainerExtension` turns fixtures into Specs, starts each Spec once + per run, injects the container and closes everything at run end. Two equal Specs mean one + container (Appendix A). +- **Optimizations on the Spec, not on tests.** Readiness is the container class's + `waitStrategy()`; disposable data is its `applyDisposableData()`; per-run names come from one + `RunNames`; dependencies are the Spec's `dependencies()`. A test cannot bypass them, and one + change reaches every test. +- **Measured.** `[CONTAINER-TIMING] container= image= ok= startMs= pullMs= containerMs= + readyMs=` is logged for every start, failed starts included. Every PR carries before-and-after + numbers from one method (Appendix H). + +What happens when a test class asks for a container: + +```mermaid +sequenceDiagram + autonumber + participant T as Test class + participant E as ContainerExtension + participant S as SharedContainers + participant C as HiveContainer + participant D as Docker + T->>E: "@HiveFixture(version = HIVE3)" read in beforeAll + E->>E: fixture -> HiveSpec(HIVE3, NONE, HDFS), then start dependencies first + E->>S: getOrStart(spec) + alt Spec already running (RUN scope) + S-->>E: running container + else first use, or scope = CLASS / METHOD + S->>C: start() with waitStrategy() + C->>D: create, start, wait (port, JMX, WebHDFS probes) + D-->>C: ready + C-->>S: log [CONTAINER-TIMING] + S-->>E: container (failed start: closed at once, test fails) + end + E-->>T: inject as parameter or @FixtureContainer field + Note over E,S: run end: root store closes every container, then Ryuk removes leftovers +``` + +Readiness, before and after. Today each container class polls with Awaitility and a command in +the container (`HiveContainer` on `main`): + +```java +protected boolean checkContainerStatus(int retryLimit) { + await() + .atMost(150, TimeUnit.SECONDS) + .pollInterval(150 / retryLimit, TimeUnit.SECONDS) // waits one interval before try 1 + .until(() -> { + Container.ExecResult r = executeInContainer("bash", "/tmp/check-status.sh"); + ... +``` + +With the stack (#14052, #14086), readiness is a Testcontainers `WaitStrategy` that `start()` +runs, composed of a port check and in-process probes: + +```java +static WaitStrategy readiness(String hadoopUser) { + return new WaitAllStrategy() + .withStrategy(Wait.forListeningPort()) + .withStrategy(liveDataNode(namenodePort)) // NameNode JMX over HTTP + .withStrategy(new ProbeWaitStrategy("WebHDFS to create " + WAREHOUSE_DIR, target -> + httpRequest("PUT", webhdfsUrl + WAREHOUSE_DIR + "?op=MKDIRS&user.name=" + hadoopUser))) +``` + +The framework is dependency injection for test containers: a fixture declares the dependency, +one extension builds it once per key and scope, the test receives it, and the extension owns its +lifetime. It removes the static lookup and hand construction `ContainerSuite` stands for, as +dependency injection does for a service locator. + +Why it will work: + +- It is the pattern Testcontainers recommends for shared containers: a hand-written singleton + keyed by configuration, since `@Testcontainers` shares nothing across classes + ([testcontainers-java#1441](https://github.com/testcontainers/testcontainers-java/issues/1441)). +- It is the pattern #13518 landed in `core` (`BackendTestExtension`, root store, + `CloseableResource`), reviewed and measured on Apache CI. +- The stack is built, and every module in Appendix H passes the same tests with the same counts + on `main` and on the stack locally; CI runs on the draft PRs. +- Each strategy is independent. The lifecycle fixes (#14006) and the JDBC change (#14052) can + merge before the framework. + +--- + +## Q4. Who cares? If successful, what difference does it make? + +Contributors, reviewers, CI and library maintainers each get a concrete change: + +| Who | Difference | +| --- | --- | +| Contributors running ITs | On a local Mac, container-heavy modules finish 32–79 % faster; test-bound ones gain about 6 % (Appendix H); a test declares a container in one line | +| Reviewers of test changes | One pattern (`@Fixture`, `Spec`, `waitStrategy()`) instead of a start method, a poll loop and a close call per service | +| Apache CI | Fewer container starts and no JVM boots inside containers; idp-basic runs in the `build` job on every PR | +| Maintainers of the library | Official modules for six services; readiness is a Testcontainers `WaitStrategy`; Ryuk removes leftovers | +| Developers on one machine | Two runs on one Linux Docker host no longer collide; on macOS the second run fails fast instead of breaking the first | +| Downstream builds | Add a service by writing one Spec and one fixture; the sharing, scope, durability and timing come for free (Appendix J) | + +Q7 lists the dimensions and how each is measured. + +--- + +## Q5. What are the risks? + +Each risk has a mitigation in the stack and a way back. Smaller ones sit with their strategy in +the appendices. + +| Risk | Mitigation | Rollback | +| --- | --- | --- | +| Shared containers leak state between classes | Tests already use unique names (random schemas, catalogs, databases per class); a test that changes the container itself (users, permissions, restarts) asks for `scope = CLASS` or `METHOD` and gets its own | Set `scope = CLASS` on the affected class | +| Nine readiness checks change at once (#14086) | Each timeout is the old port budget plus the old poll budget, and each probe keeps its old check; only the loop moves | Revert #14086 alone; the fixtures do not depend on it | +| Kerberos login from the test JVM touches global UGI and krb5 state | `KerberosHdfs` uses an explicit UGI under `doAs` with the FS cache off and leaves the JVM's Kerberos settings as it found them | Revert #14076; the `hadoop fs` path still exists on the image | +| JUnit 6 and Testcontainers 2 upgrades (S0) touch every test module | Test-only and mechanical; each is one PR that merges and reverts on its own; the unit suite and the Docker ITs pass on them (Appendix L) | Revert the one PR | +| `hive-jdbc` joins the catalog-hive test classpath | Thin jar with exclusions: +28 resolved coordinates after the final exclusion list (Appendix D); test scope only | Keep `beeline` over `docker exec` as the fallback (Appendix I) | +| The proxy-user rule HiveServer2 needs is not in the `gravitino-ci` image yet | An entrypoint wrapper appends the rule before `start.sh` and fails the start loudly if it is missing; the image change is a separate issue, after which the wrapper is deleted | The wrapper is 1 file; without it `hive -e` still works | +| #14090 deletes the Trino Compose files and launch scripts that CI workflows call | `trino_integration_test.sh` keeps its name and runs `TrinoQueryTestTool`, which starts the same `TrinoCluster`; the CI workflow needs no edit for the main path; the cascading stack it removes was disabled in CI | Revert #14090; nothing above it depends on the deletion | +| Ryuk, on by default, may not run on rootless Podman | Turn it off with Testcontainers' own switch, `TESTCONTAINERS_RYUK_DISABLED=true`; CI runs standard Docker | One environment variable | +| Review bandwidth: 25 PRs from one author | One PR ready for review per segment at a time, the rest draft; S2 needs no design approval; S0 is four mechanical PRs; the epic carries the map with own-diff sizes (Q6) | Stop at any segment boundary; nothing below is left half-migrated | +| Local numbers may not hold on CI | Local numbers are labelled local; CI totals stay "to be measured"; disposable data is a local-first win (Appendix E) | None needed; no claim rests on CI numbers | + +--- + +## Q6. How long will it take and what does it cost? + +The work is built: 25 PRs in one linear stack, merged bottom-up in eight segments. S0 is cleanup +that needs no design decision, so it sits below this document and can merge first. Each segment +stands on its own; a later segment never changes a test an earlier one migrated. + +```mermaid +flowchart LR + s0["S0 toolchain
JUnit 6, Testcontainers 2
#14103, #14083-#14085"] --> s1["S1 SPIP
#14005"] + s1 --> s2["S2 quick wins
no framework
#14006, #14052, #14076"] + s2 --> s3["S3 fixture framework
6 parts
#14007-#14012"] + s3 --> s4["S4 disposable data
#14013, #14079"] + s4 --> s5["S5 ContainerSuite gone
per-run names
#14077, #14078"] + s5 --> s6["S6 wait strategies
official modules
#14086, #14087"] + s6 --> s7["S7 compliance
#14088-#14091, #14094"] + gate{{"Appendix A agreed"}} -.-> s3 +``` + +S0 and S2 merge without a design decision; the gate before S3 is the Appendix A agreement. + +| Segment | PRs | What lands | +| ------- | ---------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| S0 | #14103, #14083, #14084, #14085 | Foundations: JUnit 4 code to Jupiter, JUnit 6.1.3, Testcontainers 2.0.5, `BackendTestExtension` on JUnit's store cleanup (Appendix L) | +| S1 | #14005 | This document and the `rumdl` config | +| S2 | #14006, #14052, #14076 | Quick wins without the framework: timing lines, lifecycle fixes, Ryuk on, Hive SQL over JDBC, in-process clients for the remaining CLIs | +| S3 | #14007, #14008, #14009, #14010, #14011, #14012 | The fixture framework in six parts: extension and first fixtures; `TestImages`, Moto, Ranger, Hive; Hive callers, ClickHouse, `scope`; field fixtures, `startup`, MySQL and PostgreSQL fixtures; dependencies and Hive variants; `TrinoCluster` (removes the old Compose scripts) | +| S4 | #14013, #14079 | Disposable data per fixture; Doris and StarRocks recipes | +| S5 | #14077, #14078 | `ContainerSuite` becomes package-private `SharedContainers`; per-run names | +| S6 | #14086, #14087 | Readiness through Testcontainers wait strategies (with the Kafka and Trino probes); official modules for Kafka, LocalStack, ClickHouse and OceanBase | +| S7 | #14088, #14089, #14090, #14091, #14094 | Compliance: tests that built their own containers move onto fixtures; `scope` and durable data for every fixture; Trino through fixtures with in-process waits; no commands run inside containers by the framework, then by any test | + +**Size.** Own change size (one commit per PR, ancestors excluded): about 13,100 lines added and +18,300 removed across the code PRs, 13,800 of the removals being the unused cascading Trino +stack in PR #14090. Every PR's own commit stays under 1,300 added lines (GitHub's PR view also +counts the ancestors); the largest are #14007, #14008 and #14009 (about +1,230 each), the +smallest #14079 (+130). #14077 moves 74 files for a net of about zero lines. + +**Merge plan.** + +1. One linear stack, no lanes. Each PR targets `main` from the fork and shows its ancestors on + GitHub; each body links the PR's own diff with its own line count. +2. Merge bottom-up, one segment at a time. S0 and S2 need no framework approval; S3 onward + waits for agreement on Appendix A (names, attributes, dependencies, Trino cluster, Ryuk). +3. Within a segment, one PR is ready for review at a time; the rest stay draft until the one + below is approved. +4. Later, outside this stack: image changes that retire the runtime proxy-user wrapper and the + Kerberos admin scripts, a CI step that sweeps stale `org.testcontainers=true` containers, and + CI upload of the timing lines. + +**How to review it.** Read Q1–Q7, then Appendix A. The appendices B–G are one strategy each: +idea, where it applies, outcome, the PR that proves it. Each PR body names its segment, the +appendix it proves, and a "review only its own commit" link; mechanical caller migrations are +named as such with a per-module count. + +**Not in scope.** Parallel container start, rewriting what tests cover, services from outside +the test JVM, self-healing containers, and test sharding across runners. + +--- + +## Q7. Mid-term and final exams + +The proposal improves testing on seven dimensions, each with a measure, a value on `main`, a +value with the stack where measured, and a target. Wall times are medians of five rounds of +per-module task wall time on one local Mac; other rows are counts or single measurements +(Appendix H). The main-vs-stack wall times are cumulative through #14012, and #14006 carries +most of the gain. + +| Dimension | Measure | Main | Stack | Target / status | +| ------------------------------ | --------------------------------------------------- | ---------------------------------------------------------------- | --------------------------- | ------------------------------------------------------------------ | +| Test wall time per module | `[TEST-TIMING]` per test task | idp-basic 488 s; hive-metastore-common 394 s; catalog-hive 578 s | 102 s; 175 s; 395 s | Apache CI totals: to be measured | +| Container starts per run | `[CONTAINER-TIMING]` lines per module | hive-metastore-common 6; catalog-hive 6 | 4; 5 | Hive: one per (version, auth, storage) | +| Readiness time per start | `containerMs` + `readyMs` | Hive 64.6 s to ready (first-poll floor 26 s); Hive probe ~8 s | 38.3 s; ~0.5 s | #14086 per-service numbers: to be measured | +| Process spawns into containers | `execInContainer` sites on test paths | 65 | 2 after #14076 (`kadmin.local` in one test); 0 after #14094 | Framework: 0 after #14091; tests: 0 after #14094 | +| Reliability | Leaks fixed; defects on `main` found and fixed | 14 (Appendix M) | 0 known leaks | No leftover container after a killed JVM | +| Isolation | Host-wide fixed names | 3 (`gravitino-ci-network`, `trino-ci-*`, `/tmp/obdata`) | 0 | Two runs on one Linux host pass; macOS fails fast (one routed subnet) | +| Maintainability | Readiness styles; direct `ContainerSuite` call sites | 9; 478 in 117 test files | 1 (`WaitStrategy`); 0 | Official modules for 6 services | + +How much less test task wall time each module needs with the stack through #14012 than on `main`, +local Mac, medians of five rounds, sorted by gain (Results at a glance, Appendix H): + +```mermaid +xychart-beta horizontal + title "Less test wall time than main (local Mac, 5-round medians, %)" + x-axis ["idp-basic", "hms-common", "flink-1.20", "kafka", "catalog-hive", "fileset", "spark-3.5"] + y-axis "% less wall time than main" 0 --> 100 + bar [79, 56, 38, 35, 32, 6, 6] +``` + +**Mid-term exam** (S2 merged): every container start logs a timing line in CI; idp-basic in the +`build` job runs its 221 tests without restarting MySQL or PostgreSQL between tests; no module +spawns a Hive CLI on a test path; a hard-killed test JVM leaves no container behind. + +**Final exam** (S7 merged): no test calls `ContainerSuite`; every container is declared with a +fixture or `Fixtures.get`; readiness is a Testcontainers `WaitStrategy` for every service; the +per-module CI job times are measured on `main` and on the stack and no module is slower; two runs +on one Linux Docker host pass at the same time; the only exceptions left are the image-bound +ones in Appendix D. + +--- + +## Appendix A. Framework design + +The target state of the container library. + +### Goals and non-goals + +| Goals | Non-goals | +| --- | --- | +| **Correct lifecycle**: one container per run per configuration, closed only by the framework | **Parallel container start**: tests start containers one at a time in `@BeforeAll`; removing the global lock alone changes nothing | +| **No fixed waits** | **Rewriting all ITs at once**: one class at a time; the old path works until it has no callers | +| **Official modules first**: custom classes only for Gravitino CI images | **Changing what tests cover** | +| **One image catalogue** with no floating tags | **Services from outside the test JVM**: remote Docker, Kubernetes; mapped ports keep the door open | +| **Reset, not restart**, as in #13518 | **Self-healing containers**: a container that dies mid-run fails the test loudly and is not restarted | +| **Measured**: every start and every test task logs its time | | +| **One way for every container**, including the scripted Trino stack, Moto and the ClickHouse cluster | | + +### Layers + +```mermaid +flowchart TD + test["Test class
@HiveFixture(version = HIVE3, auth = KERBEROS)"] + other["Base class, test utility or tool outside JUnit"] + ext["ContainerExtension (JUnit)"] + get["Fixtures.get(spec)"] + shared["SharedContainers (package-private)"] + subgraph classes["Container classes (integration-test-common)"] + official["Official modules
MySQL, PostgreSQL, Kafka,
ClickHouse, LocalStack, OceanBase"] + generic["GenericContainer + WaitStrategy
Hive, Ranger, Doris, StarRocks, RustFS, Trino"] + end + images["TestImages"] + test -->|"declares what it needs"| ext + other -->|"asks for the same Spec without an annotation"| get + ext -->|"turns each fixture into a Spec, injects the container"| shared + get -->|"same Spec, same container"| shared + shared -->|"one container per Spec, started on first use, closed at run end"| classes + classes -->|"pinned image per service and version"| images +``` + +Every container comes from a Spec. A test class declares it with a fixture annotation; code +that cannot carry one calls `Fixtures.get(spec)`: a base class that picks its backend at run +time (`BaseIT`'s Gravitino backend, idp-basic's backend parameter), a test utility +(`RangerITEnv`) or a tool outside JUnit (`SparkQueryRunner`, `TrinoQueryTestTool`). A test that +builds a container of its own (an old ClickHouse, a TLS Trino) puts it on the run's network with +`Fixtures.network()`. Both paths reach `SharedContainers`: one container per Spec key, the run's +network and closer, start timing and failed-start cleanup. Only Specs and the extension use it. +It replaces `ContainerSuite`, the public singleton tests called directly. + +### Container lifecycle + +```mermaid +stateDiagram-v2 + [*] --> Requested: fixture read or Fixtures.get(spec) + Requested --> Running: Spec not running: start() with waitStrategy() + Requested --> Running: Spec running (RUN scope): reuse + Requested --> Failed: start or readiness fails + Failed --> [*]: closed at once, test fails loudly + Running --> Replaced: another Spec of the same slot requested + Replaced --> [*]: stop, then start the requested Spec + Running --> Closed: end of scope (RUN by the root store, CLASS or METHOD by the class or test store) + Closed --> [*] + Running --> Reaped: JVM killed + Reaped --> [*]: Ryuk removes the session's containers +``` + +### Names + +| Kind | Name | Example | +| ------------------------ | ----------------------- | --------------------------------------- | +| Running container | `Container` | `HiveContainer` | +| What a test asks for | `Spec` record | `HiveSpec(HIVE3, KERBEROS)` | +| Version or flavour | `` enum | `HiveVersion`, `HiveAuth` | +| Test declaration | `@Fixture` | `@HiveFixture(version = HIVE3)` | +| Injected field | `@FixtureContainer` | `@FixtureContainer HiveContainer hive;` | +| Container lifetime | `Scope` enum | `scope = Scope.CLASS` | +| Multi-node injected type | `Cluster` | `ClickHouseCluster`, `TrinoCluster` | +| Pinned images | `TestImages` | `TestImages.HIVE` | +| Lifecycle and injection | `Extension` | `ContainerExtension` | +| Spec without annotation | `Fixtures` | `Fixtures.get(new MySQLSpec(...))` | +| Migration-time key | `ContainerSlot` | plain Hive keyed by version | + +Every Spec implements `ContainerSpec`, which names the container type a test receives and +starts or returns the running container; a service with nothing to choose has an empty Spec +(`KafkaSpec()`). A fixture annotation carries `@ContainerFixture`, which names the factory that +turns it into its Spec. `ContainerExtension` reads every fixture on the test class and its +superclasses; a subclass's fixture replaces a superclass's of the same type. + +`@FixtureContainer` fields are filled from the class's fixtures by type, on JUnit's own extension +points. They exist because `BaseIT` subclasses set up in an overridden `@BeforeAll` that cannot +take parameters. Static fields are filled before `@BeforeAll`; instance fields when the test +instance is created, which comes first for `PER_CLASS`. + +A multi-node service keeps one fixture and one Spec with a node count +(`@ClickHouseFixture(nodes = 3)`, `@TrinoFixture(workers = 2)`). The services it needs, such as +ZooKeeper or Hive, are dependencies with their own Specs. + +### Slots: what may run side by side + +A slot holds at most one running container. Specs of one slot replace each other; Specs in +different slots run side by side. Heavy services whose variants are never needed together share a +slot, so a runner holds one Hive at a time. The reason is memory, not host names: on Linux CI the +containers share no Docker network or alias, and services reach each other by IP. + +| Service | Spec (key) | Sharing | +| --- | --- | --- | +| Hive | version, auth, storage (HDFS/S3/GCS/ABS) | Replace within one slot | +| MySQL, PostgreSQL | version, disposable data | Side by side | +| Doris | version | Side by side | +| Trino | server, metalake, conf, Hive Spec | Replace on change | +| ClickHouse | nodes (1 or 3) | One per node count | +| Kafka, Ranger, StarRocks, OceanBase, ZooKeeper, LocalStack, Moto, RustFS | none | One per run | + +### Fixture attributes + +Every fixture annotation has the same three attributes: + +| Attribute | Values | Default | Meaning | +| --- | --- | --- | --- | +| `scope` | `RUN`, `CLASS`, `METHOD` | `RUN` | How long the container lives and who shares it | +| `startup` | `EAGER`, `LAZY` | `EAGER` | Before `@BeforeAll`, or when first injected | +| `durability` | `DISPOSABLE`, `DURABLE` | `DISPOSABLE` | Skip durability work (tmpfs, no fsync), or not | + +- **Scope.** A shared (`RUN`) container lives for the whole run. A test that changes or breaks + its container (`HadoopUserAuthenticationIT` adds Kerberos principals) declares + `scope = Scope.CLASS`. `CLASS` and `METHOD` start a private container through the same start + path; the JUnit store of the class or test closes it. A private container never replaces the + shared one and is never handed to another class. +- **Failure.** A container that dies mid-run fails the test loudly; the error names the container + and its Spec. No liveness check, no restart. +- **Startup.** Containers start before the class's `@BeforeAll` methods, so start failures show + up in class setup. `@TrinoFixture` defaults to `LAZY`, because Trino needs the Gravitino + server's port, which `BaseIT` starts in its own `@BeforeAll`. +- **Durability.** Recipes in Appendix E. A shared container is disposable for every class, so + `DURABLE` needs `scope = CLASS` or `METHOD`. `-PdurableTestContainers` makes every container + durable, to inspect data after a failure. Only `@HiveFixture` with HDFS storage supports a + durable private container today; #14089 extends it to every fixture that can run a second + instance. +- **Injection.** A fixture annotation on a `@FixtureContainer` field picks that field's + container, so one class can hold a plain and a Kerberos Hive (`GvfsMultipleClusterIT`). The + field or parameter type chooses between injected types (`@ClickHouseFixture(nodes = 3)` gives a + `ClickHouseCluster` or its first `ClickHouseContainer`). A class-level fixture no field or + parameter receives still starts; the extension logs a warning after the class. + +### Dependencies + +A Spec names the containers that must run before it with `dependencies()`. The extension starts +them deepest first and fails clearly on a cycle. Dependencies start as shared run-wide +containers, so they outlive a dependent of any scope. + +| Container | Depends on | +| ------------------------- | ---------- | +| ClickHouse cluster | ZooKeeper | +| Hive with `auth = RANGER` | Ranger | +| Hive with `storage = S3` | LocalStack | +| Ranger | MySQL | + +Ranger keeps its policies in the shared MySQL instead of the MariaDB inside its image, as Apache +Ranger's own Docker setup keeps its database in a separate container. Which database a service +uses is decided in one place, its Spec. + +### Databases: two fixtures + +- **Container fixture:** `@MySQLFixture(version = MYSQL_8_0)` and `@PostgreSQLFixture` provide + the container. +- **Database fixture (planned):** `@DatabaseFixture(name = ..., resetStrategy = ...)` creates one + database in that container and resets it before each test (`DatabaseResetter` from #13518, + moved to `integration-test-common`). +- **Injected:** the container, or a `TestDatabase` with JDBC URL, user, password and data source. + +The core-only `BackendTestExtension` keeps the Gravitino backend slot and its lanes; it gets its +container through `BaseIT`, which calls `Fixtures.get` with the MySQL or PostgreSQL Spec. +Stopgap: `@ClickHouseFixture` requires a `database` attribute, and the Spec creates that database +on the shared container; two Specs that differ only by database share one ClickHouse. The +attribute goes away once the database fixture creates databases. + +### User process: writing a test + +```java +@HiveFixture(version = HiveVersion.HIVE3, auth = HiveAuth.KERBEROS) +public class TestKerberosHiveCatalogIT { + @Test + void testCreateTable(HiveContainer hive) { + String uri = "thrift://" + hive.getHost() + ":" + hive.getMappedPort(HIVE_METASTORE_PORT); + // ... + } +} +``` + +A `BaseIT` subclass (`@KafkaFixture public class OwnerIT extends BaseIT`) declares +`@FixtureContainer private KafkaContainer kafka;` and finds it running inside its overridden +`startIntegrationTest()` `@BeforeAll`. + +1. The fixture annotation names the Spec. +2. The extension returns the running container for that Spec, starting it on first use, and + passes it as a parameter or sets it on a `@FixtureContainer` field. +3. The test reaches the service through `getHost()` and the mapped port. +4. The extension closes every container when the run ends; Ryuk removes any leftovers. + +### Special cases + +| Case | How it fits | +| --- | --- | +| Doris | Starts from a Compose file inside `DorisContainer`, behind its Spec | +| Three-node ClickHouse | Injected as one `ClickHouseCluster` | +| Hive variants | Attributes of `@HiveFixture`, not separate fixtures | +| `MotoGlueCatalogIT` | Started Moto itself; now `@MotoFixture` | +| `TrinoConnectorIT` (disabled) | Needs a single Trino next to the shared Hive; #14090 gives it a single-node `TrinoNodeSpec` | +| `BaseIT.startNecessaryContainers()` | Hooks became fixture annotations on the test classes | +| Scripted Trino stack | `TrinoCluster`, below | + +### Trino cluster + +`@TrinoFixture(workers = 2)` injects a `TrinoCluster` (`trino.coordinatorUri()`), described by +`TrinoSpec(version, workers, hive)`: + +- Its Hive, MySQL and PostgreSQL are configured for Trino and differ from the run's shared ones + (MySQL with `lower_case_table_names=1`, a `trino` user and seed tables; Hive with ACID tables; + PostgreSQL with its own user and seed SQL), so the cluster owns its own copies, built from the + same container classes, and they live and die with it. +- The coordinator and each worker are containers on the shared network, with the same init + script and configuration files as before, mounted from `docker-script/init/trino`. Logs are + copied to `build/trino-ci-container-log` before the cluster stops, so CI keeps uploading them. +- Readiness runs `SHOW CATALOGS LIKE 'gravitino'` on the coordinator; the HDFS check waits for at + least one live data node instead of sleeping 5 seconds. #14090 moves both onto Trino's HTTP + API (`/v1/info`, catalog listing, active workers) and Hive's own probes. +- `trino_integration_test.sh` runs `TrinoQueryTestTool`, which starts the same `TrinoCluster`; + the Compose files and launch scripts are gone, only `docker-script/init` stays. +- Trino's `gravitino` catalog needs the Gravitino server's address at start, and `BaseIT` starts + that server after the extension's `beforeAll`, so `@TrinoFixture` is `LAZY` and the port comes + from the test instance through a small interface `BaseIT` implements. + +### What we reuse, extend and write + +| Part | How | Standard piece | +| --------------------- | ------ | ------------------------------------------------ | +| Run scope | Reuse | Root `ExtensionContext.Store` | +| One container per key | Extend | `Store.getOrComputeIfAbsent(spec, ...)` | +| Dependency order | Extend | `GenericContainer.dependsOn()`, `Startables` | +| Injection | Reuse | `ParameterResolver`, `TestInstancePostProcessor` | +| Close at run end | Reuse | `AutoCloseable` store values (JUnit 6) | +| Readiness | Reuse | Testcontainers `WaitStrategy` and module waits | +| Cleanup on JVM death | Reuse | Ryuk | +| Skip without Docker | Reuse | `ExecutionCondition` + `isDockerAvailable()` | +| Our own images | Write | `GenericContainer` subclasses for `gravitino-ci` | + +We write only the Spec records, the mapping from each fixture annotation to its Spec, and the +dependency edges. Containers are shared within one module's test task, not across modules. A test +class uses either this extension or `@Testcontainers`, never both. + +### Testcontainers and Ryuk + +A Spec says what to start; Testcontainers pulls, creates, starts, maps ports, waits for readiness +and stops. Every container and network carries `org.testcontainers=true` and +`org.testcontainers.sessionId=`; Compose services (Doris) carry +`com.docker.compose.project` instead. On a normal exit the extension closes every container; +Testcontainers' shutdown hook covers any other orderly exit. When the JVM is killed, Ryuk, a +small sidecar per JVM, removes the session's containers and networks about 10 seconds after the +connection drops; it runs unprivileged (`TESTCONTAINERS_RYUK_CONTAINER_PRIVILEGED=false`) and +needs only the Docker socket. It is on by default; contributors on rootless Podman turn it off +with Testcontainers' own switch, `TESTCONTAINERS_RYUK_DISABLED=true`. Later, a CI step removes +`org.testcontainers=true` containers left by earlier sessions before the tests, replacing +`docker container prune -f`, which removes only stopped containers. Reuse across local runs +(`withReuse`) stays out of scope; Testcontainers marks it as not suited for CI. + +### Backward compatibility and migration order + +Only test code changes; no user-facing API, configuration or behavior changes. During the +migration `ContainerSuite`'s `start*` and `get*` methods kept working, so test classes moved one +module at a time while fixtures and direct calls shared the same containers. Once no caller was +left, `ContainerSuite` became the package-private `SharedContainers`, with the same images, +environment, host names, network, one container per key per run and close at run end. Two +test-only behaviors change on purpose: no test can close the shared containers, and containers +skip durability work unless a fixture asks for durable data. + +Migration order (PRs #14007–#14012): Kafka first (no key; proves the extension and the store); +LocalStack, RustFS and Moto; PostgreSQL and MySQL; StarRocks, OceanBase and ClickHouse; ZooKeeper +with the ClickHouse cluster (the first dependency); Doris; Ranger, then Hive (the first shared +slot); Trino (a Spec that holds another Spec); last the Trino cluster from `TrinoITContainers`. + +--- + +## Appendix B. Fewer starts and right-sized scope + +**Idea.** Start a container once per run per configuration, and never let a test close what +other tests share. A test that must have its own container says so in the fixture (`scope`). + +**Where it applies.** + +- `ContainerSuite.close()` resets every static field in a `finally` block. Before, nine fields + (StarRocks, OceanBase, ClickHouse, ZooKeeper, LocalStack, RustFS, S3 Hive, and two more) kept a + stopped container that the next start handed out again. `TestContainerSuite` sets every field + and asserts the reset, so a new field cannot be forgotten. +- Hive is keyed by version and flavour (`ContainerSlot`). Before, `CatalogHive2IT`, `TestHive2HMS` + and the Flink and Spark ITs closed the whole suite to get a different Hive. Six suite-wide + closes are gone. +- idp-basic (`AbstractIdpMetaStorageTest`) closed the suite in `@AfterEach`, so MySQL and + PostgreSQL restarted after every test. It now closes once per run through + `CloseContainerExtension`. +- `cleanDisk()` (`docker container prune`, `df`, `free`) runs only on a real start, not on every + `start*Container` call. +- A container whose start throws is closed at once. Before, a readiness timeout left it running + and the next class started another beside it; under load five Hive containers ran at once. +- `HadoopUserAuthenticationIT`, which adds Kerberos principals, runs on `scope = Scope.CLASS`. + +**Outcome** (local Mac, per-module task wall time, one Gradle invocation per module, one round +per module, A/B on the same base so only this step is measured; the headline table in Appendix H +measures `main` against the whole stack over five rounds): + +| Change | Before | After | Change | +| -------------------------------------------------- | ------ | ----- | ------ | +| idp-basic starts MySQL and PostgreSQL once per run | 423 s | 96 s | −77 % | +| No suite-wide closes, hive-metastore-common | 265 s | 171 s | −36 % | +| No suite-wide closes, catalog-hive | 434 s | 389 s | −10 % | +| No suite-wide closes, flink-1.20 | 355 s | 298 s | −16 % | +| No suite-wide closes, spark-3.5 | 472 s | 482 s | noise | + +Hive starts per run: hive-metastore-common 6 to 4, catalog-hive 6 to 5. Spark gains nothing +because `SparkAuthorizationIT` closed the suite last, so nothing restarted after it. + +**PRs.** #14006 (lifecycle fixes, suite-wide closes, idp-basic, `cleanDisk`), #14008 +(failed-start close), #14009 (`scope`), #14010 (only the run's closer closes). + +--- + +## Appendix C. Readiness without waste + +**Idea.** Check readiness at once, then at a steady interval, with the cheapest check the service +allows, expressed as a Testcontainers `WaitStrategy` so that `start()` includes it (code in Q3). + +**Where it applies.** + +- *First-poll floor.* Every `checkContainerStatus` used Awaitility with + `pollInterval(budget / retries)`, which waits one interval before the first try (measured: + 3,012 ms for an already-true condition at a 3 s interval). Hive paid 10 + 8 + 8 s of floor per + start, Doris 24 s. The PostgreSQL retry used `await().atLeast()` with no condition, which + returns at once. All nine loops now poll with `pollDelay(Duration.ZERO)`. +- *Hive probe.* `check-status.sh` ran `hdfs dfsadmin -report` and `hive -e "show databases"` once + a second, each a JVM boot inside the container, plus `outputHiveStatus()` (two more JVM boots) + on every failed poll, then `hive -e "CREATE TABLE default.employee"` polled once a second. The + probe is now NameNode JMX (`NumLiveDataNodes >= 1`, `FSState == Operational`), a WebHDFS + `MKDIRS /user/hive/warehouse` as the container's Hadoop user, then TCP 9083 and TCP 10000. No + CLI, no new dependency. The `default.employee` table was referenced nowhere else. +- *Kerberos Hive.* JMX is served without SPNEGO, so the same JMX and socket checks apply; WebHDFS + is behind SPNEGO, so one `hdfs dfs -mkdir -p` as the kinit'd `hive` principal creates the + warehouse directory until #14091 moves it onto `KerberosHdfs`. +- *Wait strategies (#14086).* Each container's `waitStrategy()` composes the port or log-line check + with a `ProbeWaitStrategy` (a JDBC query, a Ranger API call, a Kafka `ApiVersions` request, the + NameNode's JMX) under `WaitAllStrategy`. The startup timeout is the old port budget plus the old + poll budget. Hive sets its strategy at `start()`, since it depends on the auth. Doris adds its + backend check to the Compose FE service with `ComposeContainer.waitingFor`. The Trino cluster's + `SHOW CATALOGS` check replaces the polling in `launch.sh` and the 5 s HDFS sleep. Nine + hand-written loops become one pattern. + +**Outcome.** + +| Change | Before | After | Change | +| ---------------------------------------- | ------ | ------ | ---------------- | +| No wait before the first readiness check | 64.6 s | 38.3 s | Hive start −41 % | +| Hive probe over HTTP instead of the CLI | ~8 s | ~0.5 s | per Hive start | +| Wait strategies for nine services | — | — | to be measured | + +The Hive start number is the mean of two rounds of `TestHive2HMS`; the probe number is from the +prototype run before #14052. Ranger's 157 s first start in the verification runs included a 64 s +image pull and about 75 s of the image's own setup, so its polling added almost nothing; the +wait-strategy target for Ranger is one pattern, not time. + +**PRs.** #14006 (`pollDelay(ZERO)`, metastore socket check at 60 s), #14052 (HTTP Hive +probe), #14076 (Kafka socket wait, Trino JDBC worker count, LocalStack `/_localstack/health`), +PR #14086 (`ProbeWaitStrategy`, `waitStrategy()` for Hive, Kerberos Hive, Doris, Trino, Ranger, +StarRocks, OceanBase, RustFS, ClickHouse, Kafka). + +--- + +## Appendix D. In-process clients instead of CLIs + +**Idea.** Talk to a service through the port it exposes, with the client library the test +classpath already has, instead of spawning a process inside the container. A `hive -e` call boots +a CLI JVM with an embedded metastore (seconds per statement); `hadoop fs` boots a JVM (2–4 s); +`awslocal` boots Python (~1 s). A grep over every `src/test` tree found 65 distinct exec sites; +the Spark and Flink connectors, authorization-ranger and the clients had none, because they +already talk to Hive through sessions or the thrift port. + +Before (`CatalogHiveViewIT` on `main`): each statement is a process in the container, and the +assertion reads the exit code. + +```java +private void executeHiveSql(String sql) { + ExecResult result = + containerSuite.getHiveContainer().executeInContainer("hive", "-S", "-e", sql); + Assertions.assertEquals(0, result.getExitCode(), ...); +} +``` + +After (#14052): one JDBC connection per class to the container's HiveServer2; rows come back in +the CLI's tab-joined shape, so the assertions do not change. + +```java +@HiveFixture +public class CatalogHiveViewIT extends BaseIT { + @FixtureContainer private HiveContainer hive; + private HiveServer2Sql hiveSql; // new HiveServer2Sql(hive) in @BeforeAll + + private String describe(String qualifiedObjectName) { + return hiveSql.run("DESCRIBE FORMATTED " + qualifiedObjectName); + } +``` + +**Where it applies.** + +| Group | Sites | Replacement | +| --- | --- | --- | +| Hive SQL via `hive -e` | 24 | `HiveServer2Sql`: `hive-jdbc` 2.3.9 (thin jar) to the container's HiveServer2, one connection per class, results flattened to the CLI's tab-joined shape so assertions survive | +| HDFS via `hadoop fs` / `hdfs` | 26 | `KerberosHdfs`: Hadoop `FileSystem` under an explicit UGI logged in from `/etc/admin.keytab`, FS cache off; `mkdirs`, `chmod`, `chown`, `delete` | +| Readiness probes via exec | 7 | Appendix C | +| DB CLIs (`mysql -e`, `psql -c`) | 9 | JDBC through `db.createConnection`, which the next line of the same class already used; the one `stat -f` exec stays until #14091 (no JVM) | +| LocalStack `awslocal` | 9 | AWS SDK client (`createBucket`, `putBucketAcl`, `listObjectsV2`, IAM `createUser`) that `hadoop-aws` already brings; health over HTTP | +| Kerberos admin (`kadmin.local`) | 2 | #14094: the test declares its principals in `@HiveFixture(kerberosPrincipals = ...)`; the container creates them during its own start, keytabs come out with `docker cp` | +| Log collection (`tar` on close) | 7 | Stays through #14076, once per close; #14091 uses Docker's archive copy | +| Trino CLI (`trino --execute`) | 1 | Trino JDBC, which `TrinoContainer` already opens | + +Two findings: + +- *The anonymous proxy user.* The Hive container runs as `HADOOP_USER_NAME=anonymous`, so + every `docker exec hive -e` wrote warehouse files owned by `anonymous`. HiveServer2 with + `doAs=true` wraps each session in a proxy user, and Hadoop's `ProxyUsers` check fires even when + the session user equals the process user; the image allows only `hive`, `root` and `gravitino` + to proxy. The JDBC client therefore connects as `HiveContainer.getHadoopUser()`, and an + entrypoint wrapper appends `hadoop.proxyuser..{hosts,groups}=*` to `core-site.xml` + before `start.sh`, failing the start loudly if the rule is missing (Q5). The image carries the + rule at its next rebuild; the wrapper is deleted then. +- *A setup that never ran.* `HudiCatalogKerberosHiveIT` ran two `hive -e` statements whose SQL + was wrapped in literal quotes and whose exit codes were ignored; they had never run, and the + test asserts that the table does not exist. The calls are dropped; the in-process path checks + every result (M-10). + +**Outcome** (local Mac, medians of three rounds; the "before" side is the stack below each PR, +not `main`): + +| Module or class | Before | After | Change | +| ------------------------------------ | ------ | ----- | ------ | +| catalog-hive (#14052) | 380 s | 235 s | −38 % | +| `CatalogHiveViewIT` (#14052) | 122 s | 37 s | −70 % | +| hive-metastore-common (#14052) | 164 s | 147 s | −10 % | +| catalog-fileset, Kerberos (#14076) | 186 s | 167 s | −10 % | +| catalog-lakehouse-paimon (#14076) | 67 s | 60 s | −10 % | +| catalog-lakehouse-hudi (#14076) | 50 s | 43 s | −14 % | +| filesystem-hadoop3, GVFS S3 (#14076) | 117 s | 106 s | −9 % | +| catalog-hive, Kerberos (#14076) | 100 s | 92 s | −8 % | + +Iceberg, Iceberg REST, Kafka and Trino are flat under #14076. Dependency cost of `hive-jdbc` in +catalog-hive: 541 to 626 resolved coordinates (+85) with the prototype's first exclusions, almost +all through `hive-service` to `hive-llap-server`; the final exclusion list brings it to +28. Exec +sites on test paths after #14076: two `kadmin.local` calls in one test; after #14094: none, since +the container creates the test's principals during its own start. + +**PRs.** #14052 (Hive SQL over JDBC, HTTP readiness, proxy-user wrapper), #14076 (`KerberosHdfs`, +DB CLIs over JDBC, LocalStack SDK, Hudi fix), #14091 (no exec for readiness and logs in the +framework). Image changes (proxy rule and warehouse directory in `start.sh`, extra Kerberos +principals) are a separate issue, since they need a `gravitino-ci` image release. + +--- + +## Appendix E. Disposable data + +**Idea.** No integration test reads data back after a container stops (a test that restarts a +container asks for its own, durable one), so every container can skip the durability work its +engine does for crash safety: data on tmpfs and no per-commit syncs. #13899 did this for MySQL and +PostgreSQL in the core lanes; the gain is large on a local Mac and small on GitHub Linux CI +(numbers in Q2); this extends it to every engine with a recipe and makes it a fixture attribute, +`durability`, disposable by default (rules in Appendix A). + +**Where it applies.** Each container class says how in `applyDisposableData()`; `logDurability` +reports each database's mode and whether its data directory is on tmpfs. + +| Service | Recipe | +| --- | --- | +| MySQL | tmpfs `/var/lib/mysql`; `innodb_flush_log_at_trx_commit=0`, `skip-log-bin`, `innodb_doublewrite=OFF` (#13899) | +| PostgreSQL | tmpfs `PGDATA`; `fsync=off`, `synchronous_commit=off`, `full_page_writes=off` (#13899) | +| Hive, Kerberos Hive | tmpfs `/tmp/hadoop-root` for HDFS; a `.cnf` in `/etc/mysql/conf.d` for the metastore's MySQL 5.7 (a tmpfs over `/var/lib/mysql` would hide the system tables built into the image) | +| ZooKeeper | `ZOO_CFG_EXTRA=forceSync=no` | +| Ranger | Policies in the shared MySQL (Appendix A, Dependencies) instead of the image's MariaDB, so the MySQL recipe applies and Ranger's schema setup runs on it | +| Doris | A second Compose file: BE storage on tmpfs; FE `master_sync_policy` and `replica_sync_policy` set to `NO_SYNC` | +| StarRocks | FE metadata and BE storage on tmpfs | +| Not applied | OceanBase (uses O_DIRECT; tmpfs would need about 9 GB), ClickHouse and Kafka (no fsync by default), LocalStack and Moto (in memory), Trino (no store), RustFS (low value) | + +One finding: the Testcontainers PostgreSQL module starts the server with `fsync=off` by default, +so every PostgreSQL test on `main`, "durable" or not, already ran without fsync; that is also +where most of the core lanes' PostgreSQL gain came from. A `DURABLE` PostgreSQL now runs +`postgres` with the image's own settings. + +**Outcome.** The benefit is local first: these are local Mac (OrbStack) medians of three rounds, +disposable against durable on the same stack. Expect much less on GitHub Linux CI, where #13899's +same change was flat for MySQL and about 10 % faster for PostgreSQL; CI numbers are to be +measured. + +| Module | Durable | Disposable | Change | +| -------------------- | ------- | ---------- | -------------- | +| idp-basic | 77 s | 34 s | −54 % | +| catalog-jdbc-mysql | 55 s | 43 s | −24 % | +| authorization-ranger | | | −8 % | +| Hive modules | 173 s | 171 s | flat | +| ZooKeeper | | | flat | +| Doris, StarRocks | | | to be measured | + +Hive's recipe applies (verified inside the running containers: HDFS name and data directories on +tmpfs, `innodb_flush_log_at_trx_commit=0`, doublewrite off), so its flat result on this Mac is +real, not a misconfiguration. The Doris recipe looked about 11 % slower under load in one early +run; it stays only if the quiet re-measure shows a gain. + +Memory: tmpfs uses only what is written. Caps are 2 GB per data directory, more only where an +engine refuses to start with less free space (StarRocks FE metadata 6 GB, Doris BE storage 8 GB). + +**PRs.** #14013 (`durability` attribute, Hive, ZooKeeper, MySQL and PostgreSQL recipes, Ranger on +the shared MySQL), #14079 (Doris and StarRocks recipes), #14089 (durable private containers for +every fixture). + +--- + +## Appendix F. Per-run isolation + +**Idea.** Nothing a run creates on the Docker host carries a fixed name, so two runs on one host +(two worktrees, or later two modules under `--parallel`) cannot collide, and a killed run's +containers are removed without a manual prune. + +**Where it applies.** + +- `RunNames` picks one run id per JVM and names every container, network and Doris Compose + project `gravitino---`. Names use lowercase letters, digits and hyphens + only: Docker answers a reverse lookup of a container's IP with its container name, and Hadoop + clients reject a host name with an underscore (a Trino worker with one in its generated name + failed to reach HDFS). +- Host names and network aliases inside the network stay fixed, because the Kerberos Hive derives + its principals from its host name. Tests reach containers by IP or mapped port; no test binds a + fixed host port. +- The OceanBase data directory is a per-run temporary directory instead of `/tmp/obdata`. The + Trino stack's `container_name: trino-ci-*` pins, used by `shutdown.sh` and + `trino_integration_test.sh`, are gone with the Compose files. +- Ryuk is on (Appendix A, Testcontainers and Ryuk). +- Still shared: the macOS network's subnet that `mac-docker-connector` routes (a second run there + fails fast on the subnet check instead of removing the first run's network), the CI-only + `docker container prune -f`, and the Trino log directory, which is per worktree. + +**Outcome.** Fixed host-wide names: 3 on `main`, 0 with the stack. Neutral for CI, where one run +per runner is the rule; a developer win locally. No timing claim. Ryuk check: the test JVM was +killed with `kill -9` once MySQL was up and `docker ps` sampled at 5, 15, 30, 45 and 60 s; with +Ryuk on, MySQL and Ryuk were gone by 15 s; with Ryuk off, MySQL was still running at 60 s. + +**PRs.** #14078 (per-run names), #14012 (Trino names), #14006 (Ryuk on). + +--- + +## Appendix G. Standard building blocks + +**Idea.** Where Testcontainers ships a module for a service, use it: its wait strategy, its +listeners, its JDBC URL and credentials handling. Keep custom `GenericContainer` subclasses only +for the `gravitino-ci` images (Hive, Kerberos Hive, Ranger, Doris, Trino) and services without a +module (StarRocks, RustFS, Moto, ZooKeeper). + +**Where it applies.** + +- MySQL and PostgreSQL run on their modules since #13899; the wrappers add the disposable-data + recipe and the durability report. +- Kafka (`org.testcontainers.kafka`, for the `apache/kafka` image), ClickHouse, LocalStack and + OceanBase run on their Testcontainers 2 modules, built from the `TestImages` name with + `asCompatibleSubstituteFor`, so the `GRAVITINO_CI_*_DOCKER_IMAGE` overrides and mirrors still + work. ClickHouse and LocalStack keep the module's readiness check; Kafka replaces it with the + same log line plus the `ApiVersions` request; OceanBase adds its active-server query to the + module's boot line. Kafka keeps the module's KRaft listeners, and its `run` script still + advertises the container IP on 9092, which tests use to bootstrap. The ClickHouse cluster is + three module containers with the same `remote_servers.xml` and ZooKeeper. +- A `BaseContainer` subclass may pass in a container it built itself, so a module container and a + `GenericContainer` with its own wait strategy go through the same start path, timing and + closer. +- `TestImages` lists every pinned test image in one class; the StarRocks tag is pinned instead of + floating. `ContainerSuite` is gone: every test uses a fixture or `Fixtures.get`. +- LocalStack uses the 2.x module, `org.testcontainers.localstack.LocalStackContainer`. + +**Outcome.** Official modules: 2 services on `main` (after #13899) to 6 with the stack. Readiness +styles: 9 to 1. Image sources: 4 to 1. Direct `ContainerSuite` calls: 478 in 117 test files to 0. +No timing claim; the measured value is that each module's start and readiness time appears in the +`[CONTAINER-TIMING]` line, so a regression in a module upgrade is visible. #14087 timing: to be +measured. + +**PRs.** #14008 (`TestImages`, own-container hook), #14077 (`SharedContainers`), #14087 (Kafka, +LocalStack, ClickHouse, OceanBase modules). + +--- + +## Appendix H. Measurement method and evidence + +**Method.** Local runs on one Apple-silicon Mac (environment below), images already pulled, one +Gradle invocation per module +(`./gradlew :test --rerun -PskipDockerTests=false -DcentralBuildOutputs.disabled=true`), +one Docker run at a time, A and B +alternating per round, at least three rounds (five for the headline table), median and range +reported. A is `main` (`026a66035066`), B is the stack. Times are the build's own `[TEST-TIMING]` +task wall time, which excludes Gradle configuration. Container phases come from +`[CONTAINER-TIMING]`, which only B logs. Apache CI runners, cold image pulls and CI totals are not +measured yet; the local numbers are assumed to scale and will be checked on a draft PR. + +### Environment + +| Item | Value | +| --- | --- | +| Machine | Apple M4 Pro, 14 cores, 48 GB RAM | +| OS | macOS 27.0 | +| Docker | OrbStack 2.2.3, Docker Engine 29.4.0, overlayfs; Docker VM with 14 CPUs and about 31 GB | +| JDK | Amazon Corretto 17.0.19 (arm64) | +| Build | Gradle 8.2 (wrapper), one module per invocation, no `--parallel`, build cache not used for test tasks (`--rerun`) | +| Images | Pulled before the first round; `pullMs` = 0 in every timed start | +| Load | One Docker run at a time; runs where other builds shared the machine were discarded and rerun | +| Dates | Headline table 2026-10-20; per-PR tables 2026-10-19 to 2026-10-21 | + +Not covered: Apache CI runners (Linux x86-64, standard Docker), cold image pulls and network +variance. Local Mac numbers include Docker Desktop-class virtualization overhead (fsync, file +sharing), which is why disposable data gains more locally than on Linux CI. + +Two rules learned the hard way: a timing script stops after two consecutive failures (a Gradle +daemon that outlived a session restart lost network access and failed every run on both sides), +and `--parallel` is not used for the Docker modules (fixed ports in `HadoopUserImpersonationIT` +and `TestHudiHMSBackendOps` clash). + +Per-PR results live in the strategy appendices: #14006 in B, C and F (1 to 2 rounds per step); +PRs #14052 and #14076 in D (3 rounds); #14013 in E (3 rounds). Not measured yet: #14077–#14079, +PR #14086 and #14087. The three baselines for catalog-hive differ because they measure different +steps: 578 s is `main` (five rounds); 434 s is the stack with the first-poll fix but with the +suite-wide closes still present (one round, Appendix B); 380 s is the stack below #14052 (three +rounds, Appendix D). + +### Main against the whole stack through #14012 + +Median of five rounds per side; same tests and pass counts on both sides. This is per-module +local wall time on one Mac, not CI. Most of the gain is #14006 (Appendix B); PRs #14007–#14012 +are time-neutral by design. The chart in Q7 plots the Change column. + +| Module | Tests | Main | Stack | Change | Range main / stack | +| --------------------- | ----- | ----- | ----- | ------ | --------------------- | +| idp-basic | 221 | 488 s | 102 s | −79 % | 479–584 s / 94–148 s | +| hive-metastore-common | 52 | 394 s | 175 s | −56 % | 391–395 s / 173–186 s | +| flink-1.20 | 439 | 426 s | 263 s | −38 % | 423–471 s / 258–269 s | +| catalog-kafka | 25 | 34 s | 22 s | −35 % | 34–37 s / 22–23 s | +| catalog-hive | 308 | 578 s | 395 s | −32 % | 573–667 s / 391–404 s | +| catalog-fileset | 436 | 253 s | 237 s | −6 % | 252–259 s / 234–241 s | +| spark-3.5 | 581 | 482 s | 452 s | −6 % | 479–492 s / 423–471 s | + +Modules that start few containers (fileset, spark) gain little: most of their time is the tests. + +### Container starts per run and phases (stack, one round) + +| Module | Starts | containerMs | readyMs | +| --------------------- | ------ | ----------- | ------- | +| hive-metastore-common | 4 | 76,012 | 49,724 | +| catalog-hive | 6 | 83,842 | 63,179 | +| spark-3.5 | 3 | 25,494 | 11,745 | + +`containerMs` is Testcontainers' own start, wait strategy included; `readyMs` is post-start work +such as clients, passwords and Trino's worker count. `pullMs` was 0 everywhere (images local). +`main` logs no phases. + +### Verification runs + +Single start times seen in the first green runs: Hive 30–56 s, LocalStack 2.7 s, Kafka 3.5 s, +StarRocks 110 s, OceanBase 139 s, Doris 122 s, Ranger 157 s; the last three include an image +pull. Twelve modules ran on the full stack. After the restack onto S0, the Docker ITs at the +top of the stack (#14091) ran for integration-test-common, the core MySQL and PostgreSQL lanes, +JDBC MySQL, PostgreSQL and common, Hive, fileset (Kerberos and S3), GVFS, Paimon, the optimizer, +jobs, Kafka, ClickHouse, OceanBase and the Trino connector, and passed. + +### Defects found by measuring + +Seven of the findings in Appendix M were invisible until a start was timed or a JVM killed on +purpose: the stale `close()` fields (M-1), the PostgreSQL retry (M-2), the timed-out start that +leaked (M-3), Ryuk off (M-4), the Hudi setup that never ran (M-10), "durable" PostgreSQL without +fsync (M-11) and the shared Hive storage container (M-12). The rest came from the audit of the +library. + +### Images + +No image file is needed: every figure in this document is Mermaid, which GitHub renders. If a CI +trend view is added later (per-module job time on `main` against the stack over time), it goes to +`design-docs/testing/containers/images/ci-module-times.png`: one line per module, x-axis the CI +run date, y-axis the job time in minutes, generated from the uploaded `[TEST-TIMING]` lines. + +--- + +## Appendix I. Rejected alternatives + +| Option | Why rejected | +| --- | --- | +| A. Fix bugs only, keep `BaseContainer` as is | Keeps the poll floors, the custom readiness loops and the static lookups | +| B. Make `BaseContainer` extend `GenericContainer` | Our `getContainerIpAddress()` returns the bridge IP; the Testcontainers method of the same name returns the host, so 92 files would silently change meaning. Revisit once tests use host and mapped port | +| C. Official modules inside the existing wrappers | **Chosen.** Tuned wait strategies from the modules; the caller API stays the same; two layers until the wrappers are thinned | +| D. `@Testcontainers` / `@Container` annotations | Containers live per class or per test; nothing is shared across classes or injected; JUnit parallel execution is unsupported. Testcontainers declined a shared-container annotation and recommends a hand-written singleton | +| E. One JUnit extension owns run-wide containers | **Chosen.** Same model as `BackendTestExtension` in #13518; shared per run; injectable | +| F. Ryuk: keep off, turn on unprivileged, or write our own reaper by label | **Unprivileged Ryuk chosen.** Off leaks hard-killed JVMs' containers; Ryuk was turned off in #6480 for a contributor's rootless Docker setup (#6475), not for CI. Our own reaper is more code and misses Compose | +| Global durability switch instead of a fixture attribute | The research preferred one JVM property; the owner chose per-fixture `durability` so a test that needs durable data says so where it declares the container, with a Gradle override for debugging | +| `beeline` via `docker exec` instead of JDBC | Zero new dependencies but still a JVM per statement; kept only as the fallback for Kerberos callers until the keytab login is wired | +| Embedded HiveServer2, HiveRunner, in-process `ql.Driver` | Tests a different Hive than the container's and pulls `hive-exec` into the test JVM | +| Docker `HEALTHCHECK` + `Wait.forHealthcheck()` | Needs an image rebuild and re-runs a JVM probe every few seconds, which is the cost being removed | +| Drop `HADOOP_USER_NAME=anonymous` on the Hive container | Changes HDFS ownership of every warehouse path; fileset and permission tests depend on `anonymous` | +| Lazy start for every fixture | Start failures would surface inside tests instead of class setup; only Trino needs it | +| `@Repeatable` fixtures or a qualifier on `@FixtureContainer` | Still ambiguous by type, or a second way to name a Spec; a fixture on the field is enough | +| Two review lanes (bug fixes and framework in parallel) | One linear stack is simpler to rebase and to explain; S2 still merges before the framework | + +--- + +## Appendix J. Downstream builds + +Projects that build on Gravitino (forks, distributions, private catalogs) reuse the framework +through its extension points instead of copying `ContainerSuite`-style code: + +| Extension point | How a downstream build uses it | +| --------------- | ------------------------------ | +| Spec and fixture | Add a `Spec` and a `@Fixture` for a service only it tests; the extension, sharing, scope, durability and timing apply unchanged | +| `TestImages` | Override an image through the existing environment variables, for example a private mirror | +| `TrinoCluster` | Add extra services and connector jars to the cluster a downstream Trino test needs | +| `Fixtures.get(spec)` | Use from base classes or utilities that cannot carry an annotation | +| Disposable data | Give a new container class its own `applyDisposableData()` recipe | + +What a downstream build changes as each segment lands: + +| Segment | Downstream change | +| ------- | ----------------- | +| S0 | Its own tests move to JUnit 6 and the Testcontainers 2 artifact and package names (Appendix L) | +| S2 | None required; its own tests gain the lifecycle fixes and timing lines | +| S3 | Move its own `ContainerSuite` callers onto fixtures, or keep them until S5 | +| S4 | Nothing; containers become disposable by default, a test that reads data after a restart asks for `DURABLE` | +| S5 | `ContainerSuite` is gone: remaining callers must use fixtures or `Fixtures.get` | +| S6 | Readiness overrides become `waitStrategy()`; custom Kafka, LocalStack, ClickHouse or OceanBase setups move to the official modules | +| S7 | Its own tests that start raw Testcontainers containers move onto fixtures | + +--- + +## Appendix K. Glossary + +Names of code elements are in Appendix A, Names. Terms used in the text: + +- **Spec**: a record naming a service and its variants; the key under which one container is + shared per run. `HiveSpec(HIVE3, KERBEROS, HDFS)`. +- **Fixture**: a `@Fixture` annotation on a test class or field that declares a Spec. +- **Shared container**: a `RUN`-scoped container, one per Spec per run, handed to every class + that declares the Spec. **Private container**: a `CLASS`- or `METHOD`-scoped one, owned by one + class or test. +- **Slot**: a place that holds one running container; Specs of one slot replace each other. +- **Scope**: how long a fixture's container lives: `RUN`, `CLASS` or `METHOD`. +- **Durability**: whether a container keeps crash-safe storage (`DURABLE`) or skips it + (`DISPOSABLE`, the default). +- **Segment**: a group of PRs in the stack (S0–S7) that merges as a unit; S0 is the toolchain + cleanup below this document, S1 is this document. +- **Lane**: in `core`, one Gradle test task per database backend (#13517); not used by this + stack, which is one linear chain. +- **Wait strategy**: Testcontainers' readiness check, run as part of `start()`; + `ProbeWaitStrategy` is our strategy for checks that need a client. **Probe**: one such check. +- **Ryuk**: Testcontainers' sidecar that removes a JVM's containers after its connection drops. +- **Exec site**: a call to `execInContainer` or equivalent that spawns a process in a container. +- **`[CONTAINER-TIMING]`**: the log line per container start with pull, start and readiness time. +- **Own diff**: a PR's single commit, as opposed to the cumulative diff GitHub shows for a stacked + PR. + +--- + +## Appendix L. JUnit 6 and Testcontainers 2 + +**Why now.** The framework keeps containers in JUnit's extension store as `AutoCloseable` values, +which JUnit 6 closes for us, so no extension needs its own cleanup callback. Testcontainers 1.21 +is the last 1.x line; moving first lets every framework PR target the 2.x API once instead of +migrating it later. + +**What changes.** + +| PR | Change | +| --- | --- | +| #14103 | The last two JUnit 4 tests and 42 files using JUnit 4 `Assert` move to Jupiter, on the JUnit 5.8.1 of `main`; 123 `assertThrows` calls get Jupiter's argument order; cross-checked against OpenRewrite's `JUnit4to5Migration` (same result in 42 of 44 files); each module declares the test libraries it uses instead of getting them transitively, and a checkstyle `IllegalImport` rule bans `org.junit.*` outside `jupiter` and `platform` | +| #14083 | JUnit 5.8.1 to 6.1.3: the root build applies `junit-bom` to every test configuration (three module-level pins go away) and puts `junit-platform-launcher` on the test runtime for Gradle 8.2; `CloseableResource` becomes `AutoCloseable` in two extensions; test sources and fixtures of the release-8 modules compile at release 17, main sources unchanged | +| #14084 | Testcontainers 1.21.4 to 2.0.5 ([#13958](https://github.com/apache/gravitino/issues/13958)): artifacts renamed `testcontainers-*`, MySQL and PostgreSQL in their 2.x packages, docker-java 3.4.2 to 3.7.1, test code updated for the commons-lang3 3.18 it brings; the MySQL wrapper keeps its own configuration override because the 2.x module ships no default `my.cnf`. The trino-connector modules are not covered: their tests run in Trino's own `trino-testing` harness, which brings Testcontainers 1.x of its own | +| #14085 | `BackendTestExtension` keeps its backend in the store as an `AutoCloseable`; lanes and row reset unchanged | + +**Why the risk is low.** + +- Test code and test dependencies only; nothing in a Gravitino artifact changes. +- Every test already runs on Java 17, which JUnit 6 requires. +- Each upgrade is one PR that merges on `main` alone and reverts alone. +- The two JUnit 4 tests compiled only through a transitive `junit:junit`. `TestOwnerDetails` + (clients/cli, no vintage engine) never ran and now does; `TestGravitinoCatalogStore` already ran + through the vintage engine `flink-test-utils` brings and now runs on Jupiter (#14103). +- Evidence: `./gradlew test -PskipITs` passes on #14083 (13,785 tests, 0 failures). At the top of + the stack, which carries both upgrades, the Docker ITs of integration-test-common, the core + MySQL and PostgreSQL lanes, JDBC MySQL, PostgreSQL and common, Hive, fileset (Kerberos and S3), + GVFS, Paimon, the optimizer, jobs, Kafka, ClickHouse, OceanBase and the Trino connector pass. + +--- + +## Appendix M. Findings + +Defects that exist on `main` (or in the old container library) and that the stack fixes. Each +entry: what was wrong, how it showed up, its impact, the fix, and the PR. Issues met while +building the stack itself are not listed; they were development fixes, not findings. + +### M-1. `ContainerSuite.close()` left nine fields set + +- **Wrong:** `close()` nulled 11 fields and cleared 2 maps, but not `starRocksContainer`, + `oceanBaseContainer`, `clickHouseContainer`, the two ClickHouse cluster fields, + `zooKeeperContainer`, `gravitinoLocalStackContainer`, `rustFSContainer` and + `hiveContainerWithS3`. `network` and `initialized` also stayed set, and the nulling sat inside + the `try`, so a close failure skipped it. +- **Showed up:** reading `ContainerSuite.java:969-988`; a unit test that sets every field and + calls `close()` fails on `main` (`initialized ==> expected `). +- **Impact:** after any mid-run close, the next `start*` of those services handed out a stopped + container; the shared network was never recreated. +- **Fix:** `close()` runs `closer.close()` in `try` and `resetState()` in `finally`; + `TestContainerSuite` asserts every field is reset, so a new field cannot be forgotten. +- **PR:** [#14006](https://github.com/apache/gravitino/pull/14006). + +### M-2. The PostgreSQL readiness retry never waited + +- **Wrong:** `PostgreSQLContainer.checkContainerStatus` used `await().atLeast(...)` with no + condition, which returns at once (measured: 0 ms). +- **Showed up:** a standalone Awaitility probe (Appendix C) while measuring the first-poll floor. +- **Impact:** the retry gave the server no time between attempts; the budget the code promised + did not exist. +- **Fix:** a real Awaitility loop with a condition and `pollDelay(Duration.ZERO)`; + `TestPostgreSQLContainer` covers it. +- **PR:** [#14006](https://github.com/apache/gravitino/pull/14006). + +### M-3. A start whose readiness check timed out left the container running + +- **Wrong:** `ContainerSuite` registered a container with the closer and started it, but a + readiness timeout threw before the field was set; the container kept running and was never + stored. +- **Showed up:** a timing campaign under load: each failed Hive start left a container behind and + the next class started another; five Hive containers ran at once (Appendix H, E10). +- **Impact:** memory exhaustion and cascading start failures on a loaded runner; `main` has the + same path. +- **Fix:** a container whose `start()` throws is closed at once, even when closing it throws. +- **PR:** [#14008](https://github.com/apache/gravitino/pull/14008). + +### M-4. Containers of a hard-killed JVM were never removed + +- **Wrong:** `TESTCONTAINERS_RYUK_DISABLED=true` in `build.gradle.kts`, set in #6480 for one + contributor's rootless Docker (#6475), so nothing removed a crashed or killed JVM's containers. +- **Showed up:** kill test: `kill -9` on the test JVM once MySQL was up; with Ryuk off, MySQL was + still running at 60 s (Appendix F). +- **Impact:** every aborted local run or cancelled CI job left containers, networks and memory + behind until a manual prune; CI's `docker container prune -f` removes only stopped ones. +- **Fix:** Ryuk on, unprivileged; the same variable stays as the opt-out for rootless Podman. +- **PR:** [#14006](https://github.com/apache/gravitino/pull/14006). + +### M-5. Six test classes closed the whole suite mid-run; idp-basic after every test + +- **Wrong:** `CatalogHive2IT`, `TestHive2HMS`, `FlinkHiveCatalogIT`, `FlinkJdbcMysqlCatalogIT`, + `SparkAuthorizationIT` called `containerSuite.close()` in their teardown; + `AbstractIdpMetaStorageTest` did so in `@AfterEach`. +- **Showed up:** the audit of callers; idp-basic's task time (488 s for 221 tests) and the Hive + start counts per module (6 in hive-metastore-common and catalog-hive). +- **Impact:** every container the later classes needed restarted; idp-basic restarted MySQL and + PostgreSQL after every test. +- **Fix:** the closes are removed; idp-basic closes once per run through + `CloseContainerExtension`; with M-6, the Hive2 classes no longer need the close. +- **PR:** [#14006](https://github.com/apache/gravitino/pull/14006). Measured: idp-basic 423 s to + 96 s; hive-metastore-common 265 s to 171 s; catalog-hive 434 s to 389 s; flink-1.20 355 s to + 298 s (Appendix B). + +### M-6. Hive was not keyed by version + +- **Wrong:** one `hiveContainer` and one `kerberosHiveContainer` field for Hive2 and Hive3, + although CI runs both versions in one `catalog-hive` JVM. +- **Showed up:** the audit: whichever class ran first decided which Hive the second got, which is + why the Hive2 classes closed the whole suite (M-5). +- **Impact:** a hidden order dependency and a full suite restart per version switch. +- **Fix:** `ContainerSlot` keyed by version and flavour; a different key stops only the current + Hive and starts the requested one. #14011 extends the key with auth and storage. +- **PR:** [#14006](https://github.com/apache/gravitino/pull/14006). + +### M-7. `cleanDisk()` ran on every start call, not every start + +- **Wrong:** in CI, `ITUtils.cleanDisk()` (`df -h`, `free -m`, `docker container prune -f`, + `df -h`) ran on every `start*Container` call, including calls that found the container up. +- **Showed up:** the audit (15 `cleanDisk()` call sites, before the "not started" check). +- **Impact:** four shell commands and a Docker prune per call, once per test class that asked + for a container. +- **Fix:** `cleanDisk()` runs only inside the branch that starts a container. +- **PR:** [#14006](https://github.com/apache/gravitino/pull/14006). + +### M-8. Every readiness poll waited one interval before the first try + +- **Wrong:** all nine `checkContainerStatus` loops used `pollInterval(budget / retries)` without + `pollDelay(ZERO)`; Awaitility then waits one interval before the first check. +- **Showed up:** a standalone probe (3,012 ms for an already-true condition at a 3 s interval) + and the Hive start timing. +- **Impact:** a fixed floor per start: Hive about 26 s, Doris 24 s, Kafka 12 s, Trino, StarRocks + and OceanBase 6 s, ClickHouse 5 s per node. +- **Fix:** `pollDelay(Duration.ZERO)` and a short fixed interval in every loop; #14086 later + replaces the loops with wait strategies. +- **PR:** [#14006](https://github.com/apache/gravitino/pull/14006). Measured: Hive start 64.6 s + to 38.3 s (Appendix C). + +### M-9. The Hive readiness probe booted JVMs inside the container every second + +- **Wrong:** `check-status.sh` ran `hdfs dfsadmin -report` and `hive -e "show databases"` per + poll, `outputHiveStatus()` ran two more CLIs on every failed poll, and a + `hive -e "CREATE TABLE default.employee"` poll followed; each is a CLI JVM with an embedded + metastore. +- **Showed up:** `[CONTAINER-TIMING]` `readyMs` for Hive, about 8 s per start after the port + was open. +- **Impact:** seconds per Hive start and CPU stolen from the service that is trying to come up. +- **Fix:** NameNode JMX, a WebHDFS `MKDIRS` as the container's Hadoop user, then the metastore and + HiveServer2 ports; no process in the container. Kerberos Hive follows in #14091. +- **PR:** [#14052](https://github.com/apache/gravitino/pull/14052), + [#14091](https://github.com/apache/gravitino/pull/14091). Measured: the probe went from + about 8 s to about 0.5 s per start (Appendix C). + +### M-10. `HudiCatalogKerberosHiveIT`'s setup statements failed silently + +- **Wrong:** two `hive -e` setup calls wrapped their SQL in literal quotes and ignored the exit + code, so they had never run; the test then asserts that the table does not exist. +- **Showed up:** replacing the exec calls with an in-process client that checks every result. +- **Impact:** dead setup code that looked like coverage; a failed setup step in any test on this + path went unnoticed. +- **Fix:** the two calls are dropped; every remaining SQL path checks its result. +- **PR:** [#14076](https://github.com/apache/gravitino/pull/14076). + +### M-11. "Durable" PostgreSQL already ran with `fsync=off` + +- **Wrong:** the Testcontainers PostgreSQL module starts `postgres -c fsync=off` by default, so + every PostgreSQL test on `main`, durable or not, ran without fsync; most of the core lanes' + PostgreSQL gain in #13899 came from that, not from tmpfs. +- **Showed up:** `logDurability` reporting each database's mode while measuring the disposable + recipe. +- **Impact:** the "durable" baseline was not durable, so the disposable-data measurement for + PostgreSQL was wrong, and a test that expected crash-safe PostgreSQL did not get it. +- **Fix:** a `DURABLE` PostgreSQL runs with the image's own settings; `DISPOSABLE` applies the + recipe explicitly. +- **PR:** [#14013](https://github.com/apache/gravitino/pull/14013). + +### M-12. Hive storage variants shared one container, whichever was asked for first + +- **Wrong:** one `hiveContainerWithS3` field served the S3, GCS and ABS Hive ITs; the first + caller's configuration won. +- **Showed up:** keying Hive by (version, auth, storage) for `@HiveFixture`. +- **Impact:** a storage IT could run against a Hive configured for another storage, depending on + class order. +- **Fix:** `HiveStorage {HDFS, S3, GCS, ABS}` is part of the Hive Spec key; storage Hives share + one slot keyed by storage and version. +- **PR:** [#14011](https://github.com/apache/gravitino/pull/14011). + +### M-13. StarRocks used a floating image tag + +- **Wrong:** `starrocks/allin1-ubuntu:3.3-latest`, hard-coded. +- **Showed up:** the image audit (images from four places, Appendix G). +- **Impact:** an upstream re-tag changes the test's database without a code change; results are + not reproducible. +- **Fix:** pinned to `3.3.22`, the digest `3.3-latest` pointed at on 2026-10-19; `TestImages` + later lists every pinned image in one place. +- **PR:** [#14006](https://github.com/apache/gravitino/pull/14006), + [#14008](https://github.com/apache/gravitino/pull/14008). + +### M-14. Fixed host-wide names allowed one run per Docker host + +- **Wrong:** `gravitino-ci-network`, the Trino stack's `container_name: trino-ci-*` and the + OceanBase data directory `/tmp/obdata` were fixed names on the host. +- **Showed up:** the audit; two worktrees running ITs on one machine collided. +- **Impact:** a second run on the same Docker host removed or reused the first run's network and + containers; `--parallel` across modules is impossible. +- **Fix:** `RunNames` gives every container, network and Compose project a per-run name; the + OceanBase directory is per run; the Trino pins went with the Compose files. +- **PR:** [#14078](https://github.com/apache/gravitino/pull/14078), + [#14012](https://github.com/apache/gravitino/pull/14012). From e5fb9999edb82249dff651a907f2e300365f8843 Mon Sep 17 00:00:00 2001 From: Nevin Zheng Date: Mon, 19 Oct 2026 09:46:39 -0700 Subject: [PATCH 06/14] [#14001] improvement(test): Measure container starts and fix the ContainerSuite lifecycle - Log every container start with pull, start and readiness times; add test log4j configs so the lines show. - Reset all ContainerSuite state in close(); key Hive by version; make the PostgreSQL retry wait; pin StarRocks; drop a dead start method. - Poll readiness with no initial delay; run cleanDisk() only on real starts. - idp-basic starts its containers once per run; Ryuk back on, unprivileged. - Drop the suite-wide closes in Hive2, Flink and Spark ITs. --- build.gradle.kts | 7 +- .../hive/integration/test/CatalogHive2IT.java | 2 - .../src/test/resources/log4j2.properties | 73 ++++++ .../gravitino/hive/client/TestHive2HMS.java | 1 - .../src/test/resources/log4j2.properties | 73 ++++++ .../test/hive/FlinkHiveCatalogIT.java | 7 - .../test/jdbc/FlinkJdbcMysqlCatalogIT.java | 7 - .../src/test/resources/log4j2.properties | 73 ++++++ integration-test-common/build.gradle.kts | 2 + .../test/container/ClickHouseContainer.java | 1 + .../test/container/ContainerSlot.java | 65 +++++ .../test/container/ContainerSuite.java | 234 +++++++++--------- .../test/container/DorisContainer.java | 1 + .../test/container/HiveContainer.java | 62 ++--- .../test/container/KafkaContainer.java | 2 + .../test/container/OceanBaseContainer.java | 1 + .../test/container/RustFSContainer.java | 1 + .../test/container/StarRocksContainer.java | 3 +- .../test/container/TestContainerSlot.java | 64 +++++ .../test/container/TestContainerSuite.java | 138 +++++++++++ .../test/container/TrinoContainer.java | 3 + .../src/test/resources/log4j2.properties | 73 ++++++ .../mapper/AbstractIdpMetaStorageTest.java | 7 +- .../src/test/resources/log4j2.properties | 73 ++++++ .../authorization/SparkAuthorizationIT.java | 1 - 25 files changed, 807 insertions(+), 167 deletions(-) create mode 100644 catalogs/catalog-lakehouse-generic/src/test/resources/log4j2.properties create mode 100644 catalogs/hive-metastore-common/src/test/resources/log4j2.properties create mode 100644 iceberg/iceberg-rest-trino-it/src/test/resources/log4j2.properties create mode 100644 integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/ContainerSlot.java create mode 100644 integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/TestContainerSlot.java create mode 100644 integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/TestContainerSuite.java create mode 100644 maintenance/optimizer/src/test/resources/log4j2.properties create mode 100644 plugins/idp-basic/src/test/resources/log4j2.properties diff --git a/build.gradle.kts b/build.gradle.kts index c722594f85f..f7d9686b474 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -454,9 +454,10 @@ allprojects { param.environment("GRAVITINO_CI_KAFKA_DOCKER_IMAGE", "apache/kafka:3.7.0") param.environment("GRAVITINO_CI_LOCALSTACK_DOCKER_IMAGE", "localstack/localstack:4.14.0") - // Disable Ryuk for integration tests - // Ryuk need privileged mode, if we want to rootless or run non-privileged mode, we need to disable it. - param.environment("TESTCONTAINERS_RYUK_DISABLED", "true") + // Ryuk is on: it removes this JVM's test containers even when the JVM is killed, and needs only + // the Docker socket, not privileged mode. On rootless Podman, turn it off with Testcontainers' + // own switch: export TESTCONTAINERS_RYUK_DISABLED=true (#6475). + param.environment("TESTCONTAINERS_RYUK_CONTAINER_PRIVILEGED", "false") val dockerRunning = project.rootProject.extra["dockerRunning"] as? Boolean ?: false val macDockerConnector = project.rootProject.extra["macDockerConnector"] as? Boolean ?: false diff --git a/catalogs/catalog-hive/src/test/java/org/apache/gravitino/catalog/hive/integration/test/CatalogHive2IT.java b/catalogs/catalog-hive/src/test/java/org/apache/gravitino/catalog/hive/integration/test/CatalogHive2IT.java index cc80bbe190d..3a4b9eafe94 100644 --- a/catalogs/catalog-hive/src/test/java/org/apache/gravitino/catalog/hive/integration/test/CatalogHive2IT.java +++ b/catalogs/catalog-hive/src/test/java/org/apache/gravitino/catalog/hive/integration/test/CatalogHive2IT.java @@ -265,8 +265,6 @@ public void stop() throws IOException { sparkSession.close(); } - ContainerSuite.getInstance().close(); - if (fileSystem != null) { fileSystem.close(); } diff --git a/catalogs/catalog-lakehouse-generic/src/test/resources/log4j2.properties b/catalogs/catalog-lakehouse-generic/src/test/resources/log4j2.properties new file mode 100644 index 00000000000..66a51bbdf49 --- /dev/null +++ b/catalogs/catalog-lakehouse-generic/src/test/resources/log4j2.properties @@ -0,0 +1,73 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +# Set to debug or trace if log4j initialization is failing +status = info + +# Name of the configuration +name = ConsoleLogConfig + +# Console appender configuration +appender.console.type = Console +appender.console.name = consoleLogger +appender.console.layout.type = PatternLayout +appender.console.layout.pattern = %d{yyyy-MM-dd HH:mm:ss} %-5p [%t] %c{1}:%L - %m%n + +# Log files location +property.logPath = ${sys:gravitino.log.path:-build/catalog-lakehouse-generic-integration-test.log} + +# File appender configuration +appender.file.type = File +appender.file.name = fileLogger +appender.file.fileName = ${logPath} +appender.file.layout.type = PatternLayout +appender.file.layout.pattern = %d{yyyy-MM-dd HH:mm:ss.SSS} [%t] %-5p %c - %m%n + +# Root logger level +rootLogger.level = info + +# Root logger referring to console and file appenders +rootLogger.appenderRef.stdout.ref = consoleLogger +rootLogger.appenderRef.file.ref = fileLogger + +# File appender configuration for testcontainers +appender.testcontainersFile.type = File +appender.testcontainersFile.name = testcontainersLogger +appender.testcontainersFile.fileName = build/testcontainers.log +appender.testcontainersFile.layout.type = PatternLayout +appender.testcontainersFile.layout.pattern = %d{yyyy-MM-dd HH:mm:ss.SSS} [%t] %-5p %c - %m%n + +# Logger for testcontainers +logger.testcontainers.name = org.testcontainers +logger.testcontainers.level = debug +logger.testcontainers.additivity = false +logger.testcontainers.appenderRef.file.ref = testcontainersLogger + +logger.tc.name = tc +logger.tc.level = debug +logger.tc.additivity = false +logger.tc.appenderRef.file.ref = testcontainersLogger + +logger.docker.name = com.github.dockerjava +logger.docker.level = warn +logger.docker.additivity = false +logger.docker.appenderRef.file.ref = testcontainersLogger + +logger.http.name = com.github.dockerjava.zerodep.shaded.org.apache.hc.client5.http.wire +logger.http.level = off diff --git a/catalogs/hive-metastore-common/src/test/java/org/apache/gravitino/hive/client/TestHive2HMS.java b/catalogs/hive-metastore-common/src/test/java/org/apache/gravitino/hive/client/TestHive2HMS.java index 418ccbfe6d5..b803b940728 100644 --- a/catalogs/hive-metastore-common/src/test/java/org/apache/gravitino/hive/client/TestHive2HMS.java +++ b/catalogs/hive-metastore-common/src/test/java/org/apache/gravitino/hive/client/TestHive2HMS.java @@ -93,7 +93,6 @@ public void startHiveContainer() { @AfterAll public void stopHiveContainer() throws Exception { - containerSuite.close(); if (hiveClient != null) { hiveClient.close(); hiveClient = null; diff --git a/catalogs/hive-metastore-common/src/test/resources/log4j2.properties b/catalogs/hive-metastore-common/src/test/resources/log4j2.properties new file mode 100644 index 00000000000..26ca6a34182 --- /dev/null +++ b/catalogs/hive-metastore-common/src/test/resources/log4j2.properties @@ -0,0 +1,73 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +# Set to debug or trace if log4j initialization is failing +status = info + +# Name of the configuration +name = ConsoleLogConfig + +# Console appender configuration +appender.console.type = Console +appender.console.name = consoleLogger +appender.console.layout.type = PatternLayout +appender.console.layout.pattern = %d{yyyy-MM-dd HH:mm:ss} %-5p [%t] %c{1}:%L - %m%n + +# Log files location +property.logPath = ${sys:gravitino.log.path:-build/hive-metastore-common-integration-test.log} + +# File appender configuration +appender.file.type = File +appender.file.name = fileLogger +appender.file.fileName = ${logPath} +appender.file.layout.type = PatternLayout +appender.file.layout.pattern = %d{yyyy-MM-dd HH:mm:ss.SSS} [%t] %-5p %c - %m%n + +# Root logger level +rootLogger.level = info + +# Root logger referring to console and file appenders +rootLogger.appenderRef.stdout.ref = consoleLogger +rootLogger.appenderRef.file.ref = fileLogger + +# File appender configuration for testcontainers +appender.testcontainersFile.type = File +appender.testcontainersFile.name = testcontainersLogger +appender.testcontainersFile.fileName = build/testcontainers.log +appender.testcontainersFile.layout.type = PatternLayout +appender.testcontainersFile.layout.pattern = %d{yyyy-MM-dd HH:mm:ss.SSS} [%t] %-5p %c - %m%n + +# Logger for testcontainers +logger.testcontainers.name = org.testcontainers +logger.testcontainers.level = debug +logger.testcontainers.additivity = false +logger.testcontainers.appenderRef.file.ref = testcontainersLogger + +logger.tc.name = tc +logger.tc.level = debug +logger.tc.additivity = false +logger.tc.appenderRef.file.ref = testcontainersLogger + +logger.docker.name = com.github.dockerjava +logger.docker.level = warn +logger.docker.additivity = false +logger.docker.appenderRef.file.ref = testcontainersLogger + +logger.http.name = com.github.dockerjava.zerodep.shaded.org.apache.hc.client5.http.wire +logger.http.level = off diff --git a/flink-connector/flink-common/src/test/java/org/apache/gravitino/flink/connector/integration/test/hive/FlinkHiveCatalogIT.java b/flink-connector/flink-common/src/test/java/org/apache/gravitino/flink/connector/integration/test/hive/FlinkHiveCatalogIT.java index 3808e7506e5..eb06d80a850 100644 --- a/flink-connector/flink-common/src/test/java/org/apache/gravitino/flink/connector/integration/test/hive/FlinkHiveCatalogIT.java +++ b/flink-connector/flink-common/src/test/java/org/apache/gravitino/flink/connector/integration/test/hive/FlinkHiveCatalogIT.java @@ -143,13 +143,6 @@ protected void initCatalogEnv() throws Exception { .getDriverClassName(TestDatabaseName.FLINK_HIVE_CATALOG_IT); } - @Override - protected void stopCatalogEnv() throws Exception { - if (containerSuite != null) { - containerSuite.close(); - } - } - protected void initDefaultHiveCatalog() { Preconditions.checkArgument(metalake != null, "metalake should not be null"); hiveCatalog = diff --git a/flink-connector/flink-common/src/test/java/org/apache/gravitino/flink/connector/integration/test/jdbc/FlinkJdbcMysqlCatalogIT.java b/flink-connector/flink-common/src/test/java/org/apache/gravitino/flink/connector/integration/test/jdbc/FlinkJdbcMysqlCatalogIT.java index 77f0a4ea4bf..360a7cc69ba 100644 --- a/flink-connector/flink-common/src/test/java/org/apache/gravitino/flink/connector/integration/test/jdbc/FlinkJdbcMysqlCatalogIT.java +++ b/flink-connector/flink-common/src/test/java/org/apache/gravitino/flink/connector/integration/test/jdbc/FlinkJdbcMysqlCatalogIT.java @@ -132,13 +132,6 @@ protected void initCatalogEnv() throws Exception { mysqlDriver = containerSuite.getMySQLContainer().getDriverClassName(MYSQL_CATALOG_MYSQL_IT); } - @Override - protected void stopCatalogEnv() throws Exception { - if (null != containerSuite) { - containerSuite.close(); - } - } - @Test public void testCreateGravitinoJdbcCatalogUsingSQL() { tableEnv.useCatalog(DEFAULT_CATALOG); diff --git a/iceberg/iceberg-rest-trino-it/src/test/resources/log4j2.properties b/iceberg/iceberg-rest-trino-it/src/test/resources/log4j2.properties new file mode 100644 index 00000000000..01e5df4d53f --- /dev/null +++ b/iceberg/iceberg-rest-trino-it/src/test/resources/log4j2.properties @@ -0,0 +1,73 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +# Set to debug or trace if log4j initialization is failing +status = info + +# Name of the configuration +name = ConsoleLogConfig + +# Console appender configuration +appender.console.type = Console +appender.console.name = consoleLogger +appender.console.layout.type = PatternLayout +appender.console.layout.pattern = %d{yyyy-MM-dd HH:mm:ss} %-5p [%t] %c{1}:%L - %m%n + +# Log files location +property.logPath = ${sys:gravitino.log.path:-build/iceberg-rest-trino-it-integration-test.log} + +# File appender configuration +appender.file.type = File +appender.file.name = fileLogger +appender.file.fileName = ${logPath} +appender.file.layout.type = PatternLayout +appender.file.layout.pattern = %d{yyyy-MM-dd HH:mm:ss.SSS} [%t] %-5p %c - %m%n + +# Root logger level +rootLogger.level = info + +# Root logger referring to console and file appenders +rootLogger.appenderRef.stdout.ref = consoleLogger +rootLogger.appenderRef.file.ref = fileLogger + +# File appender configuration for testcontainers +appender.testcontainersFile.type = File +appender.testcontainersFile.name = testcontainersLogger +appender.testcontainersFile.fileName = build/testcontainers.log +appender.testcontainersFile.layout.type = PatternLayout +appender.testcontainersFile.layout.pattern = %d{yyyy-MM-dd HH:mm:ss.SSS} [%t] %-5p %c - %m%n + +# Logger for testcontainers +logger.testcontainers.name = org.testcontainers +logger.testcontainers.level = debug +logger.testcontainers.additivity = false +logger.testcontainers.appenderRef.file.ref = testcontainersLogger + +logger.tc.name = tc +logger.tc.level = debug +logger.tc.additivity = false +logger.tc.appenderRef.file.ref = testcontainersLogger + +logger.docker.name = com.github.dockerjava +logger.docker.level = warn +logger.docker.additivity = false +logger.docker.appenderRef.file.ref = testcontainersLogger + +logger.http.name = com.github.dockerjava.zerodep.shaded.org.apache.hc.client5.http.wire +logger.http.level = off diff --git a/integration-test-common/build.gradle.kts b/integration-test-common/build.gradle.kts index e20e6c7e1f2..47ef9625f95 100644 --- a/integration-test-common/build.gradle.kts +++ b/integration-test-common/build.gradle.kts @@ -88,6 +88,8 @@ dependencies { // every module that uses the test artifacts needs them at test runtime. "testArtifacts"(libs.testcontainers.mysql) "testArtifacts"(libs.testcontainers.postgresql) + // The container classes wait with Awaitility, which Gravitino's code style imports unshaded. + "testArtifacts"(libs.awaitility) } artifacts { diff --git a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/ClickHouseContainer.java b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/ClickHouseContainer.java index b24196ee4a9..72f2fd355cb 100644 --- a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/ClickHouseContainer.java +++ b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/ClickHouseContainer.java @@ -82,6 +82,7 @@ public void start() { protected boolean checkContainerStatus(int retryLimit) { Awaitility.await() .atMost(java.time.Duration.ofMinutes(2)) + .pollDelay(Duration.ZERO) .pollInterval(Duration.ofSeconds(5)) .until( () -> { diff --git a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/ContainerSlot.java b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/ContainerSlot.java new file mode 100644 index 00000000000..b70c415a396 --- /dev/null +++ b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/ContainerSlot.java @@ -0,0 +1,65 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.integration.test.container; + +import java.util.Objects; +import java.util.function.Supplier; +import javax.annotation.Nullable; + +/** + * Holds at most one running container, together with the key it was started for. + * + *

A request with the running key returns the running container. A request with another key stops + * the running one first, so a slot holds one container at a time: its keys are variants of one + * heavy service, such as Hive versions, that tests never need together and that would not fit side + * by side on a CI runner. + * + * @param what makes two requests different, such as a Hive version + * @param the container type + */ +final class ContainerSlot { + @Nullable private volatile K key; + @Nullable private volatile C container; + + synchronized C getOrStart(K requestedKey, Supplier starter) { + if (container != null && Objects.equals(key, requestedKey)) { + return container; + } + if (container != null) { + BaseContainer.LOG.info( + "Replacing {} {} with {}", container.getClass().getSimpleName(), key, requestedKey); + container.close(); + clear(); + } + C started = starter.get(); + container = started; + key = requestedKey; + return started; + } + + @Nullable + C get() { + return container; + } + + synchronized void clear() { + container = null; + key = null; + } +} diff --git a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/ContainerSuite.java b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/ContainerSuite.java index f182acfb291..1b152878ff2 100644 --- a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/ContainerSuite.java +++ b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/ContainerSuite.java @@ -73,7 +73,8 @@ public class ContainerSuite implements Closeable { private static final String CLICKHOUSE_TEST_IMAGE_ENV = "GRAVITINO_CLICKHOUSE_TEST_IMAGE"; private static Network network = null; - private static volatile HiveContainer hiveContainer; + // Keyed by Hive version: Hive2 and Hive3 tests share a JVM, and each gets its own version. + private static final ContainerSlot hiveContainer = new ContainerSlot<>(); // Enable the Ranger plugin in the Hive container private static volatile HiveContainer hiveRangerContainer; @@ -82,8 +83,8 @@ public class ContainerSuite implements Closeable { private static volatile RangerContainer rangerContainer; private static volatile KafkaContainer kafkaContainer; private static volatile DorisContainer dorisContainer; - private static volatile HiveContainer kerberosHiveContainer; - private static volatile HiveContainer sqlBaseHiveContainer; + private static final ContainerSlot kerberosHiveContainer = + new ContainerSlot<>(); private static volatile StarRocksContainer starRocksContainer; private static volatile MySQLContainer mySQLContainer; private static volatile MySQLContainer mySQLVersion5Container; @@ -153,22 +154,20 @@ public void startHiveContainer(Map env) { builder.putAll(env); builder.put("HADOOP_USER_NAME", "anonymous"); - ITUtils.cleanDisk(); - if (hiveContainer == null) { - synchronized (ContainerSuite.class) { - if (hiveContainer == null) { - initIfNecessary(); - // Start Hive container - HiveContainer.Builder hiveBuilder = - HiveContainer.builder() - .withHostName("gravitino-ci-hive") - .withEnvVars(builder.build()) - .withNetwork(network); - HiveContainer container = closer.register(hiveBuilder.build()); - container.start(); - hiveContainer = container; - } - } + synchronized (ContainerSuite.class) { + hiveContainer.getOrStart( + hiveVersion(env), + () -> { + initIfNecessary(); + HiveContainer.Builder hiveBuilder = + HiveContainer.builder() + .withHostName("gravitino-ci-hive") + .withEnvVars(builder.build()) + .withNetwork(network); + HiveContainer container = closer.register(hiveBuilder.build()); + start(container); + return container; + }); } } @@ -176,7 +175,6 @@ public void startHiveContainerWithS3(Map env) { ImmutableMap.Builder builder = ImmutableMap.builder(); builder.putAll(env); builder.put("HADOOP_USER_NAME", "anonymous"); - ITUtils.cleanDisk(); if (hiveContainerWithS3 == null) { synchronized (ContainerSuite.class) { if (hiveContainerWithS3 == null) { @@ -188,7 +186,7 @@ public void startHiveContainerWithS3(Map env) { .withEnvVars(builder.build()) .withNetwork(network); HiveContainer container = closer.register(hiveBuilder.build()); - container.start(); + start(container); hiveContainerWithS3 = container; } } @@ -221,7 +219,6 @@ public void startHiveRangerContainer(Map envVars) { throw new IllegalArgumentException("Error environment variables for Hive Ranger container"); } - ITUtils.cleanDisk(); if (hiveRangerContainer == null) { synchronized (ContainerSuite.class) { if (hiveRangerContainer == null) { @@ -232,7 +229,7 @@ public void startHiveRangerContainer(Map envVars) { .withEnvVars(envVars) .withNetwork(network); HiveContainer container = closer.register(hiveBuilder.build()); - container.start(); + start(container); hiveRangerContainer = container; } } @@ -240,23 +237,21 @@ public void startHiveRangerContainer(Map envVars) { } public void startKerberosHiveContainer(Map envVars) { - ITUtils.cleanDisk(); - if (kerberosHiveContainer == null) { - synchronized (ContainerSuite.class) { - if (kerberosHiveContainer == null) { - initIfNecessary(); - // Start Hive container - HiveContainer.Builder hiveBuilder = - HiveContainer.builder() - .withHostName("gravitino-ci-kerberos-hive") - .withKerberosEnabled(true) - .withEnvVars(envVars) - .withNetwork(network); - HiveContainer container = closer.register(hiveBuilder.build()); - container.start(); - kerberosHiveContainer = container; - } - } + synchronized (ContainerSuite.class) { + kerberosHiveContainer.getOrStart( + hiveVersion(envVars), + () -> { + initIfNecessary(); + HiveContainer.Builder hiveBuilder = + HiveContainer.builder() + .withHostName("gravitino-ci-kerberos-hive") + .withKerberosEnabled(true) + .withEnvVars(envVars) + .withNetwork(network); + HiveContainer container = closer.register(hiveBuilder.build()); + start(container); + return container; + }); } } @@ -264,34 +259,6 @@ public void startKerberosHiveContainer() { startKerberosHiveContainer(ImmutableMap.of()); } - public void startSQLBaseAuthHiveContainer(Map envVars) { - // If you want to enable SQL based authorization, you need both set the - // `ENABLE_SQL_BASE_AUTHORIZATION` environment. - if (envVars == null - || (!Objects.equals(envVars.get(HiveContainer.HIVE_RUNTIME_VERSION), HiveContainer.HIVE3)) - || (!envVars.containsKey(HiveContainer.ENABLE_SQL_BASE_AUTHORIZATION))) { - throw new IllegalArgumentException( - "Error environment variables for Hive SQL base authorization container"); - } - - if (sqlBaseHiveContainer == null) { - synchronized (ContainerSuite.class) { - if (sqlBaseHiveContainer == null) { - initIfNecessary(); - // Start Hive container - HiveContainer.Builder hiveBuilder = - HiveContainer.builder() - .withHostName("gravitino-ci-hive") - .withNetwork(network) - .withEnvVars(envVars); - HiveContainer container = closer.register(hiveBuilder.build()); - container.start(); - sqlBaseHiveContainer = container; - } - } - } - } - /** * Files copied into the Gravitino plugin directory of the Trino container: the connector lib * directory, plus every jar in the directory named by {@code @@ -318,13 +285,12 @@ public void startTrinoContainer( String trinoConnectorLibDir, int gravitinoServerPort, String metalakeName) { - ITUtils.cleanDisk(); if (trinoContainer == null) { synchronized (ContainerSuite.class) { if (trinoContainer == null) { initIfNecessary(); // Start Trino container - String hiveContainerIp = hiveContainer.getContainerIpAddress(); + String hiveContainerIp = hiveContainer.get().getContainerIpAddress(); TrinoContainer.Builder trinoBuilder = TrinoContainer.builder() .withEnvVars( @@ -347,7 +313,7 @@ public void startTrinoContainer( .withHiveContainerIP(hiveContainerIp); TrinoContainer container = closer.register(trinoBuilder.build()); - container.start(); + start(container); trinoContainer = container; } } @@ -359,7 +325,6 @@ public void startDorisContainer() { } public void startDorisContainer(DorisImageName imageName) { - ITUtils.cleanDisk(); if (!dorisContainerMap.containsKey(imageName)) { synchronized (ContainerSuite.class) { if (!dorisContainerMap.containsKey(imageName)) { @@ -371,7 +336,7 @@ public void startDorisContainer(DorisImageName imageName) { .withImage(imageName.toString()) .withNetwork(network); DorisContainer container = closer.register(dorisBuilder.build()); - container.start(); + start(container); dorisContainerMap.put(imageName, container); // Keep backward-compatible reference for getDorisContainer() if (imageName == DorisImageName.VERSION_1_2) { @@ -386,7 +351,6 @@ public void startMySQLContainer(TestDatabaseName testDatabaseName) { if (mySQLContainer == null) { synchronized (ContainerSuite.class) { if (mySQLContainer == null) { - ITUtils.cleanDisk(); initIfNecessary(); // Start MySQL container MySQLContainer.Builder mysqlBuilder = @@ -397,7 +361,7 @@ public void startMySQLContainer(TestDatabaseName testDatabaseName) { .withNetwork(network); MySQLContainer container = closer.register(mysqlBuilder.build()); - container.start(); + start(container); mySQLContainer = container; } } @@ -408,7 +372,6 @@ public void startMySQLContainer(TestDatabaseName testDatabaseName) { } public void startMySQLVersion5Container(TestDatabaseName testDatabaseName) { - ITUtils.cleanDisk(); if (mySQLVersion5Container == null) { synchronized (ContainerSuite.class) { if (mySQLVersion5Container == null) { @@ -423,7 +386,7 @@ public void startMySQLVersion5Container(TestDatabaseName testDatabaseName) { .withNetwork(network); MySQLContainer container = closer.register(mysqlBuilder.build()); - container.start(); + start(container); mySQLVersion5Container = container; } } @@ -437,7 +400,6 @@ public void startPostgreSQLContainer(TestDatabaseName testDatabaseName, PGImageN if (!pgContainerMap.containsKey(pgImageName)) { synchronized (ContainerSuite.class) { if (!pgContainerMap.containsKey(pgImageName)) { - ITUtils.cleanDisk(); initIfNecessary(); // Start PostgreSQL container PostgreSQLContainer.Builder pgBuilder = @@ -449,7 +411,7 @@ public void startPostgreSQLContainer(TestDatabaseName testDatabaseName, PGImageN .withNetwork(network); PostgreSQLContainer container = closer.register(pgBuilder.build()); - container.start(); + start(container); pgContainerMap.put(pgImageName, container); } } @@ -465,7 +427,6 @@ public void startPostgreSQLContainer(TestDatabaseName testDatabaseName) { } public void startOceanBaseContainer() { - ITUtils.cleanDisk(); // create dir /tmp/obdata by CommandExecutor Object o = CommandExecutor.executeCommandLocalHost( @@ -498,7 +459,7 @@ public void startOceanBaseContainer() { ImmutableMap.builder().put("/tmp/obdata", "/root/ob").build()) .withExposePorts(ImmutableSet.of(OceanBaseContainer.OCEANBASE_PORT)); OceanBaseContainer container = closer.register(oceanBaseBuilder.build()); - container.start(); + start(container); oceanBaseContainer = container; } } @@ -506,7 +467,6 @@ public void startOceanBaseContainer() { } public void startKafkaContainer() { - ITUtils.cleanDisk(); if (kafkaContainer == null) { synchronized (ContainerSuite.class) { if (kafkaContainer == null) { @@ -514,7 +474,7 @@ public void startKafkaContainer() { KafkaContainer.Builder builder = KafkaContainer.builder().withNetwork(network); KafkaContainer container = closer.register(builder.build()); try { - container.start(); + start(container); } catch (Exception e) { LOG.error("Failed to start Kafka container", e); throw new RuntimeException("Failed to start Kafka container", e); @@ -526,7 +486,6 @@ public void startKafkaContainer() { } public void startLocalStackContainer() { - ITUtils.cleanDisk(); if (gravitinoLocalStackContainer == null) { synchronized (ContainerSuite.class) { if (gravitinoLocalStackContainer == null) { @@ -534,7 +493,7 @@ public void startLocalStackContainer() { GravitinoLocalStackContainer.builder().withNetwork(network); GravitinoLocalStackContainer container = closer.register(builder.build()); try { - container.start(); + start(container); } catch (Exception e) { LOG.error("Failed to start LocalStack container", e); throw new RuntimeException("Failed to start LocalStack container", e); @@ -546,7 +505,6 @@ public void startLocalStackContainer() { } public void startStarRocksContainer() { - ITUtils.cleanDisk(); if (starRocksContainer == null) { synchronized (ContainerSuite.class) { if (starRocksContainer == null) { @@ -556,7 +514,7 @@ public void startStarRocksContainer() { StarRocksContainer.builder().withNetwork(network); StarRocksContainer container = closer.register(starRocksBuilder.build()); try { - container.start(); + start(container); } catch (Exception e) { LOG.error("Failed to start StarRocks container", e); throw new RuntimeException("Failed to start StarRocks container", e); @@ -568,7 +526,6 @@ public void startStarRocksContainer() { } public void startClickHouseContainer(TestDatabaseName testDatabaseName) { - ITUtils.cleanDisk(); if (clickHouseContainer == null) { synchronized (ContainerSuite.class) { if (clickHouseContainer == null) { @@ -587,7 +544,7 @@ public void startClickHouseContainer(TestDatabaseName testDatabaseName) { .withNetwork(network); ClickHouseContainer container = closer.register(clickHouseBuilder.build()); - container.start(); + start(container); clickHouseContainer = container; } } @@ -625,7 +582,7 @@ public void startClickHouseClusterContainer( .withNetwork(network); ClickHouseContainer container = closer.register(clickHouseBuilder.build()); - container.start(); + start(container); containers.add(container); } clickHouseClusterContainers = List.copyOf(containers); @@ -677,7 +634,7 @@ public void startZooKeeperContainer() { .withExposePorts(ImmutableSet.of(ZooKeeperContainer.ZK_PORT)) .withNetwork(network); ZooKeeperContainer container = closer.register(zkBuilder.build()); - container.start(); + start(container); zooKeeperContainer = container; } } @@ -712,14 +669,13 @@ public GravitinoLocalStackContainer getLocalStackContainer() { /** Starts the shared RustFS object store for S3 credential-vending tests. */ public void startRustFSContainer() { - ITUtils.cleanDisk(); if (rustFSContainer == null) { synchronized (ContainerSuite.class) { if (rustFSContainer == null) { RustFSContainer.Builder builder = RustFSContainer.builder().withNetwork(network); RustFSContainer container = closer.register(builder.build()); try { - container.start(); + start(container); } catch (Exception e) { LOG.error("Failed to start RustFS container", e); throw new RuntimeException("Failed to start RustFS container", e); @@ -760,7 +716,7 @@ public static TrinoITContainers getTrinoITContainers() { } public HiveContainer getHiveContainer() { - return hiveContainer; + return hiveContainer.get(); } public HiveContainer getHiveRangerContainer() { @@ -776,7 +732,7 @@ public void startRangerContainer() { RangerContainer.Builder rangerBuilder = RangerContainer.builder().withNetwork(network); RangerContainer container = closer.register(rangerBuilder.build()); try { - container.start(); + start(container); } catch (Exception e) { LOG.error("Failed to start Ranger container", e); throw new RuntimeException("Failed to start Ranger container", e); @@ -792,11 +748,7 @@ public RangerContainer getRangerContainer() { } public HiveContainer getKerberosHiveContainer() { - return kerberosHiveContainer; - } - - public HiveContainer getSQLBaseAuthHiveContainer() { - return sqlBaseHiveContainer; + return kerberosHiveContainer.get(); } public DorisContainer getDorisContainer() { @@ -841,6 +793,36 @@ public OceanBaseContainer getOceanBaseContainer() { // Let containers assign addresses in a fixed subnet to avoid // `mac-docker-connector` needing to // refresh the configuration + // Disk cleanup runs here, so it happens only when a container actually starts. + private static void start(BaseContainer container) { + ITUtils.cleanDisk(); + startTimed(container); + } + + // Logs one greppable line per start, including the container's own readiness check, so CI logs + // show where container time goes. Failed starts are logged too, with ok=false. + static void startTimed(BaseContainer container) { + long startNanos = System.nanoTime(); + boolean ok = false; + try { + container.start(); + ok = true; + } finally { + LOG.info( + "[CONTAINER-TIMING] container={} image={} hostName={} ok={} startMs={}", + container.getClass().getSimpleName(), + container.getContainer().getDockerImageName(), + container.hostName, + ok, + (System.nanoTime() - startNanos) / 1_000_000); + } + } + + // Hive2 and Hive3 run from one image; HIVE_RUNTIME_VERSION picks the version, Hive2 by default. + static String hiveVersion(Map env) { + return env.getOrDefault(HiveContainer.HIVE_RUNTIME_VERSION, HiveContainer.HIVE2); + } + private static Network createDockerNetwork() { DockerClient dockerClient = DockerClientFactory.instance().client(); @@ -956,26 +938,50 @@ private static long[] cidrToRange(String cidr) throws Exception { return new long[] {startIp, endIp}; } + /** + * Stops every container and resets the suite. Only {@code CloseContainerExtension} calls this, + * once at the end of the run; tests never close the shared suite, because later classes in the + * same JVM would have to restart every container. + * + * @throws IOException never; close failures are logged + */ @Override public void close() throws IOException { try { closer.close(); - mySQLContainer = null; - mySQLVersion5Container = null; - hiveContainer = null; - hiveRangerContainer = null; - trinoContainer = null; - trinoITContainers = null; - rangerContainer = null; - kafkaContainer = null; - dorisContainer = null; - dorisContainerMap.clear(); - kerberosHiveContainer = null; - sqlBaseHiveContainer = null; - pgContainerMap.clear(); } catch (Exception e) { LOG.error("Failed to close ContainerEnvironment", e); - } + } finally { + resetState(); + } + } + + // The closer has stopped every container and removed the network. Drop every reference so a + // later start* call starts a new container instead of returning a stopped one. + private static void resetState() { + network = null; + initialized = false; + hiveContainer.clear(); + hiveRangerContainer = null; + trinoContainer = null; + trinoITContainers = null; + rangerContainer = null; + kafkaContainer = null; + dorisContainer = null; + kerberosHiveContainer.clear(); + starRocksContainer = null; + mySQLContainer = null; + mySQLVersion5Container = null; + pgContainerMap.clear(); + dorisContainerMap.clear(); + oceanBaseContainer = null; + clickHouseContainer = null; + clickHouseClusterContainer = null; + clickHouseClusterContainers = List.of(); + zooKeeperContainer = null; + gravitinoLocalStackContainer = null; + rustFSContainer = null; + hiveContainerWithS3 = null; } private static String resolveClickHouseImage() { diff --git a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/DorisContainer.java b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/DorisContainer.java index c32f8b1e79d..4603f94b450 100644 --- a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/DorisContainer.java +++ b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/DorisContainer.java @@ -205,6 +205,7 @@ protected boolean checkContainerStatus(int retryLimit) { await() .atMost(STARTUP_TIMEOUT_SECONDS, TimeUnit.SECONDS) + .pollDelay(Duration.ZERO) .pollInterval(STARTUP_TIMEOUT_SECONDS / retryLimit, TimeUnit.SECONDS) .until( () -> { diff --git a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/HiveContainer.java b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/HiveContainer.java index 874116d31e9..9c3e8444319 100644 --- a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/HiveContainer.java +++ b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/HiveContainer.java @@ -25,6 +25,7 @@ import java.io.File; import java.net.InetSocketAddress; import java.net.Socket; +import java.time.Duration; import java.util.Map; import java.util.Optional; import java.util.Set; @@ -91,7 +92,11 @@ public void start() { @Override public void close() { - copyHiveLog(); + // A private container is closed by its JUnit store and again by the suite at the end of the + // run; only a running container has logs to copy. + if (container.isRunning()) { + copyHiveLog(); + } super.close(); } @@ -122,11 +127,14 @@ private void outputHiveStatus() { LOG.info("Hive report, stdout: {}, stderr: {}", execResult.getStdout(), execResult.getStderr()); } - @Override - protected boolean checkContainerStatus(int retryLimit) { + // Ready means HDFS has a live data node and the Hive CLI can list databases. + private void awaitHiveCli() { await() .atMost(150, TimeUnit.SECONDS) - .pollInterval(150 / retryLimit, TimeUnit.SECONDS) + .pollDelay(Duration.ZERO) + // Each check runs the Hive CLI, which takes seconds; a short interval only stops it from + // waiting long after Hive is up. + .pollInterval(1, TimeUnit.SECONDS) .until( () -> { try { @@ -137,11 +145,7 @@ protected boolean checkContainerStatus(int retryLimit) { format( "Command [%s] exited with %s", String.join(" ", commandAndArgs), execResult.getExitCode()); - LOG.error("{}", message); - LOG.error("stderr: {}", execResult.getStderr()); - LOG.error("stdout: {}", execResult.getStdout()); - - outputHiveStatus(); + LOG.info("{}, stderr: {}", message, execResult.getStderr()); } else { LOG.info("Hive container startup success!"); return true; @@ -151,31 +155,27 @@ protected boolean checkContainerStatus(int retryLimit) { } return false; }); + } - final String showDatabaseSQL = "show databases"; - // Starting both the regular Hive cluster and the Kerberos-enabled Hive cluster at the same time - // takes a long time. - await() - .atMost(120, TimeUnit.SECONDS) - .pollInterval(120 / retryLimit, TimeUnit.SECONDS) - .until( - () -> { - try { - Container.ExecResult result = executeInContainer("hive", "-e", showDatabaseSQL); - if (result.getStdout().contains("default")) { - return true; - } - } catch (Exception e) { - LOG.error("Failed to execute sql: {}", showDatabaseSQL, e); - } - return false; - }); + @Override + protected boolean checkContainerStatus(int retryLimit) { + try { + awaitHiveCli(); + } catch (RuntimeException e) { + // Each report runs two more CLIs, so only print it once Hive has run out of time. + outputHiveStatus(); + throw e; + } + + // Creating a table also creates /user/hive/warehouse in HDFS as the image's own user; Kerberos + // tests such as HiveUserAuthenticationIT rely on that directory and its permissions. final String createTableSQL = "CREATE TABLE IF NOT EXISTS default.employee ( eid int, name String, " + "salary String, destination String) "; await() .atMost(120, TimeUnit.SECONDS) - .pollInterval(120 / retryLimit, TimeUnit.SECONDS) + .pollDelay(Duration.ZERO) + .pollInterval(1, TimeUnit.SECONDS) .until( () -> { try { @@ -196,8 +196,12 @@ protected boolean checkContainerStatus(int retryLimit) { }); String containerIp = getContainerIpAddress(); + // The metastore opens its port only after the SQL checks above; under CI load that can take + // well over 10 seconds, so give it the same kind of budget as the other stages. await() - .atMost(10, TimeUnit.SECONDS) + .atMost(60, TimeUnit.SECONDS) + .pollDelay(Duration.ZERO) + .pollInterval(Duration.ofSeconds(1)) .until( () -> { try (Socket socket = new Socket()) { diff --git a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/KafkaContainer.java b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/KafkaContainer.java index 09ef6442e36..d926b95d613 100644 --- a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/KafkaContainer.java +++ b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/KafkaContainer.java @@ -25,6 +25,7 @@ import com.google.common.collect.ImmutableSet; import java.net.Socket; import java.nio.file.Paths; +import java.time.Duration; import java.util.Map; import java.util.Optional; import java.util.Set; @@ -94,6 +95,7 @@ protected boolean checkContainerStatus(int retryLimit) { await() .atMost(1, TimeUnit.MINUTES) + .pollDelay(Duration.ZERO) .pollInterval(60 / retryLimit, TimeUnit.SECONDS) .until( () -> { diff --git a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/OceanBaseContainer.java b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/OceanBaseContainer.java index c52a9a45b02..955c6c16f52 100644 --- a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/OceanBaseContainer.java +++ b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/OceanBaseContainer.java @@ -84,6 +84,7 @@ protected boolean checkContainerStatus(int retryLimit) { await() .atMost(30, TimeUnit.SECONDS) + .pollDelay(Duration.ZERO) .pollInterval(30 / retryLimit, TimeUnit.SECONDS) .until( () -> { diff --git a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/RustFSContainer.java b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/RustFSContainer.java index 71771980339..aa25009a9ce 100644 --- a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/RustFSContainer.java +++ b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/RustFSContainer.java @@ -112,6 +112,7 @@ protected boolean checkContainerStatus(int retryLimit) { try (S3Client client = createS3Client()) { await() .atMost(60, TimeUnit.SECONDS) + .pollDelay(Duration.ZERO) .pollInterval(Math.max(1, 60 / retryLimit), TimeUnit.SECONDS) .until( () -> { diff --git a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/StarRocksContainer.java b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/StarRocksContainer.java index 99be7a0c6b7..5fb546b1eed 100644 --- a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/StarRocksContainer.java +++ b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/StarRocksContainer.java @@ -39,7 +39,7 @@ public class StarRocksContainer extends BaseContainer { public static final Logger LOG = LoggerFactory.getLogger(StarRocksContainer.class); - public static final String DEFAULT_IMAGE = "starrocks/allin1-ubuntu:3.3-latest"; + public static final String DEFAULT_IMAGE = "starrocks/allin1-ubuntu:3.3.22"; public static final String HOST_NAME = "gravitino-ci-starrocks"; public static final String USER_NAME = "root"; public static final String PASSWORD = ""; @@ -86,6 +86,7 @@ protected boolean checkContainerStatus(int retryLimit) { await() .atMost(30, TimeUnit.SECONDS) + .pollDelay(Duration.ZERO) .pollInterval(30 / retryLimit, TimeUnit.SECONDS) .until( () -> { diff --git a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/TestContainerSlot.java b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/TestContainerSlot.java new file mode 100644 index 00000000000..70791ff19ed --- /dev/null +++ b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/TestContainerSlot.java @@ -0,0 +1,64 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.integration.test.container; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.mockito.Mockito; + +/** Tests {@link ContainerSlot} without Docker. */ +public class TestContainerSlot { + + @Test + void testSameKeyReusesTheRunningContainer() { + ContainerSlot slot = new ContainerSlot<>(); + HiveContainer hive2 = Mockito.mock(HiveContainer.class); + + Assertions.assertSame(hive2, slot.getOrStart(HiveContainer.HIVE2, () -> hive2)); + Assertions.assertSame( + hive2, slot.getOrStart(HiveContainer.HIVE2, () -> Assertions.fail("must not start"))); + Mockito.verify(hive2, Mockito.never()).close(); + } + + @Test + void testOtherKeyReplacesTheRunningContainer() { + ContainerSlot slot = new ContainerSlot<>(); + HiveContainer hive2 = Mockito.mock(HiveContainer.class); + HiveContainer hive3 = Mockito.mock(HiveContainer.class); + slot.getOrStart(HiveContainer.HIVE2, () -> hive2); + + Assertions.assertSame(hive3, slot.getOrStart(HiveContainer.HIVE3, () -> hive3)); + + Mockito.verify(hive2).close(); + Assertions.assertSame(hive3, slot.get()); + } + + @Test + void testClearForgetsTheContainer() { + ContainerSlot slot = new ContainerSlot<>(); + HiveContainer hive2 = Mockito.mock(HiveContainer.class); + HiveContainer restarted = Mockito.mock(HiveContainer.class); + slot.getOrStart(HiveContainer.HIVE2, () -> hive2); + + slot.clear(); + + Assertions.assertNull(slot.get()); + Assertions.assertSame(restarted, slot.getOrStart(HiveContainer.HIVE2, () -> restarted)); + } +} diff --git a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/TestContainerSuite.java b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/TestContainerSuite.java new file mode 100644 index 00000000000..9a8ce00bfb3 --- /dev/null +++ b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/TestContainerSuite.java @@ -0,0 +1,138 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.integration.test.container; + +import java.lang.reflect.Field; +import java.lang.reflect.Modifier; +import java.lang.reflect.ParameterizedType; +import java.util.Arrays; +import java.util.Collection; +import java.util.List; +import java.util.Map; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.mockito.Mockito; +import org.testcontainers.containers.GenericContainer; + +/** Tests for {@link ContainerSuite}. */ +public class TestContainerSuite { + + @Test + void testCloseResetsEveryContainerReference() throws Exception { + // Every mutable static field, so a field added later without a reset fails this test. + List stateFields = + Arrays.stream(ContainerSuite.class.getDeclaredFields()) + .filter(f -> Modifier.isStatic(f.getModifiers())) + .filter(f -> !Modifier.isFinal(f.getModifiers())) + .filter(f -> !f.getName().equals("instance")) + .toList(); + Assertions.assertFalse(stateFields.isEmpty()); + for (Field field : stateFields) { + field.setAccessible(true); + markStarted(field); + } + + HiveContainer hive = Mockito.mock(HiveContainer.class); + hiveSlot("hiveContainer").getOrStart(HiveContainer.HIVE3, () -> hive); + hiveSlot("kerberosHiveContainer").getOrStart(HiveContainer.HIVE3, () -> hive); + + ContainerSuite.getInstance().close(); + + Assertions.assertNull(ContainerSuite.getInstance().getHiveContainer()); + Assertions.assertNull(ContainerSuite.getInstance().getKerberosHiveContainer()); + for (Field field : stateFields) { + Object value = field.get(null); + if (value instanceof Map) { + Assertions.assertTrue(((Map) value).isEmpty(), field.getName()); + } else if (value instanceof Collection) { + Assertions.assertTrue(((Collection) value).isEmpty(), field.getName()); + } else if (field.getType() == boolean.class) { + Assertions.assertFalse((boolean) value, field.getName()); + } else { + Assertions.assertNull(value, field.getName()); + } + } + } + + @Test + void testHiveVersionDefaultsToHive2() { + Assertions.assertEquals(HiveContainer.HIVE2, ContainerSuite.hiveVersion(Map.of())); + Assertions.assertEquals( + HiveContainer.HIVE3, + ContainerSuite.hiveVersion( + Map.of(HiveContainer.HIVE_RUNTIME_VERSION, HiveContainer.HIVE3))); + } + + @Test + void testStartTimedStartsTheContainer() { + BaseContainer container = mockContainer(); + + ContainerSuite.startTimed(container); + + Mockito.verify(container).start(); + } + + @Test + void testStartTimedRethrowsStartFailure() { + BaseContainer container = mockContainer(); + IllegalStateException failure = new IllegalStateException("not ready"); + Mockito.doThrow(failure).when(container).start(); + + Assertions.assertSame( + failure, + Assertions.assertThrows( + IllegalStateException.class, () -> ContainerSuite.startTimed(container))); + } + + private static BaseContainer mockContainer() { + GenericContainer genericContainer = Mockito.mock(GenericContainer.class); + Mockito.when(genericContainer.getDockerImageName()).thenReturn("test/image:1.0"); + BaseContainer container = Mockito.mock(BaseContainer.class); + Mockito.doReturn(genericContainer).when(container).getContainer(); + return container; + } + + @SuppressWarnings("unchecked") + private static ContainerSlot hiveSlot(String fieldName) throws Exception { + Field field = ContainerSuite.class.getDeclaredField(fieldName); + field.setAccessible(true); + return (ContainerSlot) field.get(null); + } + + @SuppressWarnings({"unchecked", "rawtypes"}) + private static void markStarted(Field field) throws IllegalAccessException { + Class type = field.getType(); + if (type == boolean.class) { + field.setBoolean(null, true); + } else if (Map.class.isAssignableFrom(type)) { + // The maps are EnumMaps, so the key must be a constant of the declared key enum. + Class keyType = + (Class) ((ParameterizedType) field.getGenericType()).getActualTypeArguments()[0]; + Class valueType = + (Class) ((ParameterizedType) field.getGenericType()).getActualTypeArguments()[1]; + ((Map) field.get(null)).put(keyType.getEnumConstants()[0], Mockito.mock(valueType)); + } else if (List.class.isAssignableFrom(type)) { + Class elementType = + (Class) ((ParameterizedType) field.getGenericType()).getActualTypeArguments()[0]; + field.set(null, List.of(Mockito.mock(elementType))); + } else { + field.set(null, Mockito.mock(type)); + } + } +} diff --git a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/TrinoContainer.java b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/TrinoContainer.java index d2d16f9da7f..47f93aa04e8 100644 --- a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/TrinoContainer.java +++ b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/TrinoContainer.java @@ -32,6 +32,7 @@ import java.sql.ResultSetMetaData; import java.sql.SQLException; import java.sql.Statement; +import java.time.Duration; import java.util.ArrayList; import java.util.HashMap; import java.util.Map; @@ -108,6 +109,7 @@ public void start() { protected boolean checkContainerStatus(int retryLimit) { await() .atMost(30, TimeUnit.SECONDS) + .pollDelay(Duration.ZERO) .pollInterval(30 / retryLimit, TimeUnit.SECONDS) .until( () -> { @@ -155,6 +157,7 @@ public boolean initTrinoJdbcConnection() { public boolean checkSyncCatalogFromGravitino(int retryLimit, String catalogName) { await() .atMost(30, TimeUnit.SECONDS) + .pollDelay(Duration.ZERO) .pollInterval(60 / retryLimit, TimeUnit.SECONDS) .until( () -> { diff --git a/maintenance/optimizer/src/test/resources/log4j2.properties b/maintenance/optimizer/src/test/resources/log4j2.properties new file mode 100644 index 00000000000..6b46d06dc7e --- /dev/null +++ b/maintenance/optimizer/src/test/resources/log4j2.properties @@ -0,0 +1,73 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +# Set to debug or trace if log4j initialization is failing +status = info + +# Name of the configuration +name = ConsoleLogConfig + +# Console appender configuration +appender.console.type = Console +appender.console.name = consoleLogger +appender.console.layout.type = PatternLayout +appender.console.layout.pattern = %d{yyyy-MM-dd HH:mm:ss} %-5p [%t] %c{1}:%L - %m%n + +# Log files location +property.logPath = ${sys:gravitino.log.path:-build/optimizer-integration-test.log} + +# File appender configuration +appender.file.type = File +appender.file.name = fileLogger +appender.file.fileName = ${logPath} +appender.file.layout.type = PatternLayout +appender.file.layout.pattern = %d{yyyy-MM-dd HH:mm:ss.SSS} [%t] %-5p %c - %m%n + +# Root logger level +rootLogger.level = info + +# Root logger referring to console and file appenders +rootLogger.appenderRef.stdout.ref = consoleLogger +rootLogger.appenderRef.file.ref = fileLogger + +# File appender configuration for testcontainers +appender.testcontainersFile.type = File +appender.testcontainersFile.name = testcontainersLogger +appender.testcontainersFile.fileName = build/testcontainers.log +appender.testcontainersFile.layout.type = PatternLayout +appender.testcontainersFile.layout.pattern = %d{yyyy-MM-dd HH:mm:ss.SSS} [%t] %-5p %c - %m%n + +# Logger for testcontainers +logger.testcontainers.name = org.testcontainers +logger.testcontainers.level = debug +logger.testcontainers.additivity = false +logger.testcontainers.appenderRef.file.ref = testcontainersLogger + +logger.tc.name = tc +logger.tc.level = debug +logger.tc.additivity = false +logger.tc.appenderRef.file.ref = testcontainersLogger + +logger.docker.name = com.github.dockerjava +logger.docker.level = warn +logger.docker.additivity = false +logger.docker.appenderRef.file.ref = testcontainersLogger + +logger.http.name = com.github.dockerjava.zerodep.shaded.org.apache.hc.client5.http.wire +logger.http.level = off diff --git a/plugins/idp-basic/src/test/java/org/apache/gravitino/idp/storage/mapper/AbstractIdpMetaStorageTest.java b/plugins/idp-basic/src/test/java/org/apache/gravitino/idp/storage/mapper/AbstractIdpMetaStorageTest.java index d556fc26b54..4090577aa4c 100644 --- a/plugins/idp-basic/src/test/java/org/apache/gravitino/idp/storage/mapper/AbstractIdpMetaStorageTest.java +++ b/plugins/idp-basic/src/test/java/org/apache/gravitino/idp/storage/mapper/AbstractIdpMetaStorageTest.java @@ -38,12 +38,16 @@ import org.apache.gravitino.integration.test.container.ContainerSuite; import org.apache.gravitino.integration.test.container.MySQLContainer; import org.apache.gravitino.integration.test.container.PostgreSQLContainer; +import org.apache.gravitino.integration.test.util.CloseContainerExtension; import org.apache.gravitino.integration.test.util.TestDatabaseName; import org.apache.gravitino.storage.relational.JDBCBackend; import org.apache.gravitino.storage.relational.session.SqlSessionFactoryHelper; import org.apache.ibatis.session.SqlSession; import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.extension.ExtendWith; +// Containers are shared across tests and closed once per run; each test recreates its own database. +@ExtendWith(CloseContainerExtension.class) public abstract class AbstractIdpMetaStorageTest { private static final String H2_BACKEND = "h2"; private static final String MYSQL_BACKEND = "mysql"; @@ -66,7 +70,7 @@ public static Stream storageProvider() { } @AfterEach - void closeSuite() throws IOException { + void closeBackend() throws IOException { closeSession(); if (backend != null) { backend.close(); @@ -75,7 +79,6 @@ void closeSuite() throws IOException { SqlSessionFactoryHelper.getInstance().close(); IdpSQLExceptionConverterFactory.close(); - ContainerSuite.getInstance().close(); if (h2Path != null && Files.exists(h2Path)) { deleteDirectory(h2Path); diff --git a/plugins/idp-basic/src/test/resources/log4j2.properties b/plugins/idp-basic/src/test/resources/log4j2.properties new file mode 100644 index 00000000000..57088576d23 --- /dev/null +++ b/plugins/idp-basic/src/test/resources/log4j2.properties @@ -0,0 +1,73 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +# Set to debug or trace if log4j initialization is failing +status = info + +# Name of the configuration +name = ConsoleLogConfig + +# Console appender configuration +appender.console.type = Console +appender.console.name = consoleLogger +appender.console.layout.type = PatternLayout +appender.console.layout.pattern = %d{yyyy-MM-dd HH:mm:ss} %-5p [%t] %c{1}:%L - %m%n + +# Log files location +property.logPath = ${sys:gravitino.log.path:-build/idp-basic-integration-test.log} + +# File appender configuration +appender.file.type = File +appender.file.name = fileLogger +appender.file.fileName = ${logPath} +appender.file.layout.type = PatternLayout +appender.file.layout.pattern = %d{yyyy-MM-dd HH:mm:ss.SSS} [%t] %-5p %c - %m%n + +# Root logger level +rootLogger.level = info + +# Root logger referring to console and file appenders +rootLogger.appenderRef.stdout.ref = consoleLogger +rootLogger.appenderRef.file.ref = fileLogger + +# File appender configuration for testcontainers +appender.testcontainersFile.type = File +appender.testcontainersFile.name = testcontainersLogger +appender.testcontainersFile.fileName = build/testcontainers.log +appender.testcontainersFile.layout.type = PatternLayout +appender.testcontainersFile.layout.pattern = %d{yyyy-MM-dd HH:mm:ss.SSS} [%t] %-5p %c - %m%n + +# Logger for testcontainers +logger.testcontainers.name = org.testcontainers +logger.testcontainers.level = debug +logger.testcontainers.additivity = false +logger.testcontainers.appenderRef.file.ref = testcontainersLogger + +logger.tc.name = tc +logger.tc.level = debug +logger.tc.additivity = false +logger.tc.appenderRef.file.ref = testcontainersLogger + +logger.docker.name = com.github.dockerjava +logger.docker.level = warn +logger.docker.additivity = false +logger.docker.appenderRef.file.ref = testcontainersLogger + +logger.http.name = com.github.dockerjava.zerodep.shaded.org.apache.hc.client5.http.wire +logger.http.level = off diff --git a/spark-connector/spark-common/src/test/java/org/apache/gravitino/spark/connector/integration/test/authorization/SparkAuthorizationIT.java b/spark-connector/spark-common/src/test/java/org/apache/gravitino/spark/connector/integration/test/authorization/SparkAuthorizationIT.java index 4a85a596362..f988a4e0e77 100644 --- a/spark-connector/spark-common/src/test/java/org/apache/gravitino/spark/connector/integration/test/authorization/SparkAuthorizationIT.java +++ b/spark-connector/spark-common/src/test/java/org/apache/gravitino/spark/connector/integration/test/authorization/SparkAuthorizationIT.java @@ -117,7 +117,6 @@ public void startIntegrationTest() throws Exception { @AfterAll @Override public void stopIntegrationTest() throws IOException, InterruptedException { - containerSuite.close(); setEnv("SPARK_USER", AuthConstants.ANONYMOUS_USER); // HADOOP_USER_NAME will be set by spark auto. setEnv("HADOOP_USER_NAME", AuthConstants.ANONYMOUS_USER); From 109df9ad96e724af219627106046132d7f8c2708 Mon Sep 17 00:00:00 2001 From: Nevin Zheng Date: Tue, 20 Oct 2026 12:34:16 -0700 Subject: [PATCH 07/14] [#14049] improvement(test): Run Hive SQL over JDBC and check Hive readiness without the CLI --- catalogs/catalog-hive/build.gradle.kts | 19 ++ .../hive/integration/test/CatalogHive3IT.java | 30 +-- .../integration/test/CatalogHiveViewIT.java | 47 +---- .../hive/integration/test/HiveServer2Sql.java | 98 ++++++++++ .../test/container/HiveContainer.java | 183 ++++++++++++++++-- 5 files changed, 310 insertions(+), 67 deletions(-) create mode 100644 catalogs/catalog-hive/src/test/java/org/apache/gravitino/catalog/hive/integration/test/HiveServer2Sql.java diff --git a/catalogs/catalog-hive/build.gradle.kts b/catalogs/catalog-hive/build.gradle.kts index 5fa39a73608..a893a4afa0b 100644 --- a/catalogs/catalog-hive/build.gradle.kts +++ b/catalogs/catalog-hive/build.gradle.kts @@ -114,6 +114,25 @@ dependencies { exclude("org.eclipse.jetty.aggregate", "jetty-all") exclude("org.eclipse.jetty.orbit", "javax.servlet") } + // Runs HiveQL against the container's HiveServer2 instead of a `hive -e` CLI per statement. + testImplementation(libs.hive2.jdbc) { + exclude("org.apache.hadoop") + // hive-service pulls in the LLAP server and its HBase, Tephra, Twill and Slider stack, which + // a JDBC client never loads. + exclude("org.apache.hive", "hive-llap-server") + exclude("org.apache.hive", "hive-llap-tez") + exclude("org.apache.tez") + exclude("org.apache.hbase") + exclude("co.cask.tephra") + exclude("org.apache.twill") + exclude("org.apache.slider") + exclude("org.mortbay.jetty") + exclude("org.apache.tomcat") + exclude("org.eclipse.jetty.aggregate", "jetty-all") + exclude("org.eclipse.jetty.orbit", "javax.servlet") + exclude("org.apache.logging.log4j") + exclude("org.slf4j") + } testImplementation(libs.junit.jupiter.api) testImplementation(libs.mockito.core) testImplementation(libs.mysql.driver) diff --git a/catalogs/catalog-hive/src/test/java/org/apache/gravitino/catalog/hive/integration/test/CatalogHive3IT.java b/catalogs/catalog-hive/src/test/java/org/apache/gravitino/catalog/hive/integration/test/CatalogHive3IT.java index 431dbded283..404a9fe2605 100644 --- a/catalogs/catalog-hive/src/test/java/org/apache/gravitino/catalog/hive/integration/test/CatalogHive3IT.java +++ b/catalogs/catalog-hive/src/test/java/org/apache/gravitino/catalog/hive/integration/test/CatalogHive3IT.java @@ -19,6 +19,7 @@ package org.apache.gravitino.catalog.hive.integration.test; import com.google.common.collect.ImmutableMap; +import java.sql.SQLException; import java.util.Arrays; import java.util.Map; import org.apache.gravitino.NameIdentifier; @@ -31,15 +32,24 @@ import org.apache.gravitino.rel.expressions.transforms.Transform; import org.apache.gravitino.rel.expressions.transforms.Transforms; import org.apache.gravitino.rel.types.Types; +import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.TestInstance; -import org.testcontainers.containers.Container.ExecResult; @Tag("gravitino-docker-test") @TestInstance(TestInstance.Lifecycle.PER_CLASS) public class CatalogHive3IT extends CatalogHive2IT { + private HiveServer2Sql hiveSql; + + @AfterAll + public void closeHiveSql() throws SQLException { + if (hiveSql != null) { + hiveSql.close(); + } + } + @Override protected void startNecessaryContainer() { hmsCatalog = "hive"; @@ -102,12 +112,12 @@ protected void checkColumnConstraintsOnCreate( "INSERT INTO TABLE %s.%s (not_null_column, plain_column) " + "VALUES ('required', 'plain')", schemaName, constraintsIdent.name())); - ExecResult queryResult = + String queryResult = executeHiveSql( String.format( "SELECT default_column, default_string_column FROM %s.%s", schemaName, constraintsIdent.name())); - Assertions.assertEquals("42\tit's", queryResult.getStdout().trim()); + Assertions.assertEquals("42\tit's", queryResult.trim()); // Property-only alters must leave the constraints untouched catalog.asTableCatalog().alterTable(constraintsIdent, TableChange.setProperty("k1", "v1")); @@ -161,15 +171,11 @@ protected void checkColumnConstraintsOnCreate( } } - private ExecResult executeHiveSql(String sql) { - ExecResult result = hiveContainer().executeInContainer("hive", "-S", "-e", sql); - Assertions.assertEquals( - 0, - result.getExitCode(), - String.format( - "Failed to execute SQL with hive cli. SQL: %s, stdout: %s, stderr: %s", - sql, result.getStdout(), result.getStderr())); - return result; + private String executeHiveSql(String sql) { + if (hiveSql == null) { + hiveSql = new HiveServer2Sql(hiveContainer()); + } + return hiveSql.run(sql); } private Column[] loadHiveTableColumns(String schema, String table) { diff --git a/catalogs/catalog-hive/src/test/java/org/apache/gravitino/catalog/hive/integration/test/CatalogHiveViewIT.java b/catalogs/catalog-hive/src/test/java/org/apache/gravitino/catalog/hive/integration/test/CatalogHiveViewIT.java index 630c2e472b5..ef823785a91 100644 --- a/catalogs/catalog-hive/src/test/java/org/apache/gravitino/catalog/hive/integration/test/CatalogHiveViewIT.java +++ b/catalogs/catalog-hive/src/test/java/org/apache/gravitino/catalog/hive/integration/test/CatalogHiveViewIT.java @@ -48,7 +48,6 @@ import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.TestInstance; -import org.testcontainers.containers.Container.ExecResult; /** Integration tests for Hive catalog view CRUD operations. */ @Tag("gravitino-docker-test") @@ -60,6 +59,7 @@ public class CatalogHiveViewIT extends BaseIT { private static final String HIVE_DIALECT = "hive"; private final ContainerSuite containerSuite = ContainerSuite.getInstance(); + private HiveServer2Sql hiveSql; private String metalakeName; private String catalogName; private String schemaName; @@ -95,6 +95,7 @@ public void setup() throws Exception { catalogName, Catalog.Type.RELATIONAL, PROVIDER, "comment", catalogProperties); viewCatalog = catalog.asViewCatalog(); catalog.asSchemas().createSchema(schemaName, "schema comment", Maps.newHashMap()); + hiveSql = new HiveServer2Sql(containerSuite.getHiveContainer()); } @AfterEach @@ -107,7 +108,7 @@ public void cleanViews() { @AfterAll public void teardown() throws Exception { - try { + try (HiveServer2Sql sql = hiveSql) { catalog.asSchemas().dropSchema(schemaName, true); metalake.disableCatalog(catalogName); metalake.dropCatalog(catalogName, true); @@ -535,50 +536,14 @@ private String qualifyHiveObject(String objectName) { } private void executeHiveSql(String sql) { - ExecResult result = - containerSuite.getHiveContainer().executeInContainer("hive", "-S", "-e", sql); - Assertions.assertEquals( - 0, - result.getExitCode(), - String.format( - "Failed to execute SQL with hive cli. SQL: %s, stdout: %s, stderr: %s", - sql, result.getStdout(), result.getStderr())); + hiveSql.run(sql); } private long querySingleLong(String sql) { - ExecResult result = - containerSuite.getHiveContainer().executeInContainer("hive", "-S", "-e", sql); - Assertions.assertEquals( - 0, - result.getExitCode(), - String.format( - "Failed to query SQL with hive cli. SQL: %s, stdout: %s, stderr: %s", - sql, result.getStdout(), result.getStderr())); - - String[] lines = result.getStdout().split("\\R"); - for (int index = lines.length - 1; index >= 0; index--) { - String line = lines[index].trim(); - if (line.matches("-?\\d+")) { - return Long.parseLong(line); - } - } - throw new IllegalStateException( - String.format( - "Cannot parse numeric result from query output. SQL: %s, stdout: %s", - sql, result.getStdout())); + return Long.parseLong(hiveSql.run(sql).trim()); } private String describeHiveObject(String qualifiedObjectName) { - ExecResult result = - containerSuite - .getHiveContainer() - .executeInContainer("hive", "-S", "-e", "DESCRIBE FORMATTED " + qualifiedObjectName); - Assertions.assertEquals( - 0, - result.getExitCode(), - String.format( - "Failed to describe Hive object. Object: %s, stdout: %s, stderr: %s", - qualifiedObjectName, result.getStdout(), result.getStderr())); - return result.getStdout(); + return hiveSql.run("DESCRIBE FORMATTED " + qualifiedObjectName); } } diff --git a/catalogs/catalog-hive/src/test/java/org/apache/gravitino/catalog/hive/integration/test/HiveServer2Sql.java b/catalogs/catalog-hive/src/test/java/org/apache/gravitino/catalog/hive/integration/test/HiveServer2Sql.java new file mode 100644 index 00000000000..f88e65b212d --- /dev/null +++ b/catalogs/catalog-hive/src/test/java/org/apache/gravitino/catalog/hive/integration/test/HiveServer2Sql.java @@ -0,0 +1,98 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.catalog.hive.integration.test; + +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.ArrayList; +import java.util.List; +import org.apache.gravitino.integration.test.container.HiveContainer; + +/** + * Runs HiveQL through the HiveServer2 that already runs in the Hive container, over one JDBC + * connection opened on first use. + * + *

Each {@code hive -e} call through {@code docker exec} boots a CLI JVM with its own embedded + * metastore, which costs seconds per statement; a statement over JDBC does not. + */ +final class HiveServer2Sql implements AutoCloseable { + + private final HiveContainer hive; + private Connection connection; + + HiveServer2Sql(HiveContainer hive) { + this.hive = hive; + } + + /** + * Runs one statement and returns its rows the way {@code hive -S -e} prints them: one row per + * line, columns separated by tabs, nulls as empty strings. Statements without a result set return + * an empty string. Any error fails the call. + */ + String run(String sql) { + try (Statement statement = connection().createStatement()) { + if (!statement.execute(sql)) { + return ""; + } + try (ResultSet resultSet = statement.getResultSet()) { + return format(resultSet); + } + } catch (SQLException e) { + throw new IllegalStateException("Failed to run Hive SQL: " + sql, e); + } + } + + @Override + public void close() throws SQLException { + if (connection != null) { + connection.close(); + connection = null; + } + } + + private Connection connection() throws SQLException { + if (connection == null) { + // HiveContainer waits for HiveServer2's port and lets it impersonate the container's Hadoop + // user, the user `docker exec hive -e` ran as, so HDFS ownership stays the same. + String url = + String.format( + "jdbc:hive2://%s:%d/default", + hive.getContainerIpAddress(), HiveContainer.HIVE_SERVICE_PORT); + connection = DriverManager.getConnection(url, hive.getHadoopUser(), ""); + } + return connection; + } + + private static String format(ResultSet resultSet) throws SQLException { + int columns = resultSet.getMetaData().getColumnCount(); + List rows = new ArrayList<>(); + while (resultSet.next()) { + List values = new ArrayList<>(columns); + for (int i = 1; i <= columns; i++) { + String value = resultSet.getString(i); + values.add(value == null ? "" : value); + } + rows.add(String.join("\t", values)); + } + return String.join("\n", rows); + } +} diff --git a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/HiveContainer.java b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/HiveContainer.java index 9c3e8444319..2fb04b4768f 100644 --- a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/HiveContainer.java +++ b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/HiveContainer.java @@ -23,8 +23,13 @@ import com.google.common.collect.ImmutableSet; import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.net.HttpURLConnection; import java.net.InetSocketAddress; import java.net.Socket; +import java.net.URL; +import java.nio.charset.StandardCharsets; import java.time.Duration; import java.util.Map; import java.util.Optional; @@ -58,6 +63,18 @@ public class HiveContainer extends BaseContainer { private static final String HIVE_LOG_PATH = "/tmp/root/"; private static final String HDFS_LOG_PATH = "/usr/local/hadoop/logs/"; + // The image ships Hadoop 3 (NameNode web port 9870) for hive3 and Hadoop 2 (50070) for hive2. + private static final int[] NAMENODE_HTTP_PORTS = {9870, 50070}; + private static final String WAREHOUSE_DIR = "/user/hive/warehouse"; + // The image's core-site.xml, which its start script copies into the Hadoop configuration. + private static final String HADOOP_CONF_TO_COPY = "/tmp/hadoop-conf/core-site.xml"; + // The Hive images' entrypoint. + private static final String IMAGE_START = "/usr/local/sbin/start.sh"; + // The image's core-site.xml already lets these users impersonate. + private static final Set USERS_ALLOWED_TO_IMPERSONATE = + ImmutableSet.of("root", "hive", "gravitino"); + + private final boolean kerberosEnabled; public static Builder builder() { return new Builder(); @@ -70,14 +87,25 @@ protected HiveContainer( Map extraHosts, Map filesToMount, Map envVars, - Optional network) { + Optional network, + boolean kerberosEnabled) { super(image, hostName, ports, extraHosts, filesToMount, envVars, network); + this.kerberosEnabled = kerberosEnabled; } public String getHostName() { return hostName; } + /** + * Returns the user the container's Hadoop and Hive processes run as: {@code HADOOP_USER_NAME} + * when the test sets it, else root. Clients that write to HDFS should act as this user so they + * get the same ownership as the old {@code docker exec hive} calls. + */ + public String getHadoopUser() { + return container.getEnvMap().getOrDefault(HADOOP_USER_NAME, "root"); + } + @Override protected void setupContainer() { super.setupContainer(); @@ -86,10 +114,42 @@ protected void setupContainer() { @Override public void start() { + if (!kerberosEnabled) { + allowHadoopUserToImpersonate(); + } super.start(); Preconditions.check("Hive container startup failed!", checkContainerStatus(15)); } + /** + * Lets the Hadoop user, which every process in the container runs as, impersonate users. + * + *

HiveServer2 runs each JDBC statement as the session user (doAs), so it impersonates even + * when that user is itself; the image only allows that for {@code hive}, {@code root} and {@code + * gravitino}. The rule goes into the configuration the launch script copies in, before the + * NameNode starts. + */ + private void allowHadoopUserToImpersonate() { + String user = getHadoopUser(); + if (USERS_ALLOWED_TO_IMPERSONATE.contains(user)) { + return; + } + String rule = + format( + "hadoop.proxyuser.%1$s.hosts*" + + "hadoop.proxyuser.%1$s.groups*", + user); + // Fail the start, rather than let JDBC sessions fail later, if a new image changes the file. + String script = + format( + "sed -i 's||%1$s|' %2$s" + + " && grep -q 'hadoop.proxyuser.%3$s.hosts' %2$s" + + " || { echo 'Cannot add the proxy-user rule to %2$s' >&2; exit 1; }", + rule, HADOOP_CONF_TO_COPY, user); + container.withCreateContainerCmdModifier( + cmd -> cmd.withEntrypoint("/bin/bash", "-c", script + " && exec /bin/bash " + IMAGE_START)); + } + @Override public void close() { // A private container is closed by its JUnit store and again by the suite at the end of the @@ -127,6 +187,100 @@ private void outputHiveStatus() { LOG.info("Hive report, stdout: {}, stderr: {}", execResult.getStdout(), execResult.getStderr()); } + @Override + protected boolean checkContainerStatus(int retryLimit) { + if (kerberosEnabled) { + awaitReadyWithHiveCli(); + awaitPort(HIVE_METASTORE_PORT, "Hive Metastore"); + } else { + awaitReadyOverHttp(); + awaitPort(HIVE_METASTORE_PORT, "Hive Metastore"); + // Tests run HiveQL over JDBC; Kerberos Hive tests still use the CLI. + awaitPort(HIVE_SERVICE_PORT, "HiveServer2"); + } + return true; + } + + /** + * Waits for HDFS through the NameNode's JMX and creates the warehouse directory through WebHDFS. + * The Hive CLI checks this replaces each boot a JVM with an embedded metastore, which costs + * several seconds per call; the metastore port check that follows covers Hive itself. + */ + private void awaitReadyOverHttp() { + String containerIp = getContainerIpAddress(); + int[] namenodePort = new int[1]; + await() + .atMost(150, TimeUnit.SECONDS) + .pollDelay(Duration.ZERO) + .pollInterval(500, TimeUnit.MILLISECONDS) + .until( + () -> { + for (int port : NAMENODE_HTTP_PORTS) { + String jmx = + httpRequest( + "GET", + format( + "http://%s:%d/jmx?qry=Hadoop:service=NameNode,name=FSNamesystemState", + containerIp, port)); + if (jmx != null && jmx.matches("(?s).*\"NumLiveDataNodes\"\\s*:\\s*[1-9].*")) { + namenodePort[0] = port; + return true; + } + } + return false; + }); + + // `CREATE TABLE` through the Hive CLI used to create this directory as the Hadoop user; tests + // that write to the default database expect it to exist. + await() + .atMost(60, TimeUnit.SECONDS) + .pollDelay(Duration.ZERO) + .pollInterval(500, TimeUnit.MILLISECONDS) + .until( + () -> { + String response = + httpRequest( + "PUT", + format( + "http://%s:%d/webhdfs/v1%s?op=MKDIRS&user.name=%s", + containerIp, namenodePort[0], WAREHOUSE_DIR, getHadoopUser())); + return response != null && response.contains("true"); + }); + LOG.info("Hive container startup success!"); + } + + /** Returns the response body, or null when the request fails or returns a non-2xx status. */ + private static String httpRequest(String method, String url) { + HttpURLConnection connection = null; + try { + connection = (HttpURLConnection) new URL(url).openConnection(); + connection.setRequestMethod(method); + connection.setConnectTimeout(2000); + connection.setReadTimeout(5000); + int status = connection.getResponseCode(); + if (status / 100 != 2) { + String error = ""; + if (connection.getErrorStream() != null) { + try (InputStream body = connection.getErrorStream()) { + error = new String(body.readAllBytes(), StandardCharsets.UTF_8); + } + } + LOG.debug("{} {} returned HTTP {}: {}", method, url, status, error); + return null; + } + try (InputStream body = connection.getInputStream()) { + return new String(body.readAllBytes(), StandardCharsets.UTF_8); + } + } catch (IOException e) { + LOG.debug("{} {} failed: {}", method, url, e.toString()); + return null; + } finally { + if (connection != null) { + connection.disconnect(); + } + } + } + // Ready means HDFS has a live data node and the Hive CLI can list databases. private void awaitHiveCli() { await() @@ -157,8 +311,7 @@ private void awaitHiveCli() { }); } - @Override - protected boolean checkContainerStatus(int retryLimit) { + private void awaitReadyWithHiveCli() { try { awaitHiveCli(); } catch (RuntimeException e) { @@ -194,9 +347,11 @@ protected boolean checkContainerStatus(int retryLimit) { } return false; }); + } + private void awaitPort(int port, String service) { String containerIp = getContainerIpAddress(); - // The metastore opens its port only after the SQL checks above; under CI load that can take + // The metastore opens its port only after its schema is set up; under CI load that can take // well over 10 seconds, so give it the same kind of budget as the other stages. await() .atMost(60, TimeUnit.SECONDS) @@ -205,20 +360,13 @@ protected boolean checkContainerStatus(int retryLimit) { .until( () -> { try (Socket socket = new Socket()) { - socket.connect( - new InetSocketAddress(containerIp, HiveContainer.HIVE_METASTORE_PORT), 3000); + socket.connect(new InetSocketAddress(containerIp, port), 3000); return true; } catch (Exception e) { - LOG.warn( - "Can't connect to Hive Metastore:[{}:{}]", - containerIp, - HiveContainer.HIVE_METASTORE_PORT, - e); + LOG.warn("Can't connect to {}:[{}:{}]", service, containerIp, port, e); } return false; }); - - return true; } public static class Builder extends BaseContainer.Builder { @@ -240,7 +388,14 @@ private String generateImageName() { @Override public HiveContainer build() { return new HiveContainer( - generateImageName(), hostName, exposePorts, extraHosts, filesToMount, envVars, network); + generateImageName(), + hostName, + exposePorts, + extraHosts, + filesToMount, + envVars, + network, + kerberosEnabled); } } } From 8b40f383d08ad0ad2edeb8f2682f7cb6ac9dd3a7 Mon Sep 17 00:00:00 2001 From: Nevin Zheng Date: Wed, 21 Oct 2026 00:00:12 -0700 Subject: [PATCH 08/14] [#14049] improvement(test): Replace the remaining container CLI calls with in-process clients --- .../integration/test/FilesetS3CatalogIT.java | 47 +---- .../test/HadoopUserAuthenticationIT.java | 44 ++-- catalogs/catalog-hive/build.gradle.kts | 4 + .../integration/test/CatalogHiveS3IT.java | 52 +---- .../test/HiveUserAuthenticationIT.java | 18 +- .../test/HudiCatalogKerberosHiveIT.java | 24 +-- .../test/CatalogIcebergKerberosHiveIT.java | 16 +- .../catalog-lakehouse-paimon/build.gradle.kts | 4 + .../CatalogPaimonKerberosFilesystemIT.java | 13 +- .../integration/test/CatalogPaimonS3AIT.java | 19 +- .../integration/test/CatalogPaimonS3IT.java | 19 +- clients/filesystem-hadoop3/build.gradle.kts | 4 + .../test/GravitinoVirtualFileSystemS3IT.java | 47 +---- ...rosHiveWithUserImpersonationCatalogIT.java | 5 +- ...rosJdbcWithUserImpersonationCatalogIT.java | 5 +- .../test/IcebergRestKerberosTestEnv.java | 19 +- .../test/container/DatabaseContainer.java | 27 ++- .../GravitinoLocalStackContainer.java | 124 +++++++++++ .../test/container/MySQLContainer.java | 6 - .../test/container/PostgreSQLContainer.java | 9 - .../TestGravitinoLocalStackContainer.java | 39 ++++ .../integration/test/util/KerberosHdfs.java | 192 ++++++++++++++++++ .../test/util/TestKerberosHdfs.java | 110 ++++++++++ .../integration-test/build.gradle.kts | 4 + .../integration/test/TrinoConnectorIT.java | 91 +++------ .../TrinoTlsOAuthCredentialVendingIT.java | 11 +- 26 files changed, 616 insertions(+), 337 deletions(-) create mode 100644 integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/TestGravitinoLocalStackContainer.java create mode 100644 integration-test-common/src/test/java/org/apache/gravitino/integration/test/util/KerberosHdfs.java create mode 100644 integration-test-common/src/test/java/org/apache/gravitino/integration/test/util/TestKerberosHdfs.java diff --git a/catalogs/catalog-fileset/src/test/java/org/apache/gravitino/catalog/fileset/integration/test/FilesetS3CatalogIT.java b/catalogs/catalog-fileset/src/test/java/org/apache/gravitino/catalog/fileset/integration/test/FilesetS3CatalogIT.java index 2714321d6e7..674df6984a1 100644 --- a/catalogs/catalog-fileset/src/test/java/org/apache/gravitino/catalog/fileset/integration/test/FilesetS3CatalogIT.java +++ b/catalogs/catalog-fileset/src/test/java/org/apache/gravitino/catalog/fileset/integration/test/FilesetS3CatalogIT.java @@ -27,7 +27,6 @@ import java.net.URI; import java.util.Map; import java.util.UUID; -import java.util.concurrent.TimeUnit; import org.apache.gravitino.Catalog; import org.apache.gravitino.NameIdentifier; import org.apache.gravitino.Schema; @@ -38,7 +37,6 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; -import org.awaitility.Awaitility; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeAll; @@ -46,7 +44,6 @@ import org.junit.jupiter.api.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.testcontainers.containers.Container; @Tag("gravitino-docker-test") public class FilesetS3CatalogIT extends FilesetCatalogIT { @@ -67,44 +64,12 @@ protected void startNecessaryContainer() { containerSuite.startLocalStackContainer(); gravitinoLocalStackContainer = containerSuite.getLocalStackContainer(); - Awaitility.await() - .atMost(60, TimeUnit.SECONDS) - .pollInterval(1, TimeUnit.SECONDS) - .until( - () -> { - try { - Container.ExecResult result = - gravitinoLocalStackContainer.executeInContainer( - "awslocal", "iam", "create-user", "--user-name", "anonymous"); - return result.getExitCode() == 0; - } catch (Exception e) { - LOG.info("LocalStack is not ready yet for: ", e); - return false; - } - }); - - gravitinoLocalStackContainer.executeInContainer("awslocal", "s3", "mb", "s3://" + bucketName); - - Container.ExecResult result = - gravitinoLocalStackContainer.executeInContainer( - "awslocal", "iam", "create-access-key", "--user-name", "anonymous"); - - gravitinoLocalStackContainer.executeInContainer( - "awslocal", - "s3api", - "put-bucket-acl", - "--bucket", - "my-test-bucket", - "--acl", - "public-read-write"); - - // Get access key and secret key from result - String[] lines = result.getStdout().split("\n"); - accessKey = lines[3].split(":")[1].trim().substring(1, 21); - secretKey = lines[5].split(":")[1].trim().substring(1, 41); - - LOG.info("Access key: " + accessKey); - LOG.info("Secret key: " + secretKey); + // The container is ready once its health check passes. LocalStack does not enforce IAM, so + // the shared test keys work without creating a user. + gravitinoLocalStackContainer.createBucket(bucketName); + gravitinoLocalStackContainer.makeBucketPublic(bucketName); + accessKey = GravitinoLocalStackContainer.ACCESS_KEY; + secretKey = GravitinoLocalStackContainer.SECRET_KEY; s3Endpoint = String.format("http://localhost:%d", gravitinoLocalStackContainer.getMappedPort(4566)); diff --git a/catalogs/catalog-fileset/src/test/java/org/apache/gravitino/catalog/fileset/integration/test/HadoopUserAuthenticationIT.java b/catalogs/catalog-fileset/src/test/java/org/apache/gravitino/catalog/fileset/integration/test/HadoopUserAuthenticationIT.java index 5566c889f18..af30205e626 100644 --- a/catalogs/catalog-fileset/src/test/java/org/apache/gravitino/catalog/fileset/integration/test/HadoopUserAuthenticationIT.java +++ b/catalogs/catalog-fileset/src/test/java/org/apache/gravitino/catalog/fileset/integration/test/HadoopUserAuthenticationIT.java @@ -49,6 +49,7 @@ import org.apache.gravitino.integration.test.container.HiveContainer; import org.apache.gravitino.integration.test.util.BaseIT; import org.apache.gravitino.integration.test.util.GravitinoITUtils; +import org.apache.gravitino.integration.test.util.KerberosHdfs; import org.apache.hadoop.security.UserGroupInformation; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.Assertions; @@ -92,6 +93,8 @@ public class HadoopUserAuthenticationIT extends BaseIT { private static GravitinoAdminClient adminClient; + private static KerberosHdfs hdfs; + private static HiveContainer kerberosHiveContainer; private static final String METALAKE_NAME = @@ -117,6 +120,7 @@ public void startIntegrationTest() throws Exception { // Prepare kerberos related-config; prepareKerberosConfig(); + hdfs = KerberosHdfs.of(kerberosHiveContainer, TMP_DIR + HADOOP_CLIENT_KEYTAB); // Config kerberos configuration for Gravitino server addKerberosConfig(); @@ -255,7 +259,7 @@ public void testUserAuthentication() { properties.put("location", HDFS_URL + "/user/hadoop/"); properties.put("gravitino.bypass.dfs.namenode.kerberos.principal.pattern", "*"); - kerberosHiveContainer.executeInContainer("hadoop", "fs", "-mkdir", "/user/hadoop"); + hdfs.mkdirs("/user/hadoop"); Catalog catalog = gravitinoMetalake.createCatalog( @@ -286,7 +290,7 @@ public void testUserAuthentication() { CATALOG_NAME, CatalogChange.setProperty(PRINCIPAL_KEY, HADOOP_CLIENT_PRINCIPAL)); // Now try to give the user the permission to create schema again - kerberosHiveContainer.executeInContainer("hadoop", "fs", "-chmod", "-R", "777", "/user/hadoop"); + hdfs.setPermission("/user/hadoop", "777", true); Assertions.assertDoesNotThrow( () -> catalog.asSchemas().createSchema(SCHEMA_NAME, "comment", ImmutableMap.of())); @@ -342,8 +346,7 @@ void testCreateSchemaWithKerberos() { properties.put("location", location); properties.put("gravitino.bypass.dfs.namenode.kerberos.principal.pattern", "*"); - kerberosHiveContainer.executeInContainer( - "hadoop", "fs", "-mkdir", "-p", "/user/hadoop/" + catalogName); + hdfs.mkdirs("/user/hadoop/" + catalogName); Catalog catalog = gravitinoMetalake.createCatalog( @@ -387,8 +390,7 @@ void testCreateSchemaWithKerberos() { exceptionMessage.contains("Permission denied: user=gravitino_client, access=WRITE")); // Now try to give the user the permission to create schema again - kerberosHiveContainer.executeInContainer( - "hadoop", "fs", "-chown", "gravitino_client", "/user/hadoop/" + catalogName); + hdfs.setOwner("/user/hadoop/" + catalogName, "gravitino_client", false); Assertions.assertDoesNotThrow( () -> catalog.asSchemas().createSchema(SCHEMA_NAME, "comment", schemaProperty)); @@ -406,8 +408,7 @@ void testCreateSchemaWithKerberos() { // END of test schema creation Assertions.assertDoesNotThrow(() -> catalog.asSchemas().dropSchema(SCHEMA_NAME, true)); - kerberosHiveContainer.executeInContainer( - "hadoop", "fs", "-chown", "cli_schema", "/user/hadoop/" + catalogName); + hdfs.setOwner("/user/hadoop/" + catalogName, "cli_schema", false); Assertions.assertDoesNotThrow( () -> catalog.asSchemas().createSchema(SCHEMA_NAME, "comment", schemaProperty)); @@ -446,8 +447,7 @@ void createFilesetWithKerberos() { properties.put("location", location); properties.put("gravitino.bypass.dfs.namenode.kerberos.principal.pattern", "*"); - kerberosHiveContainer.executeInContainer( - "hadoop", "fs", "-mkdir", "-p", "/user/hadoop/" + catalogName); + hdfs.mkdirs("/user/hadoop/" + catalogName); Catalog catalog = gravitinoMetalake.createCatalog( @@ -460,8 +460,7 @@ void createFilesetWithKerberos() { schemaProperty.put(KEY_TAB_URI_KEY, TMP_DIR + HADOOP_SCHEMA_KEYTAB); schemaProperty.put(PRINCIPAL_KEY, HADOOP_SCHEMA_PRINCIPAL + "@" + REALM); - kerberosHiveContainer.executeInContainer( - "hadoop", "fs", "-chown", "-R", "cli_schema", "/user/hadoop/" + catalogName); + hdfs.setOwner("/user/hadoop/" + catalogName, "cli_schema", true); Assertions.assertDoesNotThrow( () -> catalog.asSchemas().createSchema(SCHEMA_NAME, "comment", schemaProperty)); @@ -499,8 +498,7 @@ void createFilesetWithKerberos() { exceptionMessage.contains("Permission denied: user=cli_fileset, access=WRITE")); // Now change the owner of schema directory to 'cli_fileset' - kerberosHiveContainer.executeInContainer( - "hadoop", "fs", "-chown", "-R", "cli_fileset", "/user/hadoop/" + catalogName); + hdfs.setOwner("/user/hadoop/" + catalogName, "cli_fileset", true); Assertions.assertDoesNotThrow( () -> catalog @@ -532,8 +530,7 @@ void createFilesetWithKerberos() { exceptionMessage.contains("Permission denied: user=gravitino_client, access=WRITE")); // Now change the owner of schema directory to 'gravitino_client' - kerberosHiveContainer.executeInContainer( - "hadoop", "fs", "-chown", "-R", "gravitino_client", "/user/hadoop/" + catalogName); + hdfs.setOwner("/user/hadoop/" + catalogName, "gravitino_client", true); Assertions.assertDoesNotThrow( () -> catalog @@ -550,8 +547,7 @@ void createFilesetWithKerberos() { Assertions.assertThrows( Exception.class, () -> catalog.asFilesetCatalog().dropFileset(NameIdentifier.of(SCHEMA_NAME, fileset1))); - kerberosHiveContainer.executeInContainer( - "hadoop", "fs", "-chown", "-R", "cli_fileset", "/user/hadoop/" + catalogName); + hdfs.setOwner("/user/hadoop/" + catalogName, "cli_fileset", true); // Now try to drop fileset Assertions.assertDoesNotThrow( () -> catalog.asFilesetCatalog().dropFileset(NameIdentifier.of(SCHEMA_NAME, fileset1))); @@ -559,15 +555,13 @@ void createFilesetWithKerberos() { Assertions.assertThrows( Exception.class, () -> catalog.asFilesetCatalog().dropFileset(NameIdentifier.of(SCHEMA_NAME, fileset2))); - kerberosHiveContainer.executeInContainer( - "hadoop", "fs", "-chown", "-R", "gravitino_client", "/user/hadoop/" + catalogName); + hdfs.setOwner("/user/hadoop/" + catalogName, "gravitino_client", true); Assertions.assertDoesNotThrow( () -> catalog.asFilesetCatalog().dropFileset(NameIdentifier.of(SCHEMA_NAME, fileset2))); Assertions.assertThrows( Exception.class, () -> catalog.asSchemas().dropSchema(SCHEMA_NAME, true)); - kerberosHiveContainer.executeInContainer( - "hadoop", "fs", "-chown", "-R", "cli_schema", "/user/hadoop/" + catalogName); + hdfs.setOwner("/user/hadoop/" + catalogName, "cli_schema", true); Assertions.assertDoesNotThrow(() -> catalog.asSchemas().dropSchema(SCHEMA_NAME, true)); } @@ -596,8 +590,7 @@ void testUserImpersonation() { properties.put("location", localtion); properties.put("gravitino.bypass.dfs.namenode.kerberos.principal.pattern", "*"); - kerberosHiveContainer.executeInContainer( - "hadoop", "fs", "-mkdir", "-p", "/user/hadoop/" + catalogName); + hdfs.mkdirs("/user/hadoop/" + catalogName); Catalog catalog = gravitinoMetalake.createCatalog( @@ -628,8 +621,7 @@ void testUserImpersonation() { exceptionMessage = Throwables.getStackTraceAsString(exception); Assertions.assertTrue(exceptionMessage.contains("Permission denied: user=cli, access=WRITE")); - kerberosHiveContainer.executeInContainer( - "hadoop", "fs", "-chown", "-R", "cli", "/user/hadoop/" + catalogName); + hdfs.setOwner("/user/hadoop/" + catalogName, "cli", true); Assertions.assertDoesNotThrow( () -> catalog.asSchemas().createSchema(SCHEMA_NAME, "comment", schemaProperty)); diff --git a/catalogs/catalog-hive/build.gradle.kts b/catalogs/catalog-hive/build.gradle.kts index a893a4afa0b..d3cfb6977a7 100644 --- a/catalogs/catalog-hive/build.gradle.kts +++ b/catalogs/catalog-hive/build.gradle.kts @@ -142,6 +142,10 @@ dependencies { testImplementation(libs.testcontainers.localstack) testImplementation(libs.testcontainers.mysql) + // The LocalStack fixture creates buckets with the S3 SDK v2; the synchronous client needs no Netty. + testRuntimeOnly(libs.aws.s3) { + exclude("software.amazon.awssdk", "netty-nio-client") + } testRuntimeOnly(libs.junit.jupiter.engine) } diff --git a/catalogs/catalog-hive/src/test/java/org/apache/gravitino/catalog/hive/integration/test/CatalogHiveS3IT.java b/catalogs/catalog-hive/src/test/java/org/apache/gravitino/catalog/hive/integration/test/CatalogHiveS3IT.java index 2ef3d2e6a22..7165605ee8a 100644 --- a/catalogs/catalog-hive/src/test/java/org/apache/gravitino/catalog/hive/integration/test/CatalogHiveS3IT.java +++ b/catalogs/catalog-hive/src/test/java/org/apache/gravitino/catalog/hive/integration/test/CatalogHiveS3IT.java @@ -23,23 +23,16 @@ import java.net.URI; import java.util.HashMap; import java.util.Map; -import java.util.concurrent.TimeUnit; import org.apache.gravitino.integration.test.container.GravitinoLocalStackContainer; import org.apache.gravitino.integration.test.container.HiveContainer; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.spark.sql.SparkSession; -import org.awaitility.Awaitility; import org.junit.jupiter.api.TestInstance; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.testcontainers.containers.Container; @TestInstance(TestInstance.Lifecycle.PER_CLASS) public class CatalogHiveS3IT extends CatalogHive3IT { - private static final Logger LOGGER = LoggerFactory.getLogger(CatalogHiveS3IT.class); - private static final String S3_BUCKET_NAME = "my-test-bucket"; private GravitinoLocalStackContainer gravitinoLocalStackContainer; @@ -57,49 +50,16 @@ protected void startNecessaryContainer() { containerSuite.startLocalStackContainer(); gravitinoLocalStackContainer = containerSuite.getLocalStackContainer(); - Awaitility.await() - .atMost(60, TimeUnit.SECONDS) - .pollInterval(1, TimeUnit.SECONDS) - .until( - () -> { - try { - Container.ExecResult result = - gravitinoLocalStackContainer.executeInContainer( - "awslocal", "iam", "create-user", "--user-name", "anonymous"); - return result.getExitCode() == 0; - } catch (Exception e) { - LOGGER.info("LocalStack is not ready yet for: ", e); - return false; - } - }); - - gravitinoLocalStackContainer.executeInContainer( - "awslocal", "s3", "mb", "s3://" + S3_BUCKET_NAME); - - Container.ExecResult result = - gravitinoLocalStackContainer.executeInContainer( - "awslocal", "iam", "create-access-key", "--user-name", "anonymous"); - - // Get access key and secret key from result - String[] lines = result.getStdout().split("\n"); - accessKey = lines[3].split(":")[1].trim().substring(1, 21); - secretKey = lines[5].split(":")[1].trim().substring(1, 41); - - LOGGER.info("Access key: " + accessKey); - LOGGER.info("Secret key: " + secretKey); + // The container is ready once its health check passes. LocalStack does not enforce IAM, so + // the shared test keys work without creating a user. + gravitinoLocalStackContainer.createBucket(S3_BUCKET_NAME); + gravitinoLocalStackContainer.makeBucketPublic(S3_BUCKET_NAME); + accessKey = GravitinoLocalStackContainer.ACCESS_KEY; + secretKey = GravitinoLocalStackContainer.SECRET_KEY; getS3Endpoint = String.format("http://%s:%d", gravitinoLocalStackContainer.getContainerIpAddress(), 4566); - gravitinoLocalStackContainer.executeInContainer( - "awslocal", - "s3api", - "put-bucket-acl", - "--bucket", - "my-test-bucket", - "--acl", - "public-read-write"); - Map hiveContainerEnv = ImmutableMap.of( S3_ACCESS_KEY, diff --git a/catalogs/catalog-hive/src/test/java/org/apache/gravitino/catalog/hive/integration/test/HiveUserAuthenticationIT.java b/catalogs/catalog-hive/src/test/java/org/apache/gravitino/catalog/hive/integration/test/HiveUserAuthenticationIT.java index 23a23772b90..0bbf3e69ac3 100644 --- a/catalogs/catalog-hive/src/test/java/org/apache/gravitino/catalog/hive/integration/test/HiveUserAuthenticationIT.java +++ b/catalogs/catalog-hive/src/test/java/org/apache/gravitino/catalog/hive/integration/test/HiveUserAuthenticationIT.java @@ -45,6 +45,7 @@ import org.apache.gravitino.integration.test.container.HiveContainer; import org.apache.gravitino.integration.test.util.BaseIT; import org.apache.gravitino.integration.test.util.GravitinoITUtils; +import org.apache.gravitino.integration.test.util.KerberosHdfs; import org.apache.gravitino.rel.Column; import org.apache.gravitino.rel.TableChange; import org.apache.gravitino.rel.expressions.distributions.Distributions; @@ -88,6 +89,8 @@ public class HiveUserAuthenticationIT extends BaseIT { private static GravitinoAdminClient adminClient; + private static KerberosHdfs hdfs; + protected static HiveContainer kerberosHiveContainer; static String METALAKE_NAME = GravitinoITUtils.genRandomName("test_metalake"); @@ -116,6 +119,7 @@ public void startIntegrationTest() throws Exception { // Prepare kerberos related-config; prepareKerberosConfig(); + hdfs = KerberosHdfs.of(kerberosHiveContainer, TMP_DIR + HIVE_METASTORE_CLIENT_KEYTAB); // Config kerberos configuration for Gravitino server addKerberosConfig(); @@ -126,6 +130,11 @@ public void startIntegrationTest() throws Exception { @AfterAll public void stop() { + // Restore the warehouse first: the HDFS client needs the Kerberos settings cleared below. + if (hdfs != null) { + hdfs.setPermission("/user/hive/warehouse", "755", true); + } + // Reset the UGI UserGroupInformation.reset(); @@ -236,8 +245,7 @@ public void testUserAuthentication() { exceptionMessage.contains("Permission denied: user=gravitino_client, access=WRITE")); // Now try to give the user the permission to create schema again - kerberosHiveContainer.executeInContainer( - "hadoop", "fs", "-chmod", "-R", "777", "/user/hive/warehouse"); + hdfs.setPermission("/user/hive/warehouse", "777", true); Assertions.assertDoesNotThrow( () -> catalog.asSchemas().createSchema(SCHEMA_NAME, "comment", ImmutableMap.of())); @@ -274,12 +282,6 @@ public void testUserAuthentication() { Assertions.assertTrue(gravitinoMetalake.dropCatalog(CATALOG_NAME, true)); } - @AfterAll - static void restoreFilePermission() { - kerberosHiveContainer.executeInContainer( - "hadoop", "fs", "-chmod", "-R", "755", "/user/hive/warehouse"); - } - private static Column[] createColumns() { Column col1 = Column.of(HIVE_COL_NAME1, Types.ByteType.get(), "col_1_comment"); Column col2 = Column.of(HIVE_COL_NAME2, Types.DateType.get(), "col_2_comment"); diff --git a/catalogs/catalog-lakehouse-hudi/src/test/java/org/apache/gravitino/catalog/lakehouse/hudi/integration/test/HudiCatalogKerberosHiveIT.java b/catalogs/catalog-lakehouse-hudi/src/test/java/org/apache/gravitino/catalog/lakehouse/hudi/integration/test/HudiCatalogKerberosHiveIT.java index a692051acb3..6c6da59f2ed 100644 --- a/catalogs/catalog-lakehouse-hudi/src/test/java/org/apache/gravitino/catalog/lakehouse/hudi/integration/test/HudiCatalogKerberosHiveIT.java +++ b/catalogs/catalog-lakehouse-hudi/src/test/java/org/apache/gravitino/catalog/lakehouse/hudi/integration/test/HudiCatalogKerberosHiveIT.java @@ -104,9 +104,6 @@ public void startIntegrationTest() throws Exception { // Config kerberos configuration for Gravitino server addKerberosConfig(); - // Create hive tables - createHudiTables(); - // Start Gravitino server super.startIntegrationTest(); } @@ -226,6 +223,8 @@ public void testHudiCatalogWithKerberos() { Schema schema = catalog.asSchemas().loadSchema(SCHEMA_NAME); Assertions.assertEquals(SCHEMA_NAME, schema.name()); + // TABLE_NAME is never created, so no Hive SQL setup is needed: the catalog must report the + // table as missing from the metastore, not as a non-Hudi table. TableCatalog tableOps = catalog.asTableCatalog(); NoSuchTableException exception = Assertions.assertThrows( @@ -238,23 +237,4 @@ public void testHudiCatalogWithKerberos() { .contains("Hive table " + TABLE_NAME + " does not exist in Hive Metastore"), "Unexpected exception message: " + exception.getMessage()); } - - private static void createHudiTables() { - String createTableSql = - String.format( - "CREATE TABLE %s.%s (\n" - + " id STRING,\n" - + " name STRING,\n" - + " age INT\n" - + ") \n" - + "LOCATION 'hdfs://localhost:9000/user/hive/warehouse-catalog-hudi' ", - SCHEMA_NAME, TABLE_NAME); - kerberosHiveContainer.executeInContainer("hive", "-e", "'" + createTableSql + ";'"); - LOG.info("create table {} success[{}]", TABLE_NAME, createTableSql); - - String insertTableSql = - String.format("INSERT INTO %s.%s" + " VALUES " + "(1,'Cyber',26)", SCHEMA_NAME, TABLE_NAME); - kerberosHiveContainer.executeInContainer("hive", "-e", "'" + insertTableSql + ";'"); - LOG.info("insert table {} data success[{}]", TABLE_NAME, insertTableSql); - } } diff --git a/catalogs/catalog-lakehouse-iceberg/src/test/java/org/apache/gravitino/catalog/lakehouse/iceberg/integration/test/CatalogIcebergKerberosHiveIT.java b/catalogs/catalog-lakehouse-iceberg/src/test/java/org/apache/gravitino/catalog/lakehouse/iceberg/integration/test/CatalogIcebergKerberosHiveIT.java index 65cf8c6bd60..21f41c4d1d4 100644 --- a/catalogs/catalog-lakehouse-iceberg/src/test/java/org/apache/gravitino/catalog/lakehouse/iceberg/integration/test/CatalogIcebergKerberosHiveIT.java +++ b/catalogs/catalog-lakehouse-iceberg/src/test/java/org/apache/gravitino/catalog/lakehouse/iceberg/integration/test/CatalogIcebergKerberosHiveIT.java @@ -45,6 +45,7 @@ import org.apache.gravitino.integration.test.container.HiveContainer; import org.apache.gravitino.integration.test.util.BaseIT; import org.apache.gravitino.integration.test.util.GravitinoITUtils; +import org.apache.gravitino.integration.test.util.KerberosHdfs; import org.apache.gravitino.rel.Column; import org.apache.gravitino.rel.TableChange; import org.apache.gravitino.rel.expressions.distributions.Distributions; @@ -87,6 +88,8 @@ public class CatalogIcebergKerberosHiveIT extends BaseIT { private static GravitinoAdminClient adminClient; + private static KerberosHdfs hdfs; + private static final String METALAKE_NAME = GravitinoITUtils.genRandomName("test_metalake"); private static final String CATALOG_NAME = GravitinoITUtils.genRandomName("test_catalog"); private static final String SCHEMA_NAME = GravitinoITUtils.genRandomName("test_schema"); @@ -123,6 +126,7 @@ public void startIntegrationTest() { // Prepare kerberos related-config; prepareKerberosConfig(); + hdfs = KerberosHdfs.of(kerberosHiveContainer, TMP_DIR + HIVE_METASTORE_CLIENT_KEYTAB); // Config kerberos configuration for Gravitino server addKerberosConfig(); @@ -255,15 +259,12 @@ void testIcebergWithKerberosAndUserImpersonation() throws IOException { exceptionMessage.contains("Permission denied: user=gravitino_client, access=WRITE")); // Now try to permit the user to create the schema again - kerberosHiveContainer.executeInContainer( - "hadoop", "fs", "-mkdir", "/user/hive/warehouse-catalog-iceberg"); - kerberosHiveContainer.executeInContainer( - "hadoop", "fs", "-chmod", "-R", "777", "/user/hive/warehouse-catalog-iceberg"); + hdfs.mkdirs("/user/hive/warehouse-catalog-iceberg"); + hdfs.setPermission("/user/hive/warehouse-catalog-iceberg", "777", true); Assertions.assertDoesNotThrow( () -> catalog.asSchemas().createSchema(SCHEMA_NAME, "comment", ImmutableMap.of())); - kerberosHiveContainer.executeInContainer( - "hadoop", "fs", "-chmod", "-R", "000", "/user/hive/warehouse-catalog-iceberg"); + hdfs.setPermission("/user/hive/warehouse-catalog-iceberg", "000", true); // Create a table, and the user has no EXECUTE permission to create table NameIdentifier tableNameIdentifier1 = NameIdentifier.of(SCHEMA_NAME, TABLE_NAME); @@ -287,8 +288,7 @@ void testIcebergWithKerberosAndUserImpersonation() throws IOException { exceptionMessage.contains("Permission denied: user=gravitino_client, access=EXECUTE")); // Now try to permit the user to create the table again - kerberosHiveContainer.executeInContainer( - "hadoop", "fs", "-chmod", "-R", "777", "/user/hive/warehouse-catalog-iceberg"); + hdfs.setPermission("/user/hive/warehouse-catalog-iceberg", "777", true); // Create table NameIdentifier tableNameIdentifier = NameIdentifier.of(SCHEMA_NAME, TABLE_NAME); catalog diff --git a/catalogs/catalog-lakehouse-paimon/build.gradle.kts b/catalogs/catalog-lakehouse-paimon/build.gradle.kts index 7b34b2812fa..be9f6a7becf 100644 --- a/catalogs/catalog-lakehouse-paimon/build.gradle.kts +++ b/catalogs/catalog-lakehouse-paimon/build.gradle.kts @@ -220,6 +220,10 @@ dependencies { // Keep optional Hadoop S3A dependencies available only to tests. s3aTestRuntime(libs.hadoop3.aws) + // The LocalStack fixture creates buckets with the S3 SDK v2; the synchronous client needs no Netty. + testRuntimeOnly(libs.aws.s3) { + exclude("software.amazon.awssdk", "netty-nio-client") + } testRuntimeOnly(libs.junit.jupiter.engine) } diff --git a/catalogs/catalog-lakehouse-paimon/src/test/java/org/apache/gravitino/catalog/lakehouse/paimon/integration/test/CatalogPaimonKerberosFilesystemIT.java b/catalogs/catalog-lakehouse-paimon/src/test/java/org/apache/gravitino/catalog/lakehouse/paimon/integration/test/CatalogPaimonKerberosFilesystemIT.java index e10caa5005d..16f9a94abc9 100644 --- a/catalogs/catalog-lakehouse-paimon/src/test/java/org/apache/gravitino/catalog/lakehouse/paimon/integration/test/CatalogPaimonKerberosFilesystemIT.java +++ b/catalogs/catalog-lakehouse-paimon/src/test/java/org/apache/gravitino/catalog/lakehouse/paimon/integration/test/CatalogPaimonKerberosFilesystemIT.java @@ -42,6 +42,7 @@ import org.apache.gravitino.integration.test.container.HiveContainer; import org.apache.gravitino.integration.test.util.BaseIT; import org.apache.gravitino.integration.test.util.GravitinoITUtils; +import org.apache.gravitino.integration.test.util.KerberosHdfs; import org.apache.gravitino.rel.Column; import org.apache.gravitino.rel.Table; import org.apache.gravitino.rel.expressions.distributions.Distributions; @@ -85,6 +86,8 @@ public class CatalogPaimonKerberosFilesystemIT extends BaseIT { private static GravitinoAdminClient adminClient; + private static KerberosHdfs hdfs; + private static final String CATALOG_NAME = GravitinoITUtils.genRandomName("test_catalog"); private static final String SCHEMA_NAME = GravitinoITUtils.genRandomName("test_schema"); private static final String TABLE_NAME = GravitinoITUtils.genRandomName("test_table"); @@ -115,6 +118,7 @@ public void startIntegrationTest() { // Prepare kerberos related-config; prepareKerberosConfig(); + hdfs = KerberosHdfs.of(kerberosHiveContainer, TMP_DIR + HDFS_CLIENT_KEYTAB); // Config kerberos configuration for Gravitino server addKerberosConfig(); @@ -238,10 +242,8 @@ void testPaimonWithKerberos() { Assertions.assertTrue(exceptionMessage.contains("Permission denied: user=cli, access=WRITE")); // Now try to permit the user to create the schema again - kerberosHiveContainer.executeInContainer( - "hadoop", "fs", "-mkdir", "/user/hive/paimon_catalog_warehouse"); - kerberosHiveContainer.executeInContainer( - "hadoop", "fs", "-chmod", "-R", "777", "/user/hive/paimon_catalog_warehouse"); + hdfs.mkdirs("/user/hive/paimon_catalog_warehouse"); + hdfs.setPermission("/user/hive/paimon_catalog_warehouse", "777", true); Assertions.assertDoesNotThrow( () -> catalog.asSchemas().createSchema(SCHEMA_NAME, "comment", ImmutableMap.of())); @@ -274,8 +276,7 @@ void testPaimonWithKerberos() { Assertions.assertTrue(gravitinoMetalake.dropCatalog(CATALOG_NAME, true)); // Drop warehouse path - kerberosHiveContainer.executeInContainer( - "hadoop", "fs", "-rm", "-r", "/user/hive/paimon_catalog_warehouse"); + hdfs.delete("/user/hive/paimon_catalog_warehouse", true); } private static Column[] createColumns() { diff --git a/catalogs/catalog-lakehouse-paimon/src/test/java/org/apache/gravitino/catalog/lakehouse/paimon/integration/test/CatalogPaimonS3AIT.java b/catalogs/catalog-lakehouse-paimon/src/test/java/org/apache/gravitino/catalog/lakehouse/paimon/integration/test/CatalogPaimonS3AIT.java index 969c4e7e635..941d471a19a 100644 --- a/catalogs/catalog-lakehouse-paimon/src/test/java/org/apache/gravitino/catalog/lakehouse/paimon/integration/test/CatalogPaimonS3AIT.java +++ b/catalogs/catalog-lakehouse-paimon/src/test/java/org/apache/gravitino/catalog/lakehouse/paimon/integration/test/CatalogPaimonS3AIT.java @@ -24,7 +24,6 @@ import java.util.Collections; import java.util.Map; import java.util.UUID; -import java.util.concurrent.TimeUnit; import org.apache.gravitino.Catalog; import org.apache.gravitino.Schema; import org.apache.gravitino.catalog.lakehouse.paimon.PaimonCatalogPropertiesMetadata; @@ -32,13 +31,11 @@ import org.apache.gravitino.integration.test.container.GravitinoLocalStackContainer; import org.apache.gravitino.integration.test.util.BaseIT; import org.apache.gravitino.integration.test.util.GravitinoITUtils; -import org.awaitility.Awaitility; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; -import org.testcontainers.containers.Container; /** Integration test for accessing a Paimon filesystem catalog through Hadoop S3A. */ @Tag("gravitino-docker-test") @@ -62,20 +59,8 @@ void setUp() { containerSuite.startLocalStackContainer(); localStackContainer = containerSuite.getLocalStackContainer(); - Awaitility.await() - .atMost(60, TimeUnit.SECONDS) - .pollInterval(1, TimeUnit.SECONDS) - .until( - () -> { - try { - Container.ExecResult result = - localStackContainer.executeInContainer( - "awslocal", "s3", "mb", "s3://" + bucketName); - return result.getExitCode() == 0; - } catch (Exception e) { - return false; - } - }); + // The container is ready once its health check passes. + localStackContainer.createBucket(bucketName); metalake = client.createMetalake(metalakeName, "Paimon S3A metalake", Collections.emptyMap()); } diff --git a/catalogs/catalog-lakehouse-paimon/src/test/java/org/apache/gravitino/catalog/lakehouse/paimon/integration/test/CatalogPaimonS3IT.java b/catalogs/catalog-lakehouse-paimon/src/test/java/org/apache/gravitino/catalog/lakehouse/paimon/integration/test/CatalogPaimonS3IT.java index 9a5ac8e64d7..012e1ea920e 100644 --- a/catalogs/catalog-lakehouse-paimon/src/test/java/org/apache/gravitino/catalog/lakehouse/paimon/integration/test/CatalogPaimonS3IT.java +++ b/catalogs/catalog-lakehouse-paimon/src/test/java/org/apache/gravitino/catalog/lakehouse/paimon/integration/test/CatalogPaimonS3IT.java @@ -21,16 +21,13 @@ import com.google.common.collect.Maps; import java.util.Map; -import java.util.concurrent.TimeUnit; import org.apache.gravitino.catalog.lakehouse.paimon.PaimonCatalogPropertiesMetadata; import org.apache.gravitino.integration.test.container.GravitinoLocalStackContainer; import org.apache.gravitino.integration.test.util.DownloaderUtils; import org.apache.gravitino.integration.test.util.ITUtils; import org.apache.gravitino.storage.S3Properties; import org.apache.spark.sql.SparkSession; -import org.awaitility.Awaitility; import org.junit.jupiter.api.Tag; -import org.testcontainers.containers.Container; @Tag("gravitino-docker-test") public class CatalogPaimonS3IT extends CatalogPaimonBaseIT { @@ -88,20 +85,8 @@ protected void startNecessaryContainers() { containerSuite.startLocalStackContainer(); localStackContainer = containerSuite.getLocalStackContainer(); - Awaitility.await() - .atMost(60, TimeUnit.SECONDS) - .pollInterval(1, TimeUnit.SECONDS) - .until( - () -> { - try { - Container.ExecResult result = - localStackContainer.executeInContainer( - "awslocal", "s3", "mb", "s3://" + S3_BUCKET_NAME); - return result.getExitCode() == 0; - } catch (Exception e) { - return false; - } - }); + // The container is ready once its health check passes. + localStackContainer.createBucket(S3_BUCKET_NAME); } protected void initSparkEnv() { diff --git a/clients/filesystem-hadoop3/build.gradle.kts b/clients/filesystem-hadoop3/build.gradle.kts index 404ea2f8b5d..be68b6e5af1 100644 --- a/clients/filesystem-hadoop3/build.gradle.kts +++ b/clients/filesystem-hadoop3/build.gradle.kts @@ -85,6 +85,10 @@ dependencies { testImplementation(libs.postgresql.driver) testImplementation(libs.testcontainers) + // The LocalStack fixture creates buckets with the S3 SDK v2; the synchronous client needs no Netty. + testRuntimeOnly(libs.aws.s3) { + exclude("software.amazon.awssdk", "netty-nio-client") + } testRuntimeOnly(libs.junit.jupiter.engine) } diff --git a/clients/filesystem-hadoop3/src/test/java/org/apache/gravitino/filesystem/hadoop/integration/test/GravitinoVirtualFileSystemS3IT.java b/clients/filesystem-hadoop3/src/test/java/org/apache/gravitino/filesystem/hadoop/integration/test/GravitinoVirtualFileSystemS3IT.java index cdd0d3eb32e..416924d625d 100644 --- a/clients/filesystem-hadoop3/src/test/java/org/apache/gravitino/filesystem/hadoop/integration/test/GravitinoVirtualFileSystemS3IT.java +++ b/clients/filesystem-hadoop3/src/test/java/org/apache/gravitino/filesystem/hadoop/integration/test/GravitinoVirtualFileSystemS3IT.java @@ -26,7 +26,6 @@ import java.util.Collections; import java.util.Map; import java.util.UUID; -import java.util.concurrent.TimeUnit; import org.apache.gravitino.Catalog; import org.apache.gravitino.catalog.hadoop.fs.FileSystemUtils; import org.apache.gravitino.integration.test.container.GravitinoLocalStackContainer; @@ -34,14 +33,12 @@ import org.apache.gravitino.s3.fs.S3FileSystemProvider; import org.apache.gravitino.storage.S3Properties; import org.apache.hadoop.conf.Configuration; -import org.awaitility.Awaitility; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Disabled; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.testcontainers.containers.Container; public class GravitinoVirtualFileSystemS3IT extends GravitinoVirtualFileSystemIT { private static final Logger LOG = LoggerFactory.getLogger(GravitinoVirtualFileSystemS3IT.class); @@ -62,44 +59,12 @@ private void startS3Mocker() { containerSuite.startLocalStackContainer(); gravitinoLocalStackContainer = containerSuite.getLocalStackContainer(); - Awaitility.await() - .atMost(60, TimeUnit.SECONDS) - .pollInterval(1, TimeUnit.SECONDS) - .until( - () -> { - try { - Container.ExecResult result = - gravitinoLocalStackContainer.executeInContainer( - "awslocal", "iam", "create-user", "--user-name", "anonymous"); - return result.getExitCode() == 0; - } catch (Exception e) { - LOG.info("LocalStack is not ready yet for: ", e); - return false; - } - }); - - gravitinoLocalStackContainer.executeInContainer("awslocal", "s3", "mb", "s3://" + bucketName); - - Container.ExecResult result = - gravitinoLocalStackContainer.executeInContainer( - "awslocal", "iam", "create-access-key", "--user-name", "anonymous"); - - gravitinoLocalStackContainer.executeInContainer( - "awslocal", - "s3api", - "put-bucket-acl", - "--bucket", - "my-test-bucket", - "--acl", - "public-read-write"); - - // Get access key and secret key from result - String[] lines = result.getStdout().split("\n"); - accessKey = lines[3].split(":")[1].trim().substring(1, 21); - secretKey = lines[5].split(":")[1].trim().substring(1, 41); - - LOG.info("Access key: " + accessKey); - LOG.info("Secret key: " + secretKey); + // The container is ready once its health check passes. LocalStack does not enforce IAM, so + // the shared test keys work without creating a user. + gravitinoLocalStackContainer.createBucket(bucketName); + gravitinoLocalStackContainer.makeBucketPublic(bucketName); + accessKey = GravitinoLocalStackContainer.ACCESS_KEY; + secretKey = GravitinoLocalStackContainer.SECRET_KEY; s3Endpoint = String.format("http://%s:%d", gravitinoLocalStackContainer.getContainerIpAddress(), 4566); diff --git a/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRestKerberosHiveWithUserImpersonationCatalogIT.java b/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRestKerberosHiveWithUserImpersonationCatalogIT.java index f4209305e2f..b9bc4e0988b 100644 --- a/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRestKerberosHiveWithUserImpersonationCatalogIT.java +++ b/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRestKerberosHiveWithUserImpersonationCatalogIT.java @@ -48,9 +48,8 @@ void prepareSQLContext() { // change the ownership, the normal user will not have the permission to create table in Hive // as the /user/hive is owned by user `cli`, please see what's done in `initEnv` method in // superclass. - containerSuite - .getKerberosHiveContainer() - .executeInContainer("hadoop", "fs", "-chown", "-R", NORMAL_USER, "/user/hive/"); + IcebergRestKerberosTestEnv.hdfs(containerSuite.getKerberosHiveContainer()) + .setOwner("/user/hive/", NORMAL_USER, true); super.prepareSQLContext(); } diff --git a/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRestKerberosJdbcWithUserImpersonationCatalogIT.java b/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRestKerberosJdbcWithUserImpersonationCatalogIT.java index 72d2b0b038e..2e7dc4875a6 100644 --- a/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRestKerberosJdbcWithUserImpersonationCatalogIT.java +++ b/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRestKerberosJdbcWithUserImpersonationCatalogIT.java @@ -45,9 +45,8 @@ void prepareSQLContext() { // Change the ownership of /user/hive to normal user for user impersonation test. If we do not // change the ownership, the normal user will not have the permission to create table on HDFS // warehouse as the /user/hive is owned by user `cli`. - containerSuite - .getKerberosHiveContainer() - .executeInContainer("hadoop", "fs", "-chown", "-R", NORMAL_USER, "/user/hive/"); + IcebergRestKerberosTestEnv.hdfs(containerSuite.getKerberosHiveContainer()) + .setOwner("/user/hive/", NORMAL_USER, true); super.prepareSQLContext(); } diff --git a/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRestKerberosTestEnv.java b/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRestKerberosTestEnv.java index abf24cd04bb..7bb9a5ef992 100644 --- a/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRestKerberosTestEnv.java +++ b/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRestKerberosTestEnv.java @@ -29,6 +29,7 @@ import org.apache.gravitino.integration.test.container.ContainerSuite; import org.apache.gravitino.integration.test.container.HiveContainer; import org.apache.gravitino.integration.test.util.ITUtils; +import org.apache.gravitino.integration.test.util.KerberosHdfs; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.security.UserGroupInformation; import org.apache.spark.SparkConf; @@ -170,10 +171,8 @@ public static String init(ContainerSuite containerSuite) { refreshKerberosConfig(); resetDefaultRealm(); - kerberosHiveContainer.executeInContainer( - "hadoop", "fs", "-chown", "-R", "cli", "/user/hive/"); - IcebergRestKerberosTestEnv.tempDir = tempDir; + hdfs(kerberosHiveContainer).setOwner("/user/hive/", "cli", true); return tempDir; } catch (Exception e) { reset(); @@ -181,6 +180,20 @@ public static String init(ContainerSuite containerSuite) { } } + /** + * Returns an HDFS client that logs in with the keytab {@link #init(ContainerSuite)} copied. + * + * @param kerberosHiveContainer the running Kerberos Hive container + * @return the client + */ + public static KerberosHdfs hdfs(HiveContainer kerberosHiveContainer) { + if (tempDir == null) { + throw new IllegalStateException( + "Kerberos test environment is not initialized, call init() first"); + } + return KerberosHdfs.of(kerberosHiveContainer, tempDir + CLIENT_KEYTAB); + } + /** Restores JVM Kerberos system properties changed by {@link #init(ContainerSuite)}. */ public static void reset() { if (initRefCount == 0) { diff --git a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/DatabaseContainer.java b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/DatabaseContainer.java index d668b9384fd..bece1defb1e 100644 --- a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/DatabaseContainer.java +++ b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/DatabaseContainer.java @@ -18,6 +18,10 @@ */ package org.apache.gravitino.integration.test.container; +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; import java.util.List; import java.util.Map; import java.util.Optional; @@ -116,15 +120,24 @@ protected boolean checkContainerStatus(int retryLimit) { /** * Logs whether lower durability is enabled and the settings the server reports, so each run's * output shows which mode it ran in. Warns when lower durability is enabled but the data - * directory is not on {@code tmpfs}. The query must print tab-separated values, the data - * directory first, in the order of {@code names}. + * directory is not on {@code tmpfs}. The query must return one row with the data directory first, + * in the order of {@code names}. */ - protected void logDurability(List names, String... query) { + protected void logDurability(List names, String query) { String name = getClass().getSimpleName(); - Container.ExecResult settings = executeInContainer(query); - String[] values = settings.getStdout().trim().split("\t"); - if (settings.getExitCode() != 0 || values.length != names.size()) { - LOG.warn("{}: could not read the durability settings: {}", name, settings.getStderr()); + String[] values = new String[names.size()]; + try (Connection connection = db.createConnection(""); + Statement statement = connection.createStatement(); + ResultSet row = statement.executeQuery(query)) { + if (!row.next() || row.getMetaData().getColumnCount() != names.size()) { + LOG.warn("{}: could not read the durability settings from: {}", name, query); + return; + } + for (int i = 0; i < values.length; i++) { + values[i] = row.getString(i + 1); + } + } catch (SQLException e) { + LOG.warn("{}: could not read the durability settings: {}", name, e.getMessage()); return; } String report = diff --git a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/GravitinoLocalStackContainer.java b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/GravitinoLocalStackContainer.java index 11eae352503..e6749380c43 100644 --- a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/GravitinoLocalStackContainer.java +++ b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/GravitinoLocalStackContainer.java @@ -18,18 +18,51 @@ */ package org.apache.gravitino.integration.test.container; +import static java.lang.String.format; + import com.google.common.collect.ImmutableSet; +import java.net.URI; +import java.time.Duration; +import java.util.List; import java.util.Map; import java.util.Optional; import java.util.Set; +import java.util.regex.Pattern; +import java.util.stream.Collectors; import org.testcontainers.containers.Network; +import org.testcontainers.containers.wait.strategy.Wait; +import software.amazon.awssdk.auth.credentials.AwsBasicCredentials; +import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider; +import software.amazon.awssdk.regions.Region; +import software.amazon.awssdk.services.s3.S3Client; +import software.amazon.awssdk.services.s3.model.BucketAlreadyOwnedByYouException; +import software.amazon.awssdk.services.s3.model.BucketCannedACL; +import software.amazon.awssdk.services.s3.model.S3Object; +/** + * LocalStack for S3. It is ready once its health endpoint reports S3, and {@link + * #createBucket(String)} and the other S3 helpers call it through the AWS SDK, so tests need no + * {@code awslocal} in the container. Callers need {@code libs.aws.s3} on their test runtime + * classpath. + */ public class GravitinoLocalStackContainer extends BaseContainer { public static final String DEFAULT_IMAGE = System.getenv("GRAVITINO_CI_LOCALSTACK_DOCKER_IMAGE"); public static final String HOST_NAME = "gravitino-ci-localstack"; public static final int PORT = 4566; + /** + * The access key clients use. LocalStack does not enforce IAM here, so any key works; tests share + * this one instead of creating an IAM user. + */ + public static final String ACCESS_KEY = "test"; + + /** The secret key that goes with {@link #ACCESS_KEY}. */ + public static final String SECRET_KEY = "test"; + + // "available" means S3 starts on its first request; "running" means it has started. + private static final Pattern S3_READY = Pattern.compile("\"s3\"\\s*:\\s*\"(available|running)\""); + public GravitinoLocalStackContainer( String image, String hostName, @@ -45,11 +78,102 @@ public static Builder builder() { return new Builder(); } + @Override + protected void setupContainer() { + super.setupContainer(); + container.waitingFor( + Wait.forHttp("/_localstack/health") + .forPort(PORT) + .forStatusCode(200) + .forResponsePredicate(GravitinoLocalStackContainer::reportsS3Ready)); + } + @Override protected boolean checkContainerStatus(int retryLimit) { return true; } + /** + * Creates a bucket, or keeps it if this account already owns it. + * + * @param bucketName the bucket to create + */ + public void createBucket(String bucketName) { + S3Calls.createBucket(getS3Endpoint(), bucketName); + } + + /** + * Gives a bucket the {@code public-read-write} canned ACL. + * + * @param bucketName the bucket + */ + public void makeBucketPublic(String bucketName) { + S3Calls.makeBucketPublic(getS3Endpoint(), bucketName); + } + + /** + * Lists the keys of the objects in a bucket, at most one page. + * + * @param bucketName the bucket + * @return the object keys + */ + public List listObjectKeys(String bucketName) { + return S3Calls.listObjectKeys(getS3Endpoint(), bucketName); + } + + /** + * Returns the S3 endpoint reachable from the host through the published port. + * + * @return the endpoint, for example {@code http://localhost:32768} + */ + public String getS3Endpoint() { + return format("http://%s:%d", container.getHost(), getMappedPort(PORT)); + } + + static boolean reportsS3Ready(String healthBody) { + return S3_READY.matcher(healthBody).find(); + } + + // Keeps the SDK types out of the container class, so a module without the S3 SDK can still load + // the container as long as it does not call these. + private static final class S3Calls { + private S3Calls() {} + + static void createBucket(String endpoint, String bucketName) { + try (S3Client client = client(endpoint)) { + client.createBucket(request -> request.bucket(bucketName)); + } catch (BucketAlreadyOwnedByYouException e) { + LOG.info("LocalStack bucket {} already exists", bucketName); + } + } + + static void makeBucketPublic(String endpoint, String bucketName) { + try (S3Client client = client(endpoint)) { + client.putBucketAcl( + request -> request.bucket(bucketName).acl(BucketCannedACL.PUBLIC_READ_WRITE)); + } + } + + static List listObjectKeys(String endpoint, String bucketName) { + try (S3Client client = client(endpoint)) { + return client.listObjectsV2(request -> request.bucket(bucketName)).contents().stream() + .map(S3Object::key) + .collect(Collectors.toList()); + } + } + + private static S3Client client(String endpoint) { + return S3Client.builder() + .endpointOverride(URI.create(endpoint)) + .region(Region.US_EAST_1) + .forcePathStyle(true) + .credentialsProvider( + StaticCredentialsProvider.create(AwsBasicCredentials.create(ACCESS_KEY, SECRET_KEY))) + .overrideConfiguration(config -> config.apiCallTimeout(Duration.ofSeconds(30))) + .build(); + } + } + public static class Builder extends BaseContainer.Builder< GravitinoLocalStackContainer.Builder, GravitinoLocalStackContainer> { diff --git a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/MySQLContainer.java b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/MySQLContainer.java index 445f4be6c5f..b2069ede3d2 100644 --- a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/MySQLContainer.java +++ b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/MySQLContainer.java @@ -83,12 +83,6 @@ public void start() { super.start(); logDurability( List.of("datadir", "innodb_flush_log_at_trx_commit", "log_bin", "innodb_doublewrite"), - "mysql", - "-u" + getUsername(), - "-p" + getPassword(), - "--batch", - "--skip-column-names", - "-e", "SELECT @@datadir, @@innodb_flush_log_at_trx_commit, @@log_bin, @@innodb_doublewrite"); } diff --git a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/PostgreSQLContainer.java b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/PostgreSQLContainer.java index f510f04c351..c94b90f1579 100644 --- a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/PostgreSQLContainer.java +++ b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/PostgreSQLContainer.java @@ -96,15 +96,6 @@ public void start() { super.start(); logDurability( List.of("data_directory", "fsync", "synchronous_commit", "full_page_writes"), - "psql", - "-U", - getUsername(), - "-d", - "postgres", - "--no-align", - "--tuples-only", - "--field-separator=\t", - "-c", "SELECT current_setting('data_directory'), current_setting('fsync')," + " current_setting('synchronous_commit'), current_setting('full_page_writes')"); } diff --git a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/TestGravitinoLocalStackContainer.java b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/TestGravitinoLocalStackContainer.java new file mode 100644 index 00000000000..1fcab41d0b0 --- /dev/null +++ b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/TestGravitinoLocalStackContainer.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.integration.test.container; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +/** Tests the LocalStack readiness check without Docker. */ +public class TestGravitinoLocalStackContainer { + + @Test + void testS3MustBeAvailableOrRunning() { + Assertions.assertTrue( + GravitinoLocalStackContainer.reportsS3Ready( + "{\"services\": {\"iam\": \"available\", \"s3\": \"available\"}, \"edition\": \"community\"}")); + Assertions.assertTrue( + GravitinoLocalStackContainer.reportsS3Ready("{\"services\":{\"s3\":\"running\"}}")); + Assertions.assertFalse( + GravitinoLocalStackContainer.reportsS3Ready("{\"services\":{\"s3\":\"starting\"}}")); + Assertions.assertFalse( + GravitinoLocalStackContainer.reportsS3Ready("{\"services\":{\"sqs\":\"running\"}}")); + } +} diff --git a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/util/KerberosHdfs.java b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/util/KerberosHdfs.java new file mode 100644 index 00000000000..61b553bb264 --- /dev/null +++ b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/util/KerberosHdfs.java @@ -0,0 +1,192 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.integration.test.util; + +import com.google.common.base.Preconditions; +import java.io.IOException; +import java.io.UncheckedIOException; +import java.net.URI; +import java.security.PrivilegedExceptionAction; +import org.apache.gravitino.integration.test.container.HiveContainer; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.permission.FsPermission; +import org.apache.hadoop.security.UserGroupInformation; + +/** + * Changes HDFS paths on the Kerberos Hive container from the test JVM, in place of {@code docker + * exec ... hadoop fs}. + * + *

Each call logs in as {@code hive/}, the principal the container's own {@code hadoop fs} + * ran as. The image maps every two-part principal to the HDFS superuser, so calls can change any + * path. The keytab is the container's {@code /etc/admin.keytab}, which holds that principal. The + * login returns its own {@link UserGroupInformation} and the file system is not cached, so the + * login user of a Gravitino server in the same JVM is left alone. {@code java.security.krb5.conf} + * must already point at the container's KDC. + */ +public final class KerberosHdfs { + private static final String REALM = "HADOOPKRB"; + + private final URI defaultFs; + private final String principal; + private final String nameNodePrincipal; + private final String keytabPath; + + /** + * Creates a client for the NameNode at {@code defaultFs}. + * + * @param defaultFs the NameNode address, for example {@code hdfs://172.18.0.2:9000} + * @param hostName the container's host name, which the service principals carry + * @param keytabPath a local copy of the container's {@code /etc/admin.keytab} + */ + public KerberosHdfs(String defaultFs, String hostName, String keytabPath) { + Preconditions.checkArgument( + defaultFs != null && defaultFs.startsWith("hdfs://"), "defaultFs must be an hdfs:// URI"); + Preconditions.checkArgument( + hostName != null && !hostName.isBlank(), "hostName must not be blank"); + Preconditions.checkArgument( + keytabPath != null && !keytabPath.isBlank(), "keytabPath must not be blank"); + this.defaultFs = URI.create(defaultFs); + this.principal = String.format("hive/%s@%s", hostName, REALM); + this.nameNodePrincipal = String.format("hdfs/%s@%s", hostName, REALM); + this.keytabPath = keytabPath; + } + + /** + * Creates a client for the given Kerberos Hive container. + * + * @param kerberosHive the running Kerberos Hive container + * @param keytabPath a local copy of the container's {@code /etc/admin.keytab} + * @return the client + */ + public static KerberosHdfs of(HiveContainer kerberosHive, String keytabPath) { + return new KerberosHdfs( + String.format( + "hdfs://%s:%d", + kerberosHive.getContainerIpAddress(), HiveContainer.HDFS_DEFAULTFS_PORT), + kerberosHive.getHostName(), + keytabPath); + } + + /** + * Creates a directory and any missing parents, like {@code hadoop fs -mkdir -p}. + * + * @param path the directory + */ + public void mkdirs(String path) { + run(fs -> fs.mkdirs(new Path(path))); + } + + /** + * Sets the permission, like {@code hadoop fs -chmod [-R]}. + * + * @param path the file or directory + * @param octalMode the mode in octal, for example {@code 777} + * @param recursive whether to also change everything below {@code path} + */ + public void setPermission(String path, String octalMode, boolean recursive) { + FsPermission permission = new FsPermission(Short.parseShort(octalMode, 8)); + run( + fs -> { + setPermission(fs, new Path(path), permission, recursive); + return null; + }); + } + + /** + * Sets the owner and keeps the group, like {@code hadoop fs -chown [-R] }. + * + * @param path the file or directory + * @param owner the new owner + * @param recursive whether to also change everything below {@code path} + */ + public void setOwner(String path, String owner, boolean recursive) { + run( + fs -> { + setOwner(fs, new Path(path), owner, recursive); + return null; + }); + } + + /** + * Deletes a path, like {@code hadoop fs -rm [-r]} without the trash. + * + * @param path the file or directory + * @param recursive whether a non-empty directory may be deleted + */ + public void delete(String path, boolean recursive) { + run(fs -> fs.delete(new Path(path), recursive)); + } + + // HDFS has no recursive chmod or chown; the shell walks the tree the same way. + static void setPermission(FileSystem fs, Path path, FsPermission permission, boolean recursive) + throws IOException { + fs.setPermission(path, permission); + if (recursive && fs.getFileStatus(path).isDirectory()) { + for (FileStatus child : fs.listStatus(path)) { + setPermission(fs, child.getPath(), permission, true); + } + } + } + + static void setOwner(FileSystem fs, Path path, String owner, boolean recursive) + throws IOException { + fs.setOwner(path, owner, null); + if (recursive && fs.getFileStatus(path).isDirectory()) { + for (FileStatus child : fs.listStatus(path)) { + setOwner(fs, child.getPath(), owner, true); + } + } + } + + private interface FileSystemAction { + T apply(FileSystem fs) throws IOException; + } + + private T run(FileSystemAction action) { + Configuration conf = new Configuration(); + conf.set("fs.defaultFS", defaultFs.toString()); + conf.set("hadoop.security.authentication", "kerberos"); + conf.set("dfs.namenode.kerberos.principal", nameNodePrincipal); + conf.setBoolean("fs.hdfs.impl.disable.cache", true); + try { + // A keytab login is only possible while UGI has security on. The Gravitino server in the + // same JVM turns it on for its own Kerberos catalogs, so only switch it on when it is off. + if (!UserGroupInformation.isSecurityEnabled()) { + UserGroupInformation.setConfiguration(conf); + } + UserGroupInformation ugi = + UserGroupInformation.loginUserFromKeytabAndReturnUGI(principal, keytabPath); + return ugi.doAs( + (PrivilegedExceptionAction) + () -> { + try (FileSystem fs = FileSystem.get(defaultFs, conf)) { + return action.apply(fs); + } + }); + } catch (IOException e) { + throw new UncheckedIOException("HDFS call on " + defaultFs + " failed", e); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new IllegalStateException("Interrupted during an HDFS call on " + defaultFs, e); + } + } +} diff --git a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/util/TestKerberosHdfs.java b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/util/TestKerberosHdfs.java new file mode 100644 index 00000000000..0baa289cf59 --- /dev/null +++ b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/util/TestKerberosHdfs.java @@ -0,0 +1,110 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.integration.test.util; + +import java.io.IOException; +import java.net.URI; +import java.nio.file.Files; +import java.util.ArrayList; +import java.util.List; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.RawLocalFileSystem; +import org.apache.hadoop.fs.permission.FsPermission; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +public class TestKerberosHdfs { + + @TempDir java.nio.file.Path tempDir; + + private RecordingFileSystem fs; + private Path root; + + @BeforeEach + void setUp() throws IOException { + fs = new RecordingFileSystem(); + fs.initialize(URI.create("file:///"), new Configuration()); + root = new Path(tempDir.toUri()); + Files.createDirectories(tempDir.resolve("a/b")); + Files.createFile(tempDir.resolve("a/b/file")); + Files.createFile(tempDir.resolve("top")); + } + + @Test + void testRejectsBadArguments() { + Assertions.assertThrows( + IllegalArgumentException.class, () -> new KerberosHdfs("file:///", "host", "/k")); + Assertions.assertThrows( + IllegalArgumentException.class, () -> new KerberosHdfs("hdfs://h:9000", " ", "/k")); + Assertions.assertThrows( + IllegalArgumentException.class, () -> new KerberosHdfs("hdfs://h:9000", "host", null)); + Assertions.assertDoesNotThrow(() -> new KerberosHdfs("hdfs://h:9000", "host", "/k")); + } + + @Test + void testRecursivePermissionReachesEveryPath() throws IOException { + FsPermission mode = new FsPermission(Short.parseShort("750", 8)); + + KerberosHdfs.setPermission(fs, root, mode, true); + + for (String path : new String[] {"", "a", "a/b", "a/b/file", "top"}) { + Path p = path.isEmpty() ? root : new Path(root, path); + Assertions.assertEquals(mode, fs.getFileStatus(p).getPermission(), path); + } + } + + @Test + void testNonRecursivePermissionChangesOnlyThePath() throws IOException { + FsPermission before = fs.getFileStatus(new Path(root, "a")).getPermission(); + FsPermission mode = new FsPermission(Short.parseShort("700", 8)); + + KerberosHdfs.setPermission(fs, root, mode, false); + + Assertions.assertEquals(mode, fs.getFileStatus(root).getPermission()); + Assertions.assertEquals(before, fs.getFileStatus(new Path(root, "a")).getPermission()); + } + + @Test + void testRecursiveOwnerKeepsGroupAndReachesEveryPath() throws IOException { + KerberosHdfs.setOwner(fs, root, "cli", true); + + Assertions.assertEquals(5, fs.owners.size()); + Assertions.assertTrue(fs.owners.stream().allMatch(call -> call.endsWith("cli:null"))); + } + + @Test + void testNonRecursiveOwnerChangesOnlyThePath() throws IOException { + KerberosHdfs.setOwner(fs, root, "cli", false); + + Assertions.assertEquals(List.of(fs.makeQualified(root) + " cli:null"), fs.owners); + } + + // Changing owners locally needs root, so record the calls instead. + private static class RecordingFileSystem extends RawLocalFileSystem { + private final List owners = new ArrayList<>(); + + @Override + public void setOwner(Path p, String username, String groupname) { + owners.add(makeQualified(p) + " " + username + ":" + groupname); + } + } +} diff --git a/trino-connector/integration-test/build.gradle.kts b/trino-connector/integration-test/build.gradle.kts index 5694143a035..9afdb4aeaa0 100644 --- a/trino-connector/integration-test/build.gradle.kts +++ b/trino-connector/integration-test/build.gradle.kts @@ -81,6 +81,10 @@ dependencies { } testImplementation(libs.trino.jdbc) + // The LocalStack fixture creates buckets with the S3 SDK v2; the synchronous client needs no Netty. + testRuntimeOnly(libs.aws.s3) { + exclude("software.amazon.awssdk", "netty-nio-client") + } testRuntimeOnly(libs.junit.jupiter.engine) } diff --git a/trino-connector/integration-test/src/test/java/org/apache/gravitino/trino/connector/integration/test/TrinoConnectorIT.java b/trino-connector/integration-test/src/test/java/org/apache/gravitino/trino/connector/integration/test/TrinoConnectorIT.java index 7b04f21da62..442cc1b9572 100644 --- a/trino-connector/integration-test/src/test/java/org/apache/gravitino/trino/connector/integration/test/TrinoConnectorIT.java +++ b/trino-connector/integration-test/src/test/java/org/apache/gravitino/trino/connector/integration/test/TrinoConnectorIT.java @@ -23,6 +23,10 @@ import com.google.common.collect.ImmutableMap; import com.google.common.collect.Maps; import com.google.common.collect.Sets; +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.SQLException; +import java.sql.Statement; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; @@ -597,17 +601,7 @@ private Column[] createFullTypeColumns() { void testColumnTypeNotNullByTrino() { String catalogName = GravitinoITUtils.genRandomName("mysql_catalog").toLowerCase(); GravitinoMetalake createdMetalake = client.loadMetalake(metalakeName); - String[] command = { - "mysql", - "-h127.0.0.1", - "-uroot", - "-pds123", // username and password are referred from Hive dockerfile. - "-e", - "grant all privileges on *.* to root@'%' identified by 'ds123'" - }; - - // There exists a mysql instance in Hive the container. - containerSuite.getHiveContainer().executeInContainer(command); + grantRootAccessToHiveMysql(); String hiveHost = containerSuite.getHiveContainer().getContainerIpAddress(); createdMetalake.createCatalog( @@ -1007,17 +1001,7 @@ void testIcebergCatalogCreatedByGravitino() { String catalogName = GravitinoITUtils.genRandomName("iceberg_catalog").toLowerCase(); String schemaName = GravitinoITUtils.genRandomName("iceberg_catalog").toLowerCase(); GravitinoMetalake createdMetalake = client.loadMetalake(metalakeName); - String[] command = { - "mysql", - "-h127.0.0.1", - "-uroot", - "-pds123", // username and password are referred from Hive dockerfile - "-e", - "grant all privileges on *.* to root@'%' identified by 'ds123'" - }; - - // There exists a mysql instance in Hive the container. - containerSuite.getHiveContainer().executeInContainer(command); + grantRootAccessToHiveMysql(); String hiveHost = containerSuite.getHiveContainer().getContainerIpAddress(); createdMetalake.createCatalog( @@ -1090,17 +1074,7 @@ void testIcebergCatalogCreatedByGravitino() { void testMySQLCatalogCreatedByGravitino() { String catalogName = GravitinoITUtils.genRandomName("mysql_catalog").toLowerCase(); GravitinoMetalake createdMetalake = client.loadMetalake(metalakeName); - String[] command = { - "mysql", - "-h127.0.0.1", - "-uroot", - "-pds123", // username and password are referred from Hive dockerfile. - "-e", - "grant all privileges on *.* to root@'%' identified by 'ds123'" - }; - - // There exists a mysql instance in Hive the container. - containerSuite.getHiveContainer().executeInContainer(command); + grantRootAccessToHiveMysql(); String hiveHost = containerSuite.getHiveContainer().getContainerIpAddress(); createdMetalake.createCatalog( @@ -1133,17 +1107,7 @@ void testMySQLTableCreatedByGravitino() { String schemaName = GravitinoITUtils.genRandomName("mysql_schema").toLowerCase(); String tableName = GravitinoITUtils.genRandomName("mysql_table").toLowerCase(); GravitinoMetalake createdMetalake = client.loadMetalake(metalakeName); - String[] command = { - "mysql", - "-h127.0.0.1", - "-uroot", - "-pds123", // username and password are referred from Hive dockerfile. - "-e", - "grant all privileges on *.* to root@'%' identified by 'ds123'" - }; - - // There exists a mysql instance in Hive the container. - containerSuite.getHiveContainer().executeInContainer(command); + grantRootAccessToHiveMysql(); String hiveHost = containerSuite.getHiveContainer().getContainerIpAddress(); createdMetalake.createCatalog( @@ -1241,17 +1205,7 @@ void testMySQLTableCreatedByTrino() { String schemaName = GravitinoITUtils.genRandomName("mysql_schema").toLowerCase(); String tableName = GravitinoITUtils.genRandomName("mysql_table").toLowerCase(); GravitinoMetalake createdMetalake = client.loadMetalake(metalakeName); - String[] command = { - "mysql", - "-h127.0.0.1", - "-uroot", - "-pds123", // username and password are referred from Hive dockerfile. - "-e", - "grant all privileges on *.* to root@'%' identified by 'ds123'" - }; - - // There exists a mysql instance in Hive the container. - containerSuite.getHiveContainer().executeInContainer(command); + grantRootAccessToHiveMysql(); String hiveHost = containerSuite.getHiveContainer().getContainerIpAddress(); createdMetalake.createCatalog( @@ -1359,17 +1313,7 @@ void testMySQLTableCreatedByTrino() { void testDropCatalogAndCreateAgain() { String catalogName = GravitinoITUtils.genRandomName("mysql_catalog").toLowerCase(); GravitinoMetalake createdMetalake = client.loadMetalake(metalakeName); - String[] command = { - "mysql", - "-h127.0.0.1", - "-uroot", - "-pds123", // username and password are referred from Hive dockerfile. - "-e", - "grant all privileges on *.* to root@'%' identified by 'ds123'" - }; - - // There exists a mysql instance in Hive the container. - containerSuite.getHiveContainer().executeInContainer(command); + grantRootAccessToHiveMysql(); String hiveHost = containerSuite.getHiveContainer().getContainerIpAddress(); // Create the catalog and drop it for 3 times to test the create/drop catalog function work @@ -1428,4 +1372,19 @@ private static void createCatalog() { catalog = loadCatalog; } + + // Lets the tests' MySQL catalogs reach the MySQL inside the Hive container as root. Root may only + // log in from localhost there, so the grant runs as `iceberg`, which the image creates for any + // host with the grant option. + private static void grantRootAccessToHiveMysql() { + String url = + String.format( + "jdbc:mysql://%s:3306/", containerSuite.getHiveContainer().getContainerIpAddress()); + try (Connection connection = DriverManager.getConnection(url, "iceberg", "iceberg"); + Statement statement = connection.createStatement()) { + statement.execute("GRANT ALL PRIVILEGES ON *.* TO root@'%' IDENTIFIED BY 'ds123'"); + } catch (SQLException e) { + throw new IllegalStateException("Failed to grant root access to the Hive MySQL", e); + } + } } diff --git a/trino-connector/integration-test/src/test/java/org/apache/gravitino/trino/connector/integration/test/TrinoTlsOAuthCredentialVendingIT.java b/trino-connector/integration-test/src/test/java/org/apache/gravitino/trino/connector/integration/test/TrinoTlsOAuthCredentialVendingIT.java index 408d4715b24..40349d1b741 100644 --- a/trino-connector/integration-test/src/test/java/org/apache/gravitino/trino/connector/integration/test/TrinoTlsOAuthCredentialVendingIT.java +++ b/trino-connector/integration-test/src/test/java/org/apache/gravitino/trino/connector/integration/test/TrinoTlsOAuthCredentialVendingIT.java @@ -69,7 +69,6 @@ import org.junit.jupiter.api.condition.EnabledIfSystemProperty; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.testcontainers.containers.Container; /** * Verifies the complete secured Trino connector path: HTTPS coordinator, OAuth2-authenticated @@ -160,10 +159,8 @@ public void testTlsOAuthAndCredentialVending() { oauthServer.invalidRequests(), "The mock OAuth2 server rejected an unexpected or malformed token request"); - Container.ExecResult objects = - localStack.executeInContainer("awslocal", "s3", "ls", "s3://" + bucketName, "--recursive"); - assertEquals(0, objects.getExitCode(), objects.getStderr()); - assertFalse(objects.getStdout().isBlank(), "No Iceberg objects were written to S3"); + assertFalse( + localStack.listObjectKeys(bucketName).isEmpty(), "No Iceberg objects were written to S3"); trinoContainer.executeUpdateSQL("DROP TABLE " + table); trinoContainer.executeUpdateSQL("DROP SCHEMA " + catalogName + "." + schema); @@ -413,9 +410,7 @@ private void startTrino() { } private void createBucket() { - Container.ExecResult result = - localStack.executeInContainer("awslocal", "s3", "mb", "s3://" + bucketName); - assertEquals(0, result.getExitCode(), result.getStderr()); + localStack.createBucket(bucketName); } private void copyIcebergAwsBundle() { From eef23c0d6f8ad79d8e35ba6e9b94027e382cfa1e Mon Sep 17 00:00:00 2001 From: Nevin Zheng Date: Mon, 19 Oct 2026 12:47:04 -0700 Subject: [PATCH 09/14] [#14002] improvement(test): Add a JUnit fixture framework for test containers (part 1/6) ContainerSpec, ContainerExtension and @FixtureContainer: a test declares a container with a fixture annotation and receives it as a parameter or field. Fixtures for Kafka, LocalStack, RustFS, StarRocks, OceanBase and Doris, with their callers migrated. --- .../integration/test/CatalogOceanBaseIT.java | 10 +- .../oceanbase/operation/TestOceanBase.java | 9 +- .../integration/test/FilesetS3CatalogIT.java | 8 +- .../test/FilesetS3TokenConnectionIT.java | 6 +- .../integration/test/CatalogHiveS3IT.java | 7 +- .../integration/test/CatalogDoris2xIT.java | 9 +- .../integration/test/CatalogDoris3xIT.java | 11 +- .../integration/test/CatalogDoris4xIT.java | 11 +- .../integration/test/CatalogDorisIT.java | 12 +- .../catalog/doris/operation/TestDoris.java | 9 +- .../integration/test/CatalogStarRocksIT.java | 10 +- .../starrocks/operation/TestStarRocks.java | 9 +- .../integration/test/CatalogKafkaIT.java | 12 +- .../integration/test/CatalogPaimonS3AIT.java | 7 +- .../integration/test/CatalogPaimonS3IT.java | 7 +- .../test/authorization/OwnerIT.java | 11 +- .../authorization/TopicAuthorizationIT.java | 12 +- .../test/GravitinoVirtualFileSystemS3IT.java | 7 +- ...IcebergRESTRustFSTokenAuthorizationIT.java | 9 +- integration-test-common/build.gradle.kts | 9 + .../test/container/ContainerExtension.java | 159 ++++++++++ .../test/container/ContainerFixture.java | 59 ++++ .../test/container/ContainerSpec.java | 48 +++ .../test/container/DorisFixture.java | 47 +++ .../integration/test/container/DorisSpec.java | 44 +++ .../test/container/FixtureContainer.java | 44 +++ .../test/container/KafkaFixture.java | 47 +++ .../integration/test/container/KafkaSpec.java | 41 +++ .../test/container/LocalStackFixture.java | 40 +++ .../test/container/LocalStackSpec.java | 41 +++ .../test/container/OceanBaseFixture.java | 39 +++ .../test/container/OceanBaseSpec.java | 41 +++ .../test/container/RustFSFixture.java | 39 +++ .../test/container/RustFSSpec.java | 41 +++ .../test/container/StarRocksFixture.java | 39 +++ .../test/container/StarRocksSpec.java | 41 +++ .../container/TestContainerExtension.java | 285 ++++++++++++++++++ .../TrinoTlsOAuthCredentialVendingIT.java | 7 +- .../test/web/ui/CatalogsPageDorisTest.java | 16 +- .../test/web/ui/CatalogsPageKafkaTest.java | 12 +- .../test/web/ui/CatalogsPageDorisTest.java | 16 +- .../test/web/ui/CatalogsPageKafkaTest.java | 12 +- 42 files changed, 1225 insertions(+), 118 deletions(-) create mode 100644 integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/ContainerExtension.java create mode 100644 integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/ContainerFixture.java create mode 100644 integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/ContainerSpec.java create mode 100644 integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/DorisFixture.java create mode 100644 integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/DorisSpec.java create mode 100644 integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/FixtureContainer.java create mode 100644 integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/KafkaFixture.java create mode 100644 integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/KafkaSpec.java create mode 100644 integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/LocalStackFixture.java create mode 100644 integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/LocalStackSpec.java create mode 100644 integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/OceanBaseFixture.java create mode 100644 integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/OceanBaseSpec.java create mode 100644 integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/RustFSFixture.java create mode 100644 integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/RustFSSpec.java create mode 100644 integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/StarRocksFixture.java create mode 100644 integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/StarRocksSpec.java create mode 100644 integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/TestContainerExtension.java diff --git a/catalogs-contrib/catalog-jdbc-oceanbase/src/test/java/org/apache/gravitino/catalog/oceanbase/integration/test/CatalogOceanBaseIT.java b/catalogs-contrib/catalog-jdbc-oceanbase/src/test/java/org/apache/gravitino/catalog/oceanbase/integration/test/CatalogOceanBaseIT.java index f2e5cb46bd4..f08017d1326 100644 --- a/catalogs-contrib/catalog-jdbc-oceanbase/src/test/java/org/apache/gravitino/catalog/oceanbase/integration/test/CatalogOceanBaseIT.java +++ b/catalogs-contrib/catalog-jdbc-oceanbase/src/test/java/org/apache/gravitino/catalog/oceanbase/integration/test/CatalogOceanBaseIT.java @@ -49,8 +49,9 @@ import org.apache.gravitino.exceptions.NoSuchSchemaException; import org.apache.gravitino.exceptions.NotFoundException; import org.apache.gravitino.exceptions.SchemaAlreadyExistsException; -import org.apache.gravitino.integration.test.container.ContainerSuite; +import org.apache.gravitino.integration.test.container.FixtureContainer; import org.apache.gravitino.integration.test.container.OceanBaseContainer; +import org.apache.gravitino.integration.test.container.OceanBaseFixture; import org.apache.gravitino.integration.test.util.BaseIT; import org.apache.gravitino.integration.test.util.GravitinoITUtils; import org.apache.gravitino.integration.test.util.ITUtils; @@ -86,9 +87,9 @@ @Tag("gravitino-docker-test") @TestInstance(Lifecycle.PER_CLASS) +@OceanBaseFixture public class CatalogOceanBaseIT extends BaseIT { private static final Logger LOG = LoggerFactory.getLogger(CatalogOceanBaseIT.class); - private static final ContainerSuite containerSuite = ContainerSuite.getInstance(); private static final String provider = "jdbc-oceanbase"; private final String metalakeName = GravitinoITUtils.genRandomName("oceanbase_it_metalake"); @@ -112,7 +113,7 @@ public class CatalogOceanBaseIT extends BaseIT { private OceanBaseService oceanBaseService; - private OceanBaseContainer OCEANBASE_CONTAINER; + @FixtureContainer private OceanBaseContainer OCEANBASE_CONTAINER; private String TEST_DB_NAME; @@ -124,9 +125,6 @@ boolean SupportColumnDefaultValueExpression() { public void startup() throws IOException, SQLException { TEST_DB_NAME = "oceanbase_catalog_oceanbase_it"; - containerSuite.startOceanBaseContainer(); - OCEANBASE_CONTAINER = containerSuite.getOceanBaseContainer(); - oceanBaseService = new OceanBaseService(OCEANBASE_CONTAINER, TEST_DB_NAME); createMetalake(); createCatalog(); diff --git a/catalogs-contrib/catalog-jdbc-oceanbase/src/test/java/org/apache/gravitino/catalog/oceanbase/operation/TestOceanBase.java b/catalogs-contrib/catalog-jdbc-oceanbase/src/test/java/org/apache/gravitino/catalog/oceanbase/operation/TestOceanBase.java index 4cb41d9a8f9..1d855446289 100644 --- a/catalogs-contrib/catalog-jdbc-oceanbase/src/test/java/org/apache/gravitino/catalog/oceanbase/operation/TestOceanBase.java +++ b/catalogs-contrib/catalog-jdbc-oceanbase/src/test/java/org/apache/gravitino/catalog/oceanbase/operation/TestOceanBase.java @@ -27,18 +27,19 @@ import org.apache.gravitino.catalog.jdbc.utils.DataSourceUtils; import org.apache.gravitino.catalog.oceanbase.converter.OceanBaseColumnDefaultValueConverter; import org.apache.gravitino.catalog.oceanbase.converter.OceanBaseTypeConverter; -import org.apache.gravitino.integration.test.container.ContainerSuite; +import org.apache.gravitino.integration.test.container.FixtureContainer; import org.apache.gravitino.integration.test.container.OceanBaseContainer; +import org.apache.gravitino.integration.test.container.OceanBaseFixture; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; +@OceanBaseFixture public class TestOceanBase extends TestJdbc { - private static final ContainerSuite containerSuite = ContainerSuite.getInstance(); + @FixtureContainer private static OceanBaseContainer oceanBase; protected static final String DRIVER_CLASS_NAME = "com.mysql.jdbc.Driver"; @BeforeAll public static void startup() { - containerSuite.startOceanBaseContainer(); DATA_SOURCE = DataSourceUtils.createDataSource(getOceanBaseCatalogProperties()); @@ -66,7 +67,7 @@ public static void stop() { private static Map getOceanBaseCatalogProperties() { Map catalogProperties = Maps.newHashMap(); - OceanBaseContainer oceanBaseContainer = containerSuite.getOceanBaseContainer(); + OceanBaseContainer oceanBaseContainer = oceanBase; String jdbcUrl = oceanBaseContainer.getJdbcUrl(); diff --git a/catalogs/catalog-fileset/src/test/java/org/apache/gravitino/catalog/fileset/integration/test/FilesetS3CatalogIT.java b/catalogs/catalog-fileset/src/test/java/org/apache/gravitino/catalog/fileset/integration/test/FilesetS3CatalogIT.java index 674df6984a1..3082f614af1 100644 --- a/catalogs/catalog-fileset/src/test/java/org/apache/gravitino/catalog/fileset/integration/test/FilesetS3CatalogIT.java +++ b/catalogs/catalog-fileset/src/test/java/org/apache/gravitino/catalog/fileset/integration/test/FilesetS3CatalogIT.java @@ -31,7 +31,9 @@ import org.apache.gravitino.NameIdentifier; import org.apache.gravitino.Schema; import org.apache.gravitino.file.Fileset; +import org.apache.gravitino.integration.test.container.FixtureContainer; import org.apache.gravitino.integration.test.container.GravitinoLocalStackContainer; +import org.apache.gravitino.integration.test.container.LocalStackFixture; import org.apache.gravitino.integration.test.util.GravitinoITUtils; import org.apache.gravitino.storage.S3Properties; import org.apache.hadoop.conf.Configuration; @@ -46,6 +48,7 @@ import org.slf4j.LoggerFactory; @Tag("gravitino-docker-test") +@LocalStackFixture public class FilesetS3CatalogIT extends FilesetCatalogIT { private static final Logger LOG = LoggerFactory.getLogger(FilesetS3CatalogIT.class); private String bucketName = "s3-bucket-" + UUID.randomUUID().toString().replace("-", ""); @@ -53,7 +56,7 @@ public class FilesetS3CatalogIT extends FilesetCatalogIT { private String secretKey; private String s3Endpoint; - private GravitinoLocalStackContainer gravitinoLocalStackContainer; + @FixtureContainer private GravitinoLocalStackContainer gravitinoLocalStackContainer; @VisibleForTesting public void startIntegrationTest() throws Exception {} @@ -61,9 +64,6 @@ public void startIntegrationTest() throws Exception {} @Override protected void startNecessaryContainer() { - containerSuite.startLocalStackContainer(); - gravitinoLocalStackContainer = containerSuite.getLocalStackContainer(); - // The container is ready once its health check passes. LocalStack does not enforce IAM, so // the shared test keys work without creating a user. gravitinoLocalStackContainer.createBucket(bucketName); diff --git a/catalogs/catalog-fileset/src/test/java/org/apache/gravitino/catalog/fileset/integration/test/FilesetS3TokenConnectionIT.java b/catalogs/catalog-fileset/src/test/java/org/apache/gravitino/catalog/fileset/integration/test/FilesetS3TokenConnectionIT.java index 3e5b56f90f4..a0577336563 100644 --- a/catalogs/catalog-fileset/src/test/java/org/apache/gravitino/catalog/fileset/integration/test/FilesetS3TokenConnectionIT.java +++ b/catalogs/catalog-fileset/src/test/java/org/apache/gravitino/catalog/fileset/integration/test/FilesetS3TokenConnectionIT.java @@ -29,7 +29,9 @@ import org.apache.gravitino.client.GravitinoMetalake; import org.apache.gravitino.credential.CredentialConstants; import org.apache.gravitino.credential.S3TokenCredential; +import org.apache.gravitino.integration.test.container.FixtureContainer; import org.apache.gravitino.integration.test.container.RustFSContainer; +import org.apache.gravitino.integration.test.container.RustFSFixture; import org.apache.gravitino.integration.test.util.BaseIT; import org.apache.gravitino.integration.test.util.GravitinoITUtils; import org.apache.gravitino.storage.S3Properties; @@ -40,7 +42,9 @@ /** Tests an existing Fileset catalog connection with S3 temporary credentials against RustFS. */ @Tag("gravitino-docker-test") +@RustFSFixture public class FilesetS3TokenConnectionIT extends BaseIT { + @FixtureContainer private RustFSContainer rustFSContainer; private final String bucketName = "fileset-connection-" + UUID.randomUUID().toString().replace("-", ""); @@ -54,8 +58,6 @@ public class FilesetS3TokenConnectionIT extends BaseIT { @BeforeAll public void startIntegrationTest() throws Exception { copyBundleJarsToHadoop("aws-bundle"); - containerSuite.startRustFSContainer(); - RustFSContainer rustFSContainer = containerSuite.getRustFSContainer(); rustFSContainer.createBucket(bucketName); super.startIntegrationTest(); diff --git a/catalogs/catalog-hive/src/test/java/org/apache/gravitino/catalog/hive/integration/test/CatalogHiveS3IT.java b/catalogs/catalog-hive/src/test/java/org/apache/gravitino/catalog/hive/integration/test/CatalogHiveS3IT.java index 7165605ee8a..8364bf09630 100644 --- a/catalogs/catalog-hive/src/test/java/org/apache/gravitino/catalog/hive/integration/test/CatalogHiveS3IT.java +++ b/catalogs/catalog-hive/src/test/java/org/apache/gravitino/catalog/hive/integration/test/CatalogHiveS3IT.java @@ -23,18 +23,21 @@ import java.net.URI; import java.util.HashMap; import java.util.Map; +import org.apache.gravitino.integration.test.container.FixtureContainer; import org.apache.gravitino.integration.test.container.GravitinoLocalStackContainer; import org.apache.gravitino.integration.test.container.HiveContainer; +import org.apache.gravitino.integration.test.container.LocalStackFixture; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.spark.sql.SparkSession; import org.junit.jupiter.api.TestInstance; @TestInstance(TestInstance.Lifecycle.PER_CLASS) +@LocalStackFixture public class CatalogHiveS3IT extends CatalogHive3IT { private static final String S3_BUCKET_NAME = "my-test-bucket"; - private GravitinoLocalStackContainer gravitinoLocalStackContainer; + @FixtureContainer private GravitinoLocalStackContainer gravitinoLocalStackContainer; private static final String S3_ACCESS_KEY = "S3_ACCESS_KEY"; private static final String S3_SECRET_KEY = "S3_SECRET_KEY"; @@ -47,8 +50,6 @@ public class CatalogHiveS3IT extends CatalogHive3IT { @Override protected void startNecessaryContainer() { hmsCatalog = "hive"; - containerSuite.startLocalStackContainer(); - gravitinoLocalStackContainer = containerSuite.getLocalStackContainer(); // The container is ready once its health check passes. LocalStack does not enforce IAM, so // the shared test keys work without creating a user. diff --git a/catalogs/catalog-jdbc-doris/src/test/java/org/apache/gravitino/catalog/doris/integration/test/CatalogDoris2xIT.java b/catalogs/catalog-jdbc-doris/src/test/java/org/apache/gravitino/catalog/doris/integration/test/CatalogDoris2xIT.java index a008dd0dd31..fee7dd1c7ba 100644 --- a/catalogs/catalog-jdbc-doris/src/test/java/org/apache/gravitino/catalog/doris/integration/test/CatalogDoris2xIT.java +++ b/catalogs/catalog-jdbc-doris/src/test/java/org/apache/gravitino/catalog/doris/integration/test/CatalogDoris2xIT.java @@ -35,9 +35,10 @@ import org.apache.gravitino.StringIdentifier; import org.apache.gravitino.catalog.jdbc.config.JdbcConfig; import org.apache.gravitino.client.GravitinoMetalake; -import org.apache.gravitino.integration.test.container.ContainerSuite; import org.apache.gravitino.integration.test.container.DorisContainer; +import org.apache.gravitino.integration.test.container.DorisFixture; import org.apache.gravitino.integration.test.container.DorisImageName; +import org.apache.gravitino.integration.test.container.FixtureContainer; import org.apache.gravitino.integration.test.util.BaseIT; import org.apache.gravitino.integration.test.util.GravitinoITUtils; import org.apache.gravitino.rel.Column; @@ -58,11 +59,12 @@ /** Integration tests for Doris 2.1.0 with the Nereids planner enabled. */ @Tag("gravitino-docker-test") @Tag("doris-multi-version") +@DorisFixture(version = DorisImageName.VERSION_2_1) public class CatalogDoris2xIT extends BaseIT { + @FixtureContainer private DorisContainer doris; private static final String PROVIDER = "jdbc-doris"; private static final String DRIVER_CLASS_NAME = "com.mysql.cj.jdbc.Driver"; - private static final ContainerSuite containerSuite = ContainerSuite.getInstance(); private final String metalakeName = GravitinoITUtils.genRandomName("doris2x_metalake"); private final String catalogName = GravitinoITUtils.genRandomName("doris2x_catalog"); @@ -74,7 +76,6 @@ public class CatalogDoris2xIT extends BaseIT { @BeforeAll public void startup() { - containerSuite.startDorisContainer(DorisImageName.VERSION_2_1); createMetalake(); createCatalog(); createSchema(); @@ -190,7 +191,7 @@ private void createMetalake() { } private void createCatalog() { - DorisContainer dorisContainer = containerSuite.getDorisContainer(DorisImageName.VERSION_2_1); + DorisContainer dorisContainer = doris; jdbcUrl = String.format( "jdbc:mysql://%s:%d/", diff --git a/catalogs/catalog-jdbc-doris/src/test/java/org/apache/gravitino/catalog/doris/integration/test/CatalogDoris3xIT.java b/catalogs/catalog-jdbc-doris/src/test/java/org/apache/gravitino/catalog/doris/integration/test/CatalogDoris3xIT.java index f8092f53e36..7d9b18ac5a7 100644 --- a/catalogs/catalog-jdbc-doris/src/test/java/org/apache/gravitino/catalog/doris/integration/test/CatalogDoris3xIT.java +++ b/catalogs/catalog-jdbc-doris/src/test/java/org/apache/gravitino/catalog/doris/integration/test/CatalogDoris3xIT.java @@ -49,9 +49,10 @@ import org.apache.gravitino.catalog.jdbc.config.JdbcConfig; import org.apache.gravitino.client.GravitinoMetalake; import org.apache.gravitino.exceptions.NoSuchTableException; -import org.apache.gravitino.integration.test.container.ContainerSuite; import org.apache.gravitino.integration.test.container.DorisContainer; +import org.apache.gravitino.integration.test.container.DorisFixture; import org.apache.gravitino.integration.test.container.DorisImageName; +import org.apache.gravitino.integration.test.container.FixtureContainer; import org.apache.gravitino.integration.test.util.BaseIT; import org.apache.gravitino.integration.test.util.GravitinoITUtils; import org.apache.gravitino.rel.Column; @@ -88,13 +89,14 @@ */ @Tag("gravitino-docker-test") @Tag("doris-multi-version") +@DorisFixture(version = DorisImageName.VERSION_3_0) public class CatalogDoris3xIT extends BaseIT { + @FixtureContainer private DorisContainer doris; private static final String PROVIDER = "jdbc-doris"; private static final String DRIVER_CLASS_NAME = "com.mysql.cj.jdbc.Driver"; private static final long MAX_WAIT_IN_SECONDS = 30; private static final long WAIT_INTERVAL_IN_SECONDS = 1; - private static final ContainerSuite containerSuite = ContainerSuite.getInstance(); private String metalakeName = GravitinoITUtils.genRandomName("doris3x_metalake"); private String catalogName = GravitinoITUtils.genRandomName("doris3x_catalog"); @@ -109,7 +111,6 @@ public class CatalogDoris3xIT extends BaseIT { @BeforeAll public void startup() throws IOException { - containerSuite.startDorisContainer(DorisImageName.VERSION_3_0); createMetalake(); createCatalog(); createSchema(); @@ -178,7 +179,7 @@ private void createMetalake() { } private void createCatalog() { - DorisContainer dorisContainer = containerSuite.getDorisContainer(DorisImageName.VERSION_3_0); + DorisContainer dorisContainer = doris; jdbcUrl = String.format( "jdbc:mysql://%s:%d/", @@ -1095,7 +1096,7 @@ void testNativeNgramBfIndexFailsClosed() throws Exception { null, null); - DorisContainer dorisContainer = containerSuite.getDorisContainer(DorisImageName.VERSION_3_0); + DorisContainer dorisContainer = doris; String jdbcUrl = String.format( "jdbc:mysql://%s:%d/%s", diff --git a/catalogs/catalog-jdbc-doris/src/test/java/org/apache/gravitino/catalog/doris/integration/test/CatalogDoris4xIT.java b/catalogs/catalog-jdbc-doris/src/test/java/org/apache/gravitino/catalog/doris/integration/test/CatalogDoris4xIT.java index 6cc2445827b..663fba576e7 100644 --- a/catalogs/catalog-jdbc-doris/src/test/java/org/apache/gravitino/catalog/doris/integration/test/CatalogDoris4xIT.java +++ b/catalogs/catalog-jdbc-doris/src/test/java/org/apache/gravitino/catalog/doris/integration/test/CatalogDoris4xIT.java @@ -49,9 +49,10 @@ import org.apache.gravitino.catalog.jdbc.config.JdbcConfig; import org.apache.gravitino.client.GravitinoMetalake; import org.apache.gravitino.exceptions.NoSuchTableException; -import org.apache.gravitino.integration.test.container.ContainerSuite; import org.apache.gravitino.integration.test.container.DorisContainer; +import org.apache.gravitino.integration.test.container.DorisFixture; import org.apache.gravitino.integration.test.container.DorisImageName; +import org.apache.gravitino.integration.test.container.FixtureContainer; import org.apache.gravitino.integration.test.util.BaseIT; import org.apache.gravitino.integration.test.util.GravitinoITUtils; import org.apache.gravitino.rel.Column; @@ -89,13 +90,14 @@ */ @Tag("gravitino-docker-test") @Tag("doris-multi-version") +@DorisFixture(version = DorisImageName.VERSION_4_0) public class CatalogDoris4xIT extends BaseIT { + @FixtureContainer private DorisContainer doris; private static final String PROVIDER = "jdbc-doris"; private static final String DRIVER_CLASS_NAME = "com.mysql.cj.jdbc.Driver"; private static final long MAX_WAIT_IN_SECONDS = 30; private static final long WAIT_INTERVAL_IN_SECONDS = 1; - private static final ContainerSuite containerSuite = ContainerSuite.getInstance(); private String metalakeName = GravitinoITUtils.genRandomName("doris4x_metalake"); private String catalogName = GravitinoITUtils.genRandomName("doris4x_catalog"); @@ -110,7 +112,6 @@ public class CatalogDoris4xIT extends BaseIT { @BeforeAll public void startup() throws IOException { - containerSuite.startDorisContainer(DorisImageName.VERSION_4_0); createMetalake(); createCatalog(); createSchema(); @@ -179,7 +180,7 @@ private void createMetalake() { } private void createCatalog() { - DorisContainer dorisContainer = containerSuite.getDorisContainer(DorisImageName.VERSION_4_0); + DorisContainer dorisContainer = doris; jdbcUrl = String.format( "jdbc:mysql://%s:%d/", @@ -1003,7 +1004,7 @@ void testNativeNgramBfIndexFailsClosed() throws Exception { null, null); - DorisContainer dorisContainer = containerSuite.getDorisContainer(DorisImageName.VERSION_4_0); + DorisContainer dorisContainer = doris; String jdbcUrl = String.format( "jdbc:mysql://%s:%d/%s", diff --git a/catalogs/catalog-jdbc-doris/src/test/java/org/apache/gravitino/catalog/doris/integration/test/CatalogDorisIT.java b/catalogs/catalog-jdbc-doris/src/test/java/org/apache/gravitino/catalog/doris/integration/test/CatalogDorisIT.java index 7dd97e9f5b8..126b6a3202c 100644 --- a/catalogs/catalog-jdbc-doris/src/test/java/org/apache/gravitino/catalog/doris/integration/test/CatalogDorisIT.java +++ b/catalogs/catalog-jdbc-doris/src/test/java/org/apache/gravitino/catalog/doris/integration/test/CatalogDorisIT.java @@ -61,9 +61,9 @@ import org.apache.gravitino.exceptions.NoSuchPartitionException; import org.apache.gravitino.exceptions.NoSuchSchemaException; import org.apache.gravitino.exceptions.SchemaAlreadyExistsException; -import org.apache.gravitino.integration.test.container.ContainerSuite; import org.apache.gravitino.integration.test.container.DorisContainer; -import org.apache.gravitino.integration.test.container.DorisImageName; +import org.apache.gravitino.integration.test.container.DorisFixture; +import org.apache.gravitino.integration.test.container.FixtureContainer; import org.apache.gravitino.integration.test.util.BaseIT; import org.apache.gravitino.integration.test.util.GravitinoITUtils; import org.apache.gravitino.integration.test.util.ITUtils; @@ -100,7 +100,9 @@ @Tag("gravitino-docker-test") @Tag("doris-multi-version") +@DorisFixture public class CatalogDorisIT extends BaseIT { + @FixtureContainer private DorisContainer doris; private static final String provider = "jdbc-doris"; @@ -126,17 +128,13 @@ public class CatalogDorisIT extends BaseIT { private static final long MAX_WAIT_IN_SECONDS = 30; private static final long WAIT_INTERVAL_IN_SECONDS = 1; - - private static final ContainerSuite containerSuite = ContainerSuite.getInstance(); private GravitinoMetalake metalake; private String jdbcUrl; protected Catalog catalog; - protected DorisImageName dorisImageName = DorisImageName.VERSION_1_2; @BeforeAll public void startup() throws IOException { - containerSuite.startDorisContainer(dorisImageName); createMetalake(); createCatalog(); @@ -174,7 +172,7 @@ private void createMetalake() { private void createCatalog() { Map catalogProperties = Maps.newHashMap(); - DorisContainer dorisContainer = containerSuite.getDorisContainer(); + DorisContainer dorisContainer = doris; jdbcUrl = String.format( diff --git a/catalogs/catalog-jdbc-doris/src/test/java/org/apache/gravitino/catalog/doris/operation/TestDoris.java b/catalogs/catalog-jdbc-doris/src/test/java/org/apache/gravitino/catalog/doris/operation/TestDoris.java index b1a1c8095e3..ae1a417e6b5 100644 --- a/catalogs/catalog-jdbc-doris/src/test/java/org/apache/gravitino/catalog/doris/operation/TestDoris.java +++ b/catalogs/catalog-jdbc-doris/src/test/java/org/apache/gravitino/catalog/doris/operation/TestDoris.java @@ -27,18 +27,19 @@ import org.apache.gravitino.catalog.jdbc.TestJdbc; import org.apache.gravitino.catalog.jdbc.config.JdbcConfig; import org.apache.gravitino.catalog.jdbc.utils.DataSourceUtils; -import org.apache.gravitino.integration.test.container.ContainerSuite; import org.apache.gravitino.integration.test.container.DorisContainer; +import org.apache.gravitino.integration.test.container.DorisFixture; +import org.apache.gravitino.integration.test.container.FixtureContainer; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; +@DorisFixture public class TestDoris extends TestJdbc { - private static final ContainerSuite containerSuite = ContainerSuite.getInstance(); + @FixtureContainer private static DorisContainer doris; protected static final String DRIVER_CLASS_NAME = "com.mysql.jdbc.Driver"; @BeforeAll public static void startup() { - containerSuite.startDorisContainer(); DATA_SOURCE = DataSourceUtils.createDataSource(getDorisCatalogProperties()); @@ -66,7 +67,7 @@ public static void stop() { private static Map getDorisCatalogProperties() { Map catalogProperties = Maps.newHashMap(); - DorisContainer dorisContainer = containerSuite.getDorisContainer(); + DorisContainer dorisContainer = doris; String jdbcUrl = String.format( diff --git a/catalogs/catalog-jdbc-starrocks/src/test/java/org/apache/gravitino/catalog/starrocks/integration/test/CatalogStarRocksIT.java b/catalogs/catalog-jdbc-starrocks/src/test/java/org/apache/gravitino/catalog/starrocks/integration/test/CatalogStarRocksIT.java index 81c0c8dffbd..13402b0b7de 100644 --- a/catalogs/catalog-jdbc-starrocks/src/test/java/org/apache/gravitino/catalog/starrocks/integration/test/CatalogStarRocksIT.java +++ b/catalogs/catalog-jdbc-starrocks/src/test/java/org/apache/gravitino/catalog/starrocks/integration/test/CatalogStarRocksIT.java @@ -50,8 +50,9 @@ import org.apache.gravitino.exceptions.NoSuchPartitionException; import org.apache.gravitino.exceptions.NoSuchSchemaException; import org.apache.gravitino.exceptions.SchemaAlreadyExistsException; -import org.apache.gravitino.integration.test.container.ContainerSuite; +import org.apache.gravitino.integration.test.container.FixtureContainer; import org.apache.gravitino.integration.test.container.StarRocksContainer; +import org.apache.gravitino.integration.test.container.StarRocksFixture; import org.apache.gravitino.integration.test.util.BaseIT; import org.apache.gravitino.integration.test.util.GravitinoITUtils; import org.apache.gravitino.integration.test.util.ITUtils; @@ -85,7 +86,9 @@ import org.junit.jupiter.api.Test; @Tag("gravitino-docker-test") +@StarRocksFixture public class CatalogStarRocksIT extends BaseIT { + @FixtureContainer private StarRocksContainer starRocks; private static final String provider = "jdbc-starrocks"; @@ -111,8 +114,6 @@ public class CatalogStarRocksIT extends BaseIT { private static final long MAX_WAIT_IN_SECONDS = 30; private static final long WAIT_INTERVAL_IN_SECONDS = 1; - - private static final ContainerSuite containerSuite = ContainerSuite.getInstance(); private GravitinoMetalake metalake; private String jdbcUrl; @@ -120,7 +121,6 @@ public class CatalogStarRocksIT extends BaseIT { @BeforeAll public void startup() throws IOException { - containerSuite.startStarRocksContainer(); createMetalake(); createCatalog(); @@ -158,7 +158,7 @@ private void createMetalake() { private void createCatalog() { Map catalogProperties = Maps.newHashMap(); - StarRocksContainer starRocksContainer = containerSuite.getStarRocksContainer(); + StarRocksContainer starRocksContainer = starRocks; jdbcUrl = String.format( diff --git a/catalogs/catalog-jdbc-starrocks/src/test/java/org/apache/gravitino/catalog/starrocks/operation/TestStarRocks.java b/catalogs/catalog-jdbc-starrocks/src/test/java/org/apache/gravitino/catalog/starrocks/operation/TestStarRocks.java index 7f5abf97bcc..aea6d567615 100644 --- a/catalogs/catalog-jdbc-starrocks/src/test/java/org/apache/gravitino/catalog/starrocks/operation/TestStarRocks.java +++ b/catalogs/catalog-jdbc-starrocks/src/test/java/org/apache/gravitino/catalog/starrocks/operation/TestStarRocks.java @@ -29,18 +29,19 @@ import org.apache.gravitino.catalog.starrocks.converter.StarRocksTypeConverter; import org.apache.gravitino.catalog.starrocks.operations.StarRocksDatabaseOperations; import org.apache.gravitino.catalog.starrocks.operations.StarRocksTableOperations; -import org.apache.gravitino.integration.test.container.ContainerSuite; +import org.apache.gravitino.integration.test.container.FixtureContainer; import org.apache.gravitino.integration.test.container.StarRocksContainer; +import org.apache.gravitino.integration.test.container.StarRocksFixture; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; +@StarRocksFixture public class TestStarRocks extends TestJdbc { - private static final ContainerSuite containerSuite = ContainerSuite.getInstance(); + @FixtureContainer private static StarRocksContainer starRocks; protected static final String DRIVER_CLASS_NAME = "com.mysql.jdbc.Driver"; @BeforeAll public static void startup() { - containerSuite.startStarRocksContainer(); DATA_SOURCE = DataSourceUtils.createDataSource(getStarRocksCatalogProperties()); @@ -68,7 +69,7 @@ public static void stop() { private static Map getStarRocksCatalogProperties() { Map catalogProperties = Maps.newHashMap(); - StarRocksContainer starRocksContainer = containerSuite.getStarRocksContainer(); + StarRocksContainer starRocksContainer = starRocks; String jdbcUrl = String.format( diff --git a/catalogs/catalog-kafka/src/test/java/org/apache/gravitino/catalog/kafka/integration/test/CatalogKafkaIT.java b/catalogs/catalog-kafka/src/test/java/org/apache/gravitino/catalog/kafka/integration/test/CatalogKafkaIT.java index 12c19eab1d8..c83c849ad51 100644 --- a/catalogs/catalog-kafka/src/test/java/org/apache/gravitino/catalog/kafka/integration/test/CatalogKafkaIT.java +++ b/catalogs/catalog-kafka/src/test/java/org/apache/gravitino/catalog/kafka/integration/test/CatalogKafkaIT.java @@ -41,7 +41,8 @@ import org.apache.gravitino.SchemaChange; import org.apache.gravitino.client.GravitinoMetalake; import org.apache.gravitino.exceptions.NoSuchCatalogException; -import org.apache.gravitino.integration.test.container.ContainerSuite; +import org.apache.gravitino.integration.test.container.KafkaContainer; +import org.apache.gravitino.integration.test.container.KafkaFixture; import org.apache.gravitino.integration.test.util.BaseIT; import org.apache.gravitino.integration.test.util.GravitinoITUtils; import org.apache.gravitino.messaging.Topic; @@ -75,9 +76,9 @@ import org.slf4j.LoggerFactory; @Tag("gravitino-docker-test") +@KafkaFixture public class CatalogKafkaIT extends BaseIT { private static final Logger LOG = LoggerFactory.getLogger(CatalogKafkaIT.class); - private static final ContainerSuite CONTAINER_SUITE = ContainerSuite.getInstance(); private static final String METALAKE_NAME = GravitinoITUtils.genRandomName("catalogKafkaIT_metalake"); private static final String CATALOG_NAME = @@ -90,12 +91,9 @@ public class CatalogKafkaIT extends BaseIT { private static AdminClient adminClient; @BeforeAll - public void startUp() throws ExecutionException, InterruptedException { - CONTAINER_SUITE.startKafkaContainer(); + public void startUp(KafkaContainer kafka) throws ExecutionException, InterruptedException { kafkaBootstrapServers = - String.format( - "%s:%d", - CONTAINER_SUITE.getKafkaContainer().getContainerIpAddress(), DEFAULT_BROKER_PORT); + String.format("%s:%d", kafka.getContainerIpAddress(), DEFAULT_BROKER_PORT); adminClient = AdminClient.create(ImmutableMap.of(BOOTSTRAP_SERVERS, kafkaBootstrapServers)); // create topics for testing diff --git a/catalogs/catalog-lakehouse-paimon/src/test/java/org/apache/gravitino/catalog/lakehouse/paimon/integration/test/CatalogPaimonS3AIT.java b/catalogs/catalog-lakehouse-paimon/src/test/java/org/apache/gravitino/catalog/lakehouse/paimon/integration/test/CatalogPaimonS3AIT.java index 941d471a19a..cf59f581b2e 100644 --- a/catalogs/catalog-lakehouse-paimon/src/test/java/org/apache/gravitino/catalog/lakehouse/paimon/integration/test/CatalogPaimonS3AIT.java +++ b/catalogs/catalog-lakehouse-paimon/src/test/java/org/apache/gravitino/catalog/lakehouse/paimon/integration/test/CatalogPaimonS3AIT.java @@ -28,7 +28,9 @@ import org.apache.gravitino.Schema; import org.apache.gravitino.catalog.lakehouse.paimon.PaimonCatalogPropertiesMetadata; import org.apache.gravitino.client.GravitinoMetalake; +import org.apache.gravitino.integration.test.container.FixtureContainer; import org.apache.gravitino.integration.test.container.GravitinoLocalStackContainer; +import org.apache.gravitino.integration.test.container.LocalStackFixture; import org.apache.gravitino.integration.test.util.BaseIT; import org.apache.gravitino.integration.test.util.GravitinoITUtils; import org.junit.jupiter.api.AfterAll; @@ -39,6 +41,7 @@ /** Integration test for accessing a Paimon filesystem catalog through Hadoop S3A. */ @Tag("gravitino-docker-test") +@LocalStackFixture public class CatalogPaimonS3AIT extends BaseIT { private static final String PROVIDER = "lakehouse-paimon"; @@ -52,12 +55,10 @@ public class CatalogPaimonS3AIT extends BaseIT { private final String schemaName = GravitinoITUtils.genRandomName("paimon_s3a_schema"); private GravitinoMetalake metalake; - private GravitinoLocalStackContainer localStackContainer; + @FixtureContainer private GravitinoLocalStackContainer localStackContainer; @BeforeAll void setUp() { - containerSuite.startLocalStackContainer(); - localStackContainer = containerSuite.getLocalStackContainer(); // The container is ready once its health check passes. localStackContainer.createBucket(bucketName); diff --git a/catalogs/catalog-lakehouse-paimon/src/test/java/org/apache/gravitino/catalog/lakehouse/paimon/integration/test/CatalogPaimonS3IT.java b/catalogs/catalog-lakehouse-paimon/src/test/java/org/apache/gravitino/catalog/lakehouse/paimon/integration/test/CatalogPaimonS3IT.java index 012e1ea920e..904e4a08585 100644 --- a/catalogs/catalog-lakehouse-paimon/src/test/java/org/apache/gravitino/catalog/lakehouse/paimon/integration/test/CatalogPaimonS3IT.java +++ b/catalogs/catalog-lakehouse-paimon/src/test/java/org/apache/gravitino/catalog/lakehouse/paimon/integration/test/CatalogPaimonS3IT.java @@ -22,7 +22,9 @@ import com.google.common.collect.Maps; import java.util.Map; import org.apache.gravitino.catalog.lakehouse.paimon.PaimonCatalogPropertiesMetadata; +import org.apache.gravitino.integration.test.container.FixtureContainer; import org.apache.gravitino.integration.test.container.GravitinoLocalStackContainer; +import org.apache.gravitino.integration.test.container.LocalStackFixture; import org.apache.gravitino.integration.test.util.DownloaderUtils; import org.apache.gravitino.integration.test.util.ITUtils; import org.apache.gravitino.storage.S3Properties; @@ -30,10 +32,11 @@ import org.junit.jupiter.api.Tag; @Tag("gravitino-docker-test") +@LocalStackFixture public class CatalogPaimonS3IT extends CatalogPaimonBaseIT { private static final String S3_BUCKET_NAME = "my-test-bucket"; - private static GravitinoLocalStackContainer localStackContainer; + @FixtureContainer private static GravitinoLocalStackContainer localStackContainer; private String accessKey; private String secretKey; private String endpoint; @@ -82,8 +85,6 @@ private void downloadS3Dependency() { @Override protected void startNecessaryContainers() { - containerSuite.startLocalStackContainer(); - localStackContainer = containerSuite.getLocalStackContainer(); // The container is ready once its health check passes. localStackContainer.createBucket(S3_BUCKET_NAME); diff --git a/clients/client-java/src/test/java/org/apache/gravitino/client/integration/test/authorization/OwnerIT.java b/clients/client-java/src/test/java/org/apache/gravitino/client/integration/test/authorization/OwnerIT.java index 4427dcaf53f..4de1ebd3ae8 100644 --- a/clients/client-java/src/test/java/org/apache/gravitino/client/integration/test/authorization/OwnerIT.java +++ b/clients/client-java/src/test/java/org/apache/gravitino/client/integration/test/authorization/OwnerIT.java @@ -38,8 +38,10 @@ import org.apache.gravitino.exceptions.NotFoundException; import org.apache.gravitino.file.Fileset; import org.apache.gravitino.integration.test.container.ContainerSuite; +import org.apache.gravitino.integration.test.container.FixtureContainer; import org.apache.gravitino.integration.test.container.HiveContainer; import org.apache.gravitino.integration.test.container.KafkaContainer; +import org.apache.gravitino.integration.test.container.KafkaFixture; import org.apache.gravitino.integration.test.util.BaseIT; import org.apache.gravitino.rel.Column; import org.apache.gravitino.rel.types.Types; @@ -53,7 +55,9 @@ import org.slf4j.LoggerFactory; @Tag("gravitino-docker-test") +@KafkaFixture public class OwnerIT extends BaseIT { + @FixtureContainer private KafkaContainer kafka; private static final Logger LOG = LoggerFactory.getLogger(OwnerIT.class); @@ -78,13 +82,8 @@ public void startIntegrationTest() throws Exception { "thrift://%s:%d", containerSuite.getHiveContainer().getContainerIpAddress(), HiveContainer.HIVE_METASTORE_PORT); - - containerSuite.startKafkaContainer(); kafkaBootstrapServers = - String.format( - "%s:%d", - containerSuite.getKafkaContainer().getContainerIpAddress(), - KafkaContainer.DEFAULT_BROKER_PORT); + String.format("%s:%d", kafka.getContainerIpAddress(), KafkaContainer.DEFAULT_BROKER_PORT); } @AfterAll diff --git a/clients/client-java/src/test/java/org/apache/gravitino/client/integration/test/authorization/TopicAuthorizationIT.java b/clients/client-java/src/test/java/org/apache/gravitino/client/integration/test/authorization/TopicAuthorizationIT.java index 19ae04f75ae..f41d2bea11c 100644 --- a/clients/client-java/src/test/java/org/apache/gravitino/client/integration/test/authorization/TopicAuthorizationIT.java +++ b/clients/client-java/src/test/java/org/apache/gravitino/client/integration/test/authorization/TopicAuthorizationIT.java @@ -40,8 +40,9 @@ import org.apache.gravitino.authorization.SecurableObjects; import org.apache.gravitino.client.GravitinoMetalake; import org.apache.gravitino.exceptions.ForbiddenException; -import org.apache.gravitino.integration.test.container.ContainerSuite; +import org.apache.gravitino.integration.test.container.FixtureContainer; import org.apache.gravitino.integration.test.container.KafkaContainer; +import org.apache.gravitino.integration.test.container.KafkaFixture; import org.apache.gravitino.messaging.Topic; import org.apache.gravitino.messaging.TopicCatalog; import org.apache.gravitino.messaging.TopicChange; @@ -55,23 +56,20 @@ @Tag("gravitino-docker-test") @TestMethodOrder(MethodOrderer.OrderAnnotation.class) +@KafkaFixture public class TopicAuthorizationIT extends BaseRestApiAuthorizationIT { + @FixtureContainer private KafkaContainer kafka; private static final String CATALOG = "catalog"; private static final String SCHEMA = "default"; - private static final ContainerSuite containerSuite = ContainerSuite.getInstance(); private static String kafkaBootstrapServers; private static String role = "role"; @BeforeAll public void startIntegrationTest() throws Exception { - containerSuite.startKafkaContainer(); super.startIntegrationTest(); kafkaBootstrapServers = - String.format( - "%s:%d", - containerSuite.getKafkaContainer().getContainerIpAddress(), - KafkaContainer.DEFAULT_BROKER_PORT); + String.format("%s:%d", kafka.getContainerIpAddress(), KafkaContainer.DEFAULT_BROKER_PORT); Map properties = Maps.newHashMap(); properties.put("bootstrap.servers", kafkaBootstrapServers); client diff --git a/clients/filesystem-hadoop3/src/test/java/org/apache/gravitino/filesystem/hadoop/integration/test/GravitinoVirtualFileSystemS3IT.java b/clients/filesystem-hadoop3/src/test/java/org/apache/gravitino/filesystem/hadoop/integration/test/GravitinoVirtualFileSystemS3IT.java index 416924d625d..54e6e62d5d3 100644 --- a/clients/filesystem-hadoop3/src/test/java/org/apache/gravitino/filesystem/hadoop/integration/test/GravitinoVirtualFileSystemS3IT.java +++ b/clients/filesystem-hadoop3/src/test/java/org/apache/gravitino/filesystem/hadoop/integration/test/GravitinoVirtualFileSystemS3IT.java @@ -28,7 +28,9 @@ import java.util.UUID; import org.apache.gravitino.Catalog; import org.apache.gravitino.catalog.hadoop.fs.FileSystemUtils; +import org.apache.gravitino.integration.test.container.FixtureContainer; import org.apache.gravitino.integration.test.container.GravitinoLocalStackContainer; +import org.apache.gravitino.integration.test.container.LocalStackFixture; import org.apache.gravitino.integration.test.util.GravitinoITUtils; import org.apache.gravitino.s3.fs.S3FileSystemProvider; import org.apache.gravitino.storage.S3Properties; @@ -40,6 +42,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +@LocalStackFixture public class GravitinoVirtualFileSystemS3IT extends GravitinoVirtualFileSystemIT { private static final Logger LOG = LoggerFactory.getLogger(GravitinoVirtualFileSystemS3IT.class); @@ -48,7 +51,7 @@ public class GravitinoVirtualFileSystemS3IT extends GravitinoVirtualFileSystemIT private String secretKey; private String s3Endpoint; - private GravitinoLocalStackContainer gravitinoLocalStackContainer; + @FixtureContainer private GravitinoLocalStackContainer gravitinoLocalStackContainer; @BeforeAll public void startIntegrationTest() { @@ -56,8 +59,6 @@ public void startIntegrationTest() { } private void startS3Mocker() { - containerSuite.startLocalStackContainer(); - gravitinoLocalStackContainer = containerSuite.getLocalStackContainer(); // The container is ready once its health check passes. LocalStack does not enforce IAM, so // the shared test keys work without creating a user. diff --git a/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTRustFSTokenAuthorizationIT.java b/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTRustFSTokenAuthorizationIT.java index 3c58d5f8473..435dafca044 100644 --- a/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTRustFSTokenAuthorizationIT.java +++ b/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTRustFSTokenAuthorizationIT.java @@ -23,8 +23,9 @@ import org.apache.gravitino.catalog.lakehouse.iceberg.IcebergConstants; import org.apache.gravitino.credential.CredentialConstants; import org.apache.gravitino.credential.S3TokenCredential; -import org.apache.gravitino.integration.test.container.ContainerSuite; +import org.apache.gravitino.integration.test.container.FixtureContainer; import org.apache.gravitino.integration.test.container.RustFSContainer; +import org.apache.gravitino.integration.test.container.RustFSFixture; import org.apache.gravitino.integration.test.util.BaseIT; import org.apache.gravitino.integration.test.util.ITUtils; import org.apache.gravitino.storage.S3Properties; @@ -41,22 +42,20 @@ * {@code MODIFY_TABLE} is observable without a cloud account. */ @Tag("gravitino-docker-test") +@RustFSFixture public class IcebergRESTRustFSTokenAuthorizationIT extends IcebergRESTCloudTokenAuthorizationBaseIT { + @FixtureContainer private RustFSContainer rustFSContainer; private static final String BUCKET_NAME = "gravitino-rustfs-it"; private static final String BUNDLE_NAME = "iceberg-aws-bundle"; - private final ContainerSuite containerSuite = ContainerSuite.getInstance(); - private String s3Endpoint; private String warehouse; @BeforeAll public void startIntegrationTest() throws Exception { - containerSuite.startRustFSContainer(); - RustFSContainer rustFSContainer = containerSuite.getRustFSContainer(); rustFSContainer.createBucket(BUCKET_NAME); this.s3Endpoint = rustFSContainer.getS3Endpoint(); this.warehouse = String.format("s3://%s/test1", BUCKET_NAME); diff --git a/integration-test-common/build.gradle.kts b/integration-test-common/build.gradle.kts index 47ef9625f95..3bd7c418903 100644 --- a/integration-test-common/build.gradle.kts +++ b/integration-test-common/build.gradle.kts @@ -48,6 +48,8 @@ dependencies { testImplementation(libs.guava) testImplementation(libs.httpclient5) testImplementation(libs.junit.jupiter.api) + testImplementation(libs.junit.platform.commons) + testImplementation(libs.junit.platform.engine) testImplementation(libs.mockito.core) testImplementation(libs.testcontainers) testImplementation(libs.testcontainers.jdbc) @@ -68,12 +70,19 @@ dependencies { testImplementation(libs.hadoop3.client.runtime) testImplementation("org.junit.jupiter:junit-jupiter") + testImplementation("org.junit.platform:junit-platform-testkit") } tasks.test { useJUnitPlatform() } +// Nested fixture classes TestContainerExtension runs through EngineTestKit; Gradle's class scan +// would run them directly too. Keep in sync with TestContainerExtension.FIXTURE. +tasks.withType().configureEach { + useJUnitPlatform { excludeTags("gravitino-test-fixture") } +} + val testJar by tasks.registering(Jar::class) { archiveClassifier.set("tests") from(sourceSets["test"].output) diff --git a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/ContainerExtension.java b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/ContainerExtension.java new file mode 100644 index 00000000000..9f5582463b9 --- /dev/null +++ b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/ContainerExtension.java @@ -0,0 +1,159 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.integration.test.container; + +import java.lang.annotation.Annotation; +import java.lang.reflect.Constructor; +import java.lang.reflect.Field; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import java.util.function.Predicate; +import javax.annotation.Nullable; +import org.apache.gravitino.integration.test.util.CloseContainerExtension; +import org.junit.jupiter.api.extension.BeforeAllCallback; +import org.junit.jupiter.api.extension.ExtensionConfigurationException; +import org.junit.jupiter.api.extension.ExtensionContext; +import org.junit.jupiter.api.extension.ParameterContext; +import org.junit.jupiter.api.extension.ParameterResolutionException; +import org.junit.jupiter.api.extension.ParameterResolver; +import org.junit.jupiter.api.extension.TestInstancePostProcessor; +import org.junit.platform.commons.support.AnnotationSupport; +import org.junit.platform.commons.support.HierarchyTraversalMode; +import org.junit.platform.commons.support.ModifierSupport; + +/** + * Starts the containers a test class declares with fixture annotations, such as {@link + * KafkaFixture}, and passes them to test and lifecycle methods by parameter type and to fields + * marked {@link FixtureContainer}. + * + *

This is dependency injection built on JUnit's own extension points ({@link ParameterResolver} + * and {@link TestInstancePostProcessor}), so tests need no DI framework. + * + *

Each container starts once per run and is shared by every class that asks for the same spec. + * The containers close when the run ends, through {@link CloseContainerExtension}. + */ +public class ContainerExtension + implements BeforeAllCallback, ParameterResolver, TestInstancePostProcessor { + + private final CloseContainerExtension suiteCloser = new CloseContainerExtension(); + + @Override + public void beforeAll(ExtensionContext context) { + suiteCloser.beforeAll(context); + Class testClass = context.getRequiredTestClass(); + for (ContainerSpec spec : specs(testClass)) { + spec.getOrStart(ContainerSuite.getInstance()); + } + injectFields(testClass, null, ModifierSupport::isStatic); + } + + // Runs when the instance is created, before beforeAll(), so instance fields are filled before an + // overridden @BeforeAll method such as BaseIT.startIntegrationTest() runs. + @Override + public void postProcessTestInstance(Object testInstance, ExtensionContext context) { + injectFields(testInstance.getClass(), testInstance, ModifierSupport::isNotStatic); + } + + private static void injectFields( + Class testClass, @Nullable Object testInstance, Predicate which) { + for (Field field : + AnnotationSupport.findAnnotatedFields( + testClass, FixtureContainer.class, which, HierarchyTraversalMode.TOP_DOWN)) { + if (ModifierSupport.isFinal(field)) { + throw new ExtensionConfigurationException( + String.format( + "%s: @FixtureContainer field %s must not be final", + testClass.getSimpleName(), field.getName())); + } + try { + field.setAccessible(true); + field.set(testInstance, resolve(field.getType(), testClass, "field " + field.getName())); + } catch (IllegalAccessException e) { + throw new ExtensionConfigurationException("Cannot set field " + field.getName(), e); + } + } + } + + @Override + public boolean supportsParameter(ParameterContext parameter, ExtensionContext context) { + return !matching(parameter.getParameter().getType(), context.getRequiredTestClass()).isEmpty(); + } + + @Override + public Object resolveParameter(ParameterContext parameter, ExtensionContext context) { + return resolve( + parameter.getParameter().getType(), + context.getRequiredTestClass(), + "parameter " + parameter.getParameter().getName()); + } + + private static Object resolve(Class wanted, Class testClass, String target) { + List> matches = matching(wanted, testClass); + if (matches.size() != 1) { + throw new ParameterResolutionException( + String.format( + "%s: %s of type %s matches %d fixtures; declare exactly one fixture for it", + testClass.getSimpleName(), target, wanted.getSimpleName(), matches.size())); + } + return matches.get(0).getOrStart(ContainerSuite.getInstance()); + } + + private static List> matching(Class wanted, Class testClass) { + List> matches = new ArrayList<>(); + for (ContainerSpec spec : specs(testClass)) { + if (wanted.isAssignableFrom(spec.containerType())) { + matches.add(spec); + } + } + return matches; + } + + // Fixtures on the class and its superclasses, nearest class first, one per annotation type. + // A subclass that re-declares a fixture, for example with another version, replaces the + // superclass's; keeping both would start two containers and make injection by type ambiguous. + static List> specs(Class testClass) { + List> specs = new ArrayList<>(); + Set> seen = new HashSet<>(); + for (Class c = testClass; c != null && c != Object.class; c = c.getSuperclass()) { + for (Annotation annotation : c.getDeclaredAnnotations()) { + ContainerFixture fixture = + annotation.annotationType().getAnnotation(ContainerFixture.class); + if (fixture != null && seen.add(annotation.annotationType())) { + specs.add(specFor(fixture, annotation)); + } + } + } + return specs; + } + + @SuppressWarnings({"unchecked", "rawtypes"}) + private static ContainerSpec specFor(ContainerFixture fixture, Annotation annotation) { + try { + Constructor> constructor = + fixture.value().getDeclaredConstructor(); + constructor.setAccessible(true); + return ((ContainerFixture.SpecFactory) constructor.newInstance()).specFor(annotation); + } catch (ReflectiveOperationException e) { + throw new IllegalStateException( + "Cannot create the spec factory " + fixture.value().getName(), e); + } + } +} diff --git a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/ContainerFixture.java b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/ContainerFixture.java new file mode 100644 index 00000000000..8015d8fec94 --- /dev/null +++ b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/ContainerFixture.java @@ -0,0 +1,59 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.integration.test.container; + +import java.lang.annotation.Annotation; +import java.lang.annotation.Documented; +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +/** + * Marks an annotation as a container fixture, such as {@link KafkaFixture}, and names how to turn + * it into a {@link ContainerSpec}. {@link ContainerExtension} reads every fixture on a test class. + */ +@Documented +@Target(ElementType.ANNOTATION_TYPE) +@Retention(RetentionPolicy.RUNTIME) +public @interface ContainerFixture { + + /** + * Returns the class that builds the spec from the fixture annotation. + * + * @return the spec factory class; it needs a no-argument constructor + */ + Class> value(); + + /** + * Builds a {@link ContainerSpec} from a fixture annotation. + * + * @param the fixture annotation type + */ + interface SpecFactory { + + /** + * Returns the spec the fixture asks for. + * + * @param fixture the fixture annotation on the test class + * @return the spec + */ + ContainerSpec specFor(A fixture); + } +} diff --git a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/ContainerSpec.java b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/ContainerSpec.java new file mode 100644 index 00000000000..1a90757c5bd --- /dev/null +++ b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/ContainerSpec.java @@ -0,0 +1,48 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.integration.test.container; + +/** + * What a test asks for: one kind of container in one configuration. + * + *

Two equal specs mean the same container, so a spec is a record of everything that makes two + * requests different, such as a version. A service with nothing to choose has an empty spec. + * + * @param the container type a test receives + */ +public interface ContainerSpec { + + /** + * Returns the container type a test receives for this spec. + * + * @return the container type + */ + Class containerType(); + + /** + * Returns the running container for this spec, starting it on first use. + * + *

{@link ContainerSuite} keeps the container for the whole run, so tests that use fixtures and + * tests that call {@code ContainerSuite} directly share it. + * + * @param suite the shared container suite + * @return the running container + */ + C getOrStart(ContainerSuite suite); +} diff --git a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/DorisFixture.java b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/DorisFixture.java new file mode 100644 index 00000000000..e2cc1f8602a --- /dev/null +++ b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/DorisFixture.java @@ -0,0 +1,47 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.integration.test.container; + +import java.lang.annotation.Documented; +import java.lang.annotation.ElementType; +import java.lang.annotation.Inherited; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; +import org.junit.jupiter.api.extension.ExtendWith; + +/** + * Declares that a test class needs a Doris server. Test and lifecycle methods receive it as a + * {@link DorisContainer} parameter, and fields marked {@link FixtureContainer} receive it too. + */ +@Documented +@Inherited +@Target(ElementType.TYPE) +@Retention(RetentionPolicy.RUNTIME) +@ExtendWith(ContainerExtension.class) +@ContainerFixture(DorisSpec.Factory.class) +public @interface DorisFixture { + + /** + * Returns the Doris version to run. + * + * @return the Doris image; 1.2 by default + */ + DorisImageName version() default DorisImageName.VERSION_1_2; +} diff --git a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/DorisSpec.java b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/DorisSpec.java new file mode 100644 index 00000000000..b82c5c13cac --- /dev/null +++ b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/DorisSpec.java @@ -0,0 +1,44 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.integration.test.container; + +/** + * A Doris server of one version, given by {@code version}. Versions run side by side, each under + * its own host name. + */ +public record DorisSpec(DorisImageName version) implements ContainerSpec { + + @Override + public Class containerType() { + return DorisContainer.class; + } + + @Override + public DorisContainer getOrStart(ContainerSuite suite) { + suite.startDorisContainer(version); + return suite.getDorisContainer(version); + } + + static final class Factory implements ContainerFixture.SpecFactory { + @Override + public DorisSpec specFor(DorisFixture fixture) { + return new DorisSpec(fixture.version()); + } + } +} diff --git a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/FixtureContainer.java b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/FixtureContainer.java new file mode 100644 index 00000000000..b69561488f0 --- /dev/null +++ b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/FixtureContainer.java @@ -0,0 +1,44 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.integration.test.container; + +import java.lang.annotation.Documented; +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +/** + * Marks a field into which {@link ContainerExtension} injects a container from the test class's + * fixtures, such as {@link KafkaFixture}. This is dependency injection: the test declares what it + * needs, and the extension supplies it, by type, before {@code @BeforeAll} runs. + * + *

Use it where a method parameter cannot reach, such as an overridden {@code @BeforeAll} method. + * + * 

{@code
+ * @KafkaFixture
+ * class OwnerIT extends BaseIT {
+ *   @FixtureContainer private KafkaContainer kafka;
+ * }
+ * }
+ */ +@Documented +@Target(ElementType.FIELD) +@Retention(RetentionPolicy.RUNTIME) +public @interface FixtureContainer {} diff --git a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/KafkaFixture.java b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/KafkaFixture.java new file mode 100644 index 00000000000..d2352630133 --- /dev/null +++ b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/KafkaFixture.java @@ -0,0 +1,47 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.integration.test.container; + +import java.lang.annotation.Documented; +import java.lang.annotation.ElementType; +import java.lang.annotation.Inherited; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; +import org.junit.jupiter.api.extension.ExtendWith; + +/** + * Declares that a test class needs the Kafka broker. Test and lifecycle methods receive it as a + * {@link KafkaContainer} parameter. + * + * 
{@code
+ * @KafkaFixture
+ * class CatalogKafkaIT {
+ *   @BeforeAll
+ *   void startUp(KafkaContainer kafka) { ... }
+ * }
+ * }
+ */ +@Documented +@Inherited +@Target(ElementType.TYPE) +@Retention(RetentionPolicy.RUNTIME) +@ExtendWith(ContainerExtension.class) +@ContainerFixture(KafkaSpec.Factory.class) +public @interface KafkaFixture {} diff --git a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/KafkaSpec.java b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/KafkaSpec.java new file mode 100644 index 00000000000..7ce2f989f73 --- /dev/null +++ b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/KafkaSpec.java @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.integration.test.container; + +/** The Kafka broker. There is one configuration, so the spec has no fields. */ +public record KafkaSpec() implements ContainerSpec { + + @Override + public Class containerType() { + return KafkaContainer.class; + } + + @Override + public KafkaContainer getOrStart(ContainerSuite suite) { + suite.startKafkaContainer(); + return suite.getKafkaContainer(); + } + + static final class Factory implements ContainerFixture.SpecFactory { + @Override + public KafkaSpec specFor(KafkaFixture fixture) { + return new KafkaSpec(); + } + } +} diff --git a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/LocalStackFixture.java b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/LocalStackFixture.java new file mode 100644 index 00000000000..5301a060dfd --- /dev/null +++ b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/LocalStackFixture.java @@ -0,0 +1,40 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.integration.test.container; + +import java.lang.annotation.Documented; +import java.lang.annotation.ElementType; +import java.lang.annotation.Inherited; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; +import org.junit.jupiter.api.extension.ExtendWith; + +/** + * Declares that a test class needs the LocalStack S3 emulator. Test and lifecycle methods receive + * it as a {@link GravitinoLocalStackContainer} parameter, and fields marked {@link + * FixtureContainer} receive it too. + */ +@Documented +@Inherited +@Target(ElementType.TYPE) +@Retention(RetentionPolicy.RUNTIME) +@ExtendWith(ContainerExtension.class) +@ContainerFixture(LocalStackSpec.Factory.class) +public @interface LocalStackFixture {} diff --git a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/LocalStackSpec.java b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/LocalStackSpec.java new file mode 100644 index 00000000000..6a66d8dd059 --- /dev/null +++ b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/LocalStackSpec.java @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.integration.test.container; + +/** The LocalStack S3 emulator. There is one configuration, so the spec has no fields. */ +public record LocalStackSpec() implements ContainerSpec { + + @Override + public Class containerType() { + return GravitinoLocalStackContainer.class; + } + + @Override + public GravitinoLocalStackContainer getOrStart(ContainerSuite suite) { + suite.startLocalStackContainer(); + return suite.getLocalStackContainer(); + } + + static final class Factory implements ContainerFixture.SpecFactory { + @Override + public LocalStackSpec specFor(LocalStackFixture fixture) { + return new LocalStackSpec(); + } + } +} diff --git a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/OceanBaseFixture.java b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/OceanBaseFixture.java new file mode 100644 index 00000000000..04d917f7d61 --- /dev/null +++ b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/OceanBaseFixture.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.integration.test.container; + +import java.lang.annotation.Documented; +import java.lang.annotation.ElementType; +import java.lang.annotation.Inherited; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; +import org.junit.jupiter.api.extension.ExtendWith; + +/** + * Declares that a test class needs the OceanBase server. Test and lifecycle methods receive it as a + * {@link OceanBaseContainer} parameter, and fields marked {@link FixtureContainer} receive it too. + */ +@Documented +@Inherited +@Target(ElementType.TYPE) +@Retention(RetentionPolicy.RUNTIME) +@ExtendWith(ContainerExtension.class) +@ContainerFixture(OceanBaseSpec.Factory.class) +public @interface OceanBaseFixture {} diff --git a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/OceanBaseSpec.java b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/OceanBaseSpec.java new file mode 100644 index 00000000000..bb100b2e662 --- /dev/null +++ b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/OceanBaseSpec.java @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.integration.test.container; + +/** The OceanBase server. There is one configuration, so the spec has no fields. */ +public record OceanBaseSpec() implements ContainerSpec { + + @Override + public Class containerType() { + return OceanBaseContainer.class; + } + + @Override + public OceanBaseContainer getOrStart(ContainerSuite suite) { + suite.startOceanBaseContainer(); + return suite.getOceanBaseContainer(); + } + + static final class Factory implements ContainerFixture.SpecFactory { + @Override + public OceanBaseSpec specFor(OceanBaseFixture fixture) { + return new OceanBaseSpec(); + } + } +} diff --git a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/RustFSFixture.java b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/RustFSFixture.java new file mode 100644 index 00000000000..60a357e512b --- /dev/null +++ b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/RustFSFixture.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.integration.test.container; + +import java.lang.annotation.Documented; +import java.lang.annotation.ElementType; +import java.lang.annotation.Inherited; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; +import org.junit.jupiter.api.extension.ExtendWith; + +/** + * Declares that a test class needs the RustFS S3 server. Test and lifecycle methods receive it as a + * {@link RustFSContainer} parameter, and fields marked {@link FixtureContainer} receive it too. + */ +@Documented +@Inherited +@Target(ElementType.TYPE) +@Retention(RetentionPolicy.RUNTIME) +@ExtendWith(ContainerExtension.class) +@ContainerFixture(RustFSSpec.Factory.class) +public @interface RustFSFixture {} diff --git a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/RustFSSpec.java b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/RustFSSpec.java new file mode 100644 index 00000000000..b739bda6bbf --- /dev/null +++ b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/RustFSSpec.java @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.integration.test.container; + +/** The RustFS S3 server. There is one configuration, so the spec has no fields. */ +public record RustFSSpec() implements ContainerSpec { + + @Override + public Class containerType() { + return RustFSContainer.class; + } + + @Override + public RustFSContainer getOrStart(ContainerSuite suite) { + suite.startRustFSContainer(); + return suite.getRustFSContainer(); + } + + static final class Factory implements ContainerFixture.SpecFactory { + @Override + public RustFSSpec specFor(RustFSFixture fixture) { + return new RustFSSpec(); + } + } +} diff --git a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/StarRocksFixture.java b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/StarRocksFixture.java new file mode 100644 index 00000000000..e0b79ed9f84 --- /dev/null +++ b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/StarRocksFixture.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.integration.test.container; + +import java.lang.annotation.Documented; +import java.lang.annotation.ElementType; +import java.lang.annotation.Inherited; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; +import org.junit.jupiter.api.extension.ExtendWith; + +/** + * Declares that a test class needs the StarRocks server. Test and lifecycle methods receive it as a + * {@link StarRocksContainer} parameter, and fields marked {@link FixtureContainer} receive it too. + */ +@Documented +@Inherited +@Target(ElementType.TYPE) +@Retention(RetentionPolicy.RUNTIME) +@ExtendWith(ContainerExtension.class) +@ContainerFixture(StarRocksSpec.Factory.class) +public @interface StarRocksFixture {} diff --git a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/StarRocksSpec.java b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/StarRocksSpec.java new file mode 100644 index 00000000000..19501562fa4 --- /dev/null +++ b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/StarRocksSpec.java @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.integration.test.container; + +/** The StarRocks all-in-one server. There is one configuration, so the spec has no fields. */ +public record StarRocksSpec() implements ContainerSpec { + + @Override + public Class containerType() { + return StarRocksContainer.class; + } + + @Override + public StarRocksContainer getOrStart(ContainerSuite suite) { + suite.startStarRocksContainer(); + return suite.getStarRocksContainer(); + } + + static final class Factory implements ContainerFixture.SpecFactory { + @Override + public StarRocksSpec specFor(StarRocksFixture fixture) { + return new StarRocksSpec(); + } + } +} diff --git a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/TestContainerExtension.java b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/TestContainerExtension.java new file mode 100644 index 00000000000..b8ec75289bb --- /dev/null +++ b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/TestContainerExtension.java @@ -0,0 +1,285 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.integration.test.container; + +import static org.junit.platform.testkit.engine.EventConditions.event; +import static org.junit.platform.testkit.engine.EventConditions.finishedWithFailure; +import static org.junit.platform.testkit.engine.EventConditions.test; +import static org.junit.platform.testkit.engine.TestExecutionResultConditions.instanceOf; +import static org.junit.platform.testkit.engine.TestExecutionResultConditions.message; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestInstance; +import org.junit.jupiter.api.extension.ExtendWith; +import org.junit.jupiter.api.extension.ExtensionConfigurationException; +import org.junit.jupiter.api.extension.ParameterResolutionException; +import org.junit.platform.engine.discovery.DiscoverySelectors; +import org.junit.platform.testkit.engine.EngineExecutionResults; +import org.junit.platform.testkit.engine.EngineTestKit; +import org.mockito.Mockito; + +/** Runs small fixture classes through a real JUnit engine, with a fake spec instead of Docker. */ +public class TestContainerExtension { + + // Gradle would otherwise discover and run these nested fixtures directly; + // integration-test-common/build.gradle.kts excludes the tag. Keep the two in sync. + private static final String FIXTURE = "gravitino-test-fixture"; + + private static final List LOG = new ArrayList<>(); + private static final FakeContainer FIRST = Mockito.mock(FakeContainer.class); + private static final FakeContainer SECOND = Mockito.mock(FakeContainer.class); + + @BeforeEach + void clearLog() { + LOG.clear(); + } + + @Test + void testFixtureStartsBeforeAllAndInjectsByType() { + run(InjectedFixture.class).testEvents().assertStatistics(stats -> stats.succeeded(2).failed(0)); + + // The callback starts the container before @BeforeAll; each parameter then gets the running + // one. + Assertions.assertEquals( + List.of( + "start first", + "start first", + "beforeAll", + "start first", + "test", + "start first", + "test"), + LOG); + } + + @Test + void testFieldIsFilledBeforeAnOverriddenBeforeAll() { + run(FieldFixture.class).testEvents().assertStatistics(stats -> stats.succeeded(1).failed(0)); + + Assertions.assertEquals(List.of("start first", "start first", "setUp"), LOG); + } + + @Test + void testStaticFieldIsFilledBeforeAStaticBeforeAll() { + run(StaticFieldFixture.class) + .testEvents() + .assertStatistics(stats -> stats.succeeded(1).failed(0)); + } + + @Test + void testFinalFieldFailsClearly() { + run(FinalFieldFixture.class) + .testEvents() + .assertThatEvents() + .haveExactly( + 1, + event( + test("test"), + finishedWithFailure( + instanceOf(ExtensionConfigurationException.class), + message(m -> m.contains("must not be final"))))); + } + + @Test + void testSuperclassFixtureApplies() { + run(SubclassFixture.class).testEvents().assertStatistics(stats -> stats.succeeded(1).failed(0)); + } + + @Test + void testTwoFixturesOfOneTypeFailClearly() { + run(AmbiguousFixture.class) + .testEvents() + .assertThatEvents() + .haveExactly( + 1, + event( + test("test"), finishedWithFailure(instanceOf(ParameterResolutionException.class)))); + } + + @Test + void testSpecsReadsEveryFixture() { + Assertions.assertEquals( + List.of(new FakeSpec("first"), new FakeSpec("second")), + ContainerExtension.specs(AmbiguousFixture.class)); + Assertions.assertEquals( + List.of(new KafkaSpec()), ContainerExtension.specs(KafkaFixtureOnly.class)); + } + + private static EngineExecutionResults run(Class fixture) { + return EngineTestKit.engine("junit-jupiter") + .selectors(DiscoverySelectors.selectClass(fixture)) + .enableImplicitConfigurationParameters(false) + .execute(); + } + + // Only ever mocked, so the constructor never runs. + abstract static class FakeContainer extends BaseContainer { + FakeContainer() { + super((String) null, null, null, null, null, null, Optional.empty()); + } + } + + record FakeSpec(String name) implements ContainerSpec { + @Override + public Class containerType() { + return FakeContainer.class; + } + + @Override + public FakeContainer getOrStart(ContainerSuite suite) { + LOG.add("start " + name); + return name.equals("first") ? FIRST : SECOND; + } + } + + static final class FirstFactory implements ContainerFixture.SpecFactory { + @Override + public ContainerSpec specFor(FakeFirstFixture fixture) { + return new FakeSpec("first"); + } + } + + static final class SecondFactory implements ContainerFixture.SpecFactory { + @Override + public ContainerSpec specFor(FakeSecondFixture fixture) { + return new FakeSpec("second"); + } + } + + @Target(ElementType.TYPE) + @Retention(RetentionPolicy.RUNTIME) + @ExtendWith(ContainerExtension.class) + @ContainerFixture(FirstFactory.class) + @interface FakeFirstFixture {} + + @Target(ElementType.TYPE) + @Retention(RetentionPolicy.RUNTIME) + @ExtendWith(ContainerExtension.class) + @ContainerFixture(SecondFactory.class) + @interface FakeSecondFixture {} + + @Tag(FIXTURE) + @FakeFirstFixture + @TestInstance(TestInstance.Lifecycle.PER_CLASS) + static class InjectedFixture { + @BeforeAll + void beforeAll(FakeContainer container) { + Assertions.assertSame(FIRST, container); + LOG.add("beforeAll"); + } + + @Test + void first(FakeContainer container) { + Assertions.assertSame(FIRST, container); + LOG.add("test"); + } + + @Test + void second(BaseContainer container) { + Assertions.assertSame(FIRST, container); + LOG.add("test"); + } + } + + // Shaped like BaseIT: the subclass overrides a no-argument @BeforeAll method. + @TestInstance(TestInstance.Lifecycle.PER_CLASS) + static class SetUpBase { + @BeforeAll + void setUp() {} + } + + @Tag(FIXTURE) + @FakeFirstFixture + static class FieldFixture extends SetUpBase { + @FixtureContainer private FakeContainer container; + + @Override + @BeforeAll + void setUp() { + Assertions.assertSame(FIRST, container); + LOG.add("setUp"); + } + + @Test + void test() { + Assertions.assertSame(FIRST, container); + } + } + + @Tag(FIXTURE) + @FakeFirstFixture + static class StaticFieldFixture { + @FixtureContainer private static FakeContainer container; + + @BeforeAll + static void setUp() { + Assertions.assertSame(FIRST, container); + } + + @Test + void test() { + Assertions.assertSame(FIRST, container); + } + } + + @Tag(FIXTURE) + @FakeFirstFixture + static class FinalFieldFixture { + @SuppressWarnings("UnusedVariable") + @FixtureContainer + private final FakeContainer container = null; + + @Test + void test() {} + } + + @FakeFirstFixture + abstract static class FixtureBase {} + + @Tag(FIXTURE) + static class SubclassFixture extends FixtureBase { + @Test + void test(FakeContainer container) { + Assertions.assertSame(FIRST, container); + } + } + + @Tag(FIXTURE) + @FakeFirstFixture + @FakeSecondFixture + static class AmbiguousFixture { + @Test + void test(FakeContainer container) {} + } + + @Tag(FIXTURE) + @KafkaFixture + static class KafkaFixtureOnly {} +} diff --git a/trino-connector/integration-test/src/test/java/org/apache/gravitino/trino/connector/integration/test/TrinoTlsOAuthCredentialVendingIT.java b/trino-connector/integration-test/src/test/java/org/apache/gravitino/trino/connector/integration/test/TrinoTlsOAuthCredentialVendingIT.java index 40349d1b741..f23230e0bca 100644 --- a/trino-connector/integration-test/src/test/java/org/apache/gravitino/trino/connector/integration/test/TrinoTlsOAuthCredentialVendingIT.java +++ b/trino-connector/integration-test/src/test/java/org/apache/gravitino/trino/connector/integration/test/TrinoTlsOAuthCredentialVendingIT.java @@ -54,7 +54,9 @@ import org.apache.gravitino.client.GravitinoMetalake; import org.apache.gravitino.credential.CredentialConstants; import org.apache.gravitino.credential.S3TokenCredential; +import org.apache.gravitino.integration.test.container.FixtureContainer; import org.apache.gravitino.integration.test.container.GravitinoLocalStackContainer; +import org.apache.gravitino.integration.test.container.LocalStackFixture; import org.apache.gravitino.integration.test.container.TrinoContainer; import org.apache.gravitino.integration.test.util.BaseIT; import org.apache.gravitino.integration.test.util.ITUtils; @@ -76,6 +78,7 @@ */ @Tag("gravitino-docker-test") @EnabledIfSystemProperty(named = ITUtils.TEST_MODE, matches = ITUtils.DEPLOY_TEST_MODE) +@LocalStackFixture public class TrinoTlsOAuthCredentialVendingIT extends BaseIT { private static final Logger LOG = LoggerFactory.getLogger(TrinoTlsOAuthCredentialVendingIT.class); @@ -97,14 +100,12 @@ public class TrinoTlsOAuthCredentialVendingIT extends BaseIT { private OAuthServer oauthServer; private TrinoContainer trinoContainer; - private GravitinoLocalStackContainer localStack; + @FixtureContainer private GravitinoLocalStackContainer localStack; private Path trinoConfigDirectory; @BeforeAll @Override public void startIntegrationTest() throws Exception { - containerSuite.startLocalStackContainer(); - localStack = containerSuite.getLocalStackContainer(); createBucket(); containerSuite.startPostgreSQLContainer(TestDatabaseName.PG_ICEBERG_AUTHZ_IT); diff --git a/web-v2/integration-test/src/test/java/org/apache/gravitino/integration/test/web/ui/CatalogsPageDorisTest.java b/web-v2/integration-test/src/test/java/org/apache/gravitino/integration/test/web/ui/CatalogsPageDorisTest.java index 92c2fd12624..ed6483bba65 100644 --- a/web-v2/integration-test/src/test/java/org/apache/gravitino/integration/test/web/ui/CatalogsPageDorisTest.java +++ b/web-v2/integration-test/src/test/java/org/apache/gravitino/integration/test/web/ui/CatalogsPageDorisTest.java @@ -27,7 +27,9 @@ import org.apache.gravitino.NameIdentifier; import org.apache.gravitino.client.GravitinoAdminClient; import org.apache.gravitino.client.GravitinoMetalake; -import org.apache.gravitino.integration.test.container.ContainerSuite; +import org.apache.gravitino.integration.test.container.DorisContainer; +import org.apache.gravitino.integration.test.container.DorisFixture; +import org.apache.gravitino.integration.test.container.FixtureContainer; import org.apache.gravitino.integration.test.web.ui.pages.CatalogsPage; import org.apache.gravitino.integration.test.web.ui.pages.MetalakePage; import org.apache.gravitino.integration.test.web.ui.utils.BaseWebIT; @@ -46,11 +48,12 @@ @Tag("gravitino-docker-test") @TestMethodOrder(MethodOrderer.OrderAnnotation.class) +@DorisFixture public class CatalogsPageDorisTest extends BaseWebIT { + @FixtureContainer private DorisContainer doris; + private MetalakePage metalakePage; private CatalogsPage catalogsPage; - - private static final ContainerSuite containerSuite = ContainerSuite.getInstance(); protected static GravitinoAdminClient gravitinoClient; private static GravitinoMetalake metalake; @@ -79,13 +82,8 @@ public void before() throws Exception { gravitinoClient = getGravitinoClient(); gravitinoUri = String.format("http://127.0.0.1:%d", getGravitinoServerPort()); - containerSuite.startDorisContainer(); - dorisJdbcConnectionUri = - String.format( - "jdbc:mysql://%s:%d/", - containerSuite.getDorisContainer().getContainerIpAddress(), - containerSuite.getDorisContainer().getFeMysqlPort()); + String.format("jdbc:mysql://%s:%d/", doris.getContainerIpAddress(), doris.getFeMysqlPort()); LOG.info("Doris jdbc url: {}", dorisJdbcConnectionUri); metalakePage = new MetalakePage(driver); diff --git a/web-v2/integration-test/src/test/java/org/apache/gravitino/integration/test/web/ui/CatalogsPageKafkaTest.java b/web-v2/integration-test/src/test/java/org/apache/gravitino/integration/test/web/ui/CatalogsPageKafkaTest.java index 04c120967a0..a7670595488 100644 --- a/web-v2/integration-test/src/test/java/org/apache/gravitino/integration/test/web/ui/CatalogsPageKafkaTest.java +++ b/web-v2/integration-test/src/test/java/org/apache/gravitino/integration/test/web/ui/CatalogsPageKafkaTest.java @@ -22,7 +22,9 @@ import java.util.Arrays; import java.util.List; import org.apache.gravitino.client.GravitinoAdminClient; -import org.apache.gravitino.integration.test.container.ContainerSuite; +import org.apache.gravitino.integration.test.container.FixtureContainer; +import org.apache.gravitino.integration.test.container.KafkaContainer; +import org.apache.gravitino.integration.test.container.KafkaFixture; import org.apache.gravitino.integration.test.web.ui.pages.CatalogsPage; import org.apache.gravitino.integration.test.web.ui.pages.MetalakePage; import org.apache.gravitino.integration.test.web.ui.utils.BaseWebIT; @@ -36,11 +38,13 @@ @Tag("gravitino-docker-test") @TestMethodOrder(MethodOrderer.OrderAnnotation.class) +@KafkaFixture public class CatalogsPageKafkaTest extends BaseWebIT { + @FixtureContainer private KafkaContainer kafka; + private MetalakePage metalakePage; private CatalogsPage catalogsPage; - private static final ContainerSuite containerSuite = ContainerSuite.getInstance(); protected static GravitinoAdminClient gravitinoClient; protected static String gravitinoUri = "http://127.0.0.1:8090"; @@ -62,9 +66,7 @@ public void before() throws Exception { gravitinoUri = String.format("http://127.0.0.1:%d", getGravitinoServerPort()); - containerSuite.startKafkaContainer(); - - String address = containerSuite.getKafkaContainer().getContainerIpAddress(); + String address = kafka.getContainerIpAddress(); kafkaUri = String.format("%s:%d", address, DEFAULT_BROKER_PORT); metalakePage = new MetalakePage(driver); diff --git a/web/integration-test/src/test/java/org/apache/gravitino/integration/test/web/ui/CatalogsPageDorisTest.java b/web/integration-test/src/test/java/org/apache/gravitino/integration/test/web/ui/CatalogsPageDorisTest.java index d3258def9de..bdbfd562e8b 100644 --- a/web/integration-test/src/test/java/org/apache/gravitino/integration/test/web/ui/CatalogsPageDorisTest.java +++ b/web/integration-test/src/test/java/org/apache/gravitino/integration/test/web/ui/CatalogsPageDorisTest.java @@ -27,7 +27,9 @@ import org.apache.gravitino.NameIdentifier; import org.apache.gravitino.client.GravitinoAdminClient; import org.apache.gravitino.client.GravitinoMetalake; -import org.apache.gravitino.integration.test.container.ContainerSuite; +import org.apache.gravitino.integration.test.container.DorisContainer; +import org.apache.gravitino.integration.test.container.DorisFixture; +import org.apache.gravitino.integration.test.container.FixtureContainer; import org.apache.gravitino.integration.test.web.ui.pages.CatalogsPage; import org.apache.gravitino.integration.test.web.ui.pages.MetalakePage; import org.apache.gravitino.integration.test.web.ui.utils.BaseWebIT; @@ -46,11 +48,12 @@ @Tag("gravitino-docker-test") @TestMethodOrder(MethodOrderer.OrderAnnotation.class) +@DorisFixture public class CatalogsPageDorisTest extends BaseWebIT { + @FixtureContainer private DorisContainer doris; + private MetalakePage metalakePage; private CatalogsPage catalogsPage; - - private static final ContainerSuite containerSuite = ContainerSuite.getInstance(); protected static GravitinoAdminClient gravitinoClient; private static GravitinoMetalake metalake; @@ -79,13 +82,8 @@ public void before() throws Exception { gravitinoClient = getGravitinoClient(); gravitinoUri = String.format("http://127.0.0.1:%d", getGravitinoServerPort()); - containerSuite.startDorisContainer(); - dorisJdbcConnectionUri = - String.format( - "jdbc:mysql://%s:%d/", - containerSuite.getDorisContainer().getContainerIpAddress(), - containerSuite.getDorisContainer().getFeMysqlPort()); + String.format("jdbc:mysql://%s:%d/", doris.getContainerIpAddress(), doris.getFeMysqlPort()); LOG.info("Doris jdbc url: {}", dorisJdbcConnectionUri); metalakePage = new MetalakePage(driver); diff --git a/web/integration-test/src/test/java/org/apache/gravitino/integration/test/web/ui/CatalogsPageKafkaTest.java b/web/integration-test/src/test/java/org/apache/gravitino/integration/test/web/ui/CatalogsPageKafkaTest.java index 61278114e3d..c27bea0c1aa 100644 --- a/web/integration-test/src/test/java/org/apache/gravitino/integration/test/web/ui/CatalogsPageKafkaTest.java +++ b/web/integration-test/src/test/java/org/apache/gravitino/integration/test/web/ui/CatalogsPageKafkaTest.java @@ -22,7 +22,9 @@ import java.util.Arrays; import java.util.List; import org.apache.gravitino.client.GravitinoAdminClient; -import org.apache.gravitino.integration.test.container.ContainerSuite; +import org.apache.gravitino.integration.test.container.FixtureContainer; +import org.apache.gravitino.integration.test.container.KafkaContainer; +import org.apache.gravitino.integration.test.container.KafkaFixture; import org.apache.gravitino.integration.test.web.ui.pages.CatalogsPage; import org.apache.gravitino.integration.test.web.ui.pages.MetalakePage; import org.apache.gravitino.integration.test.web.ui.utils.BaseWebIT; @@ -36,11 +38,13 @@ @Tag("gravitino-docker-test") @TestMethodOrder(MethodOrderer.OrderAnnotation.class) +@KafkaFixture public class CatalogsPageKafkaTest extends BaseWebIT { + @FixtureContainer private KafkaContainer kafka; + private MetalakePage metalakePage; private CatalogsPage catalogsPage; - private static final ContainerSuite containerSuite = ContainerSuite.getInstance(); protected static GravitinoAdminClient gravitinoClient; protected static String gravitinoUri = "http://127.0.0.1:8090"; @@ -62,9 +66,7 @@ public void before() throws Exception { gravitinoUri = String.format("http://127.0.0.1:%d", getGravitinoServerPort()); - containerSuite.startKafkaContainer(); - - String address = containerSuite.getKafkaContainer().getContainerIpAddress(); + String address = kafka.getContainerIpAddress(); kafkaUri = String.format("%s:%d", address, DEFAULT_BROKER_PORT); metalakePage = new MetalakePage(driver); From 79ebbfc374d38d04b252bfc1c4a0eb591f4d9467 Mon Sep 17 00:00:00 2001 From: Nevin Zheng Date: Mon, 19 Oct 2026 13:57:45 -0700 Subject: [PATCH 10/14] [#14002] improvement(test): Add TestImages and fixtures for Moto, Ranger and Hive (part 2/6) - TestImages declares every test image in one place. - BaseContainer subclasses can supply their own container (ready for the official Testcontainers modules). - Moto, Ranger and Hive (version, auth) fixtures. - A container whose start fails is closed instead of leaking. --- .../test/TestChainedAuthorizationIT.java | 2 +- .../test/RangerAuthorizationHDFSPluginIT.java | 7 +- .../test/RangerAuthorizationPluginIT.java | 7 +- .../integration/test/RangerBaseE2EIT.java | 6 + .../integration/test/RangerFilesetIT.java | 10 +- .../integration/test/RangerHiveE2EIT.java | 2 +- .../integration/test/RangerHiveHdfsE2EIT.java | 2 +- .../ranger/integration/test/RangerHiveIT.java | 6 +- .../ranger/integration/test/RangerITEnv.java | 20 +-- .../integration/test/RangerIcebergE2EIT.java | 2 +- .../integration/test/RangerPaimonE2EIT.java | 2 +- .../integration/test/MotoGlueCatalogIT.java | 32 +--- .../integration/test/CatalogHiveViewIT.java | 15 +- .../integration/test/ProxyCatalogHiveIT.java | 23 +-- .../gravitino/hive/client/TestHive2HMS.java | 20 +-- .../hive/client/TestHive2HMSWithKerberos.java | 18 +- .../gravitino/hive/client/TestHive3HMS.java | 13 +- .../hive/client/TestHive3HMSWithCatalog.java | 2 +- .../hive/client/TestHive3HMSWithKerberos.java | 14 +- .../test/container/BaseContainer.java | 70 ++++++-- .../test/container/ClickHouseContainer.java | 2 +- .../test/container/ContainerExtension.java | 8 +- .../test/container/ContainerSlot.java | 6 + .../test/container/ContainerSuite.java | 66 ++++++-- .../test/container/DatabaseContainer.java | 10 ++ .../test/container/DorisContainer.java | 4 +- .../test/container/DorisImageName.java | 13 +- .../GravitinoLocalStackContainer.java | 17 +- .../container/GravitinoMotoContainer.java | 7 +- .../integration/test/container/HiveAuth.java | 32 ++++ .../test/container/HiveContainer.java | 5 +- .../test/container/HiveFixture.java | 56 +++++++ .../integration/test/container/HiveSpec.java | 59 +++++++ .../test/container/HiveVersion.java | 42 +++++ .../test/container/KafkaContainer.java | 3 +- .../test/container/MotoFixture.java | 47 ++++++ .../integration/test/container/MotoSpec.java | 41 +++++ .../test/container/MySQLContainer.java | 2 +- .../test/container/OceanBaseContainer.java | 2 +- .../test/container/PGImageName.java | 10 +- .../test/container/RangerContainer.java | 2 +- .../test/container/RangerFixture.java | 47 ++++++ .../test/container/RangerSpec.java | 41 +++++ .../test/container/RustFSContainer.java | 34 +--- .../test/container/StarRocksContainer.java | 2 +- .../test/container/TestBaseContainer.java | 141 ++++++++++++++++ .../container/TestContainerExtension.java | 33 +++- .../test/container/TestContainerSlot.java | 43 +++++ .../test/container/TestContainerSuite.java | 44 ++++- .../TestGravitinoLocalStackContainer.java | 8 + .../test/container/TestImages.java | 157 ++++++++++++++++++ .../test/container/TestMotoSpec.java | 45 +++++ .../test/container/TestRangerSpec.java | 56 +++++++ .../test/container/TestRustFSContainer.java | 10 +- .../test/container/TestTestImages.java | 77 +++++++++ .../test/container/TrinoContainer.java | 2 +- .../test/container/ZooKeeperContainer.java | 2 +- 57 files changed, 1233 insertions(+), 216 deletions(-) create mode 100644 integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/HiveAuth.java create mode 100644 integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/HiveFixture.java create mode 100644 integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/HiveSpec.java create mode 100644 integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/HiveVersion.java create mode 100644 integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/MotoFixture.java create mode 100644 integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/MotoSpec.java create mode 100644 integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/RangerFixture.java create mode 100644 integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/RangerSpec.java create mode 100644 integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/TestBaseContainer.java create mode 100644 integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/TestImages.java create mode 100644 integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/TestMotoSpec.java create mode 100644 integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/TestRangerSpec.java create mode 100644 integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/TestTestImages.java diff --git a/authorizations/authorization-chain/src/test/java/org/apache/gravitino/authorization/chain/integration/test/TestChainedAuthorizationIT.java b/authorizations/authorization-chain/src/test/java/org/apache/gravitino/authorization/chain/integration/test/TestChainedAuthorizationIT.java index 30b45b2fef1..d9d823cff0e 100644 --- a/authorizations/authorization-chain/src/test/java/org/apache/gravitino/authorization/chain/integration/test/TestChainedAuthorizationIT.java +++ b/authorizations/authorization-chain/src/test/java/org/apache/gravitino/authorization/chain/integration/test/TestChainedAuthorizationIT.java @@ -80,7 +80,7 @@ public void startIntegrationTest() throws Exception { registerCustomConfigs(configs); super.startIntegrationTest(); - RangerITEnv.init(RangerBaseE2EIT.metalakeName, false); + RangerITEnv.init(ranger, RangerBaseE2EIT.metalakeName, false); RangerITEnv.startHiveRangerContainer(); HIVE_METASTORE_URIS = diff --git a/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerAuthorizationHDFSPluginIT.java b/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerAuthorizationHDFSPluginIT.java index be81b390039..a55e453a93b 100644 --- a/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerAuthorizationHDFSPluginIT.java +++ b/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerAuthorizationHDFSPluginIT.java @@ -40,6 +40,8 @@ import org.apache.gravitino.authorization.common.PathBasedSecurableObject; import org.apache.gravitino.authorization.ranger.RangerAuthorizationPlugin; import org.apache.gravitino.catalog.CatalogDispatcher; +import org.apache.gravitino.integration.test.container.RangerContainer; +import org.apache.gravitino.integration.test.container.RangerFixture; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeAll; @@ -49,6 +51,7 @@ import org.mockito.Mockito; @Tag("gravitino-docker-test") +@RangerFixture public class RangerAuthorizationHDFSPluginIT { private static RangerAuthorizationPlugin rangerAuthPlugin; @@ -57,8 +60,8 @@ public class RangerAuthorizationHDFSPluginIT { private static CatalogDispatcher previousInternalCatalogDispatcher; @BeforeAll - public static void setup() throws Exception { - RangerITEnv.init(RangerITEnv.currentFunName(), true); + public static void setup(RangerContainer ranger) throws Exception { + RangerITEnv.init(ranger, RangerITEnv.currentFunName(), true); rangerAuthPlugin = RangerITEnv.rangerAuthHDFSPlugin; previousCatalogDispatcher = (CatalogDispatcher) diff --git a/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerAuthorizationPluginIT.java b/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerAuthorizationPluginIT.java index 0f16927bc79..5b9eb1f4a26 100644 --- a/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerAuthorizationPluginIT.java +++ b/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerAuthorizationPluginIT.java @@ -33,6 +33,8 @@ import org.apache.gravitino.authorization.ranger.RangerAuthorizationPlugin; import org.apache.gravitino.authorization.ranger.RangerHadoopSQLMetadataObject; import org.apache.gravitino.authorization.ranger.RangerHelper; +import org.apache.gravitino.integration.test.container.RangerContainer; +import org.apache.gravitino.integration.test.container.RangerFixture; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeAll; @@ -40,12 +42,13 @@ import org.junit.jupiter.api.Test; @Tag("gravitino-docker-test") +@RangerFixture public class RangerAuthorizationPluginIT { private static RangerAuthorizationPlugin rangerAuthPlugin; @BeforeAll - public static void setup() { - RangerITEnv.init(RangerITEnv.currentFunName(), true); + public static void setup(RangerContainer ranger) { + RangerITEnv.init(ranger, RangerITEnv.currentFunName(), true); rangerAuthPlugin = RangerITEnv.rangerAuthHivePlugin; } diff --git a/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerBaseE2EIT.java b/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerBaseE2EIT.java index b1f1dc32051..67e21db183e 100644 --- a/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerBaseE2EIT.java +++ b/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerBaseE2EIT.java @@ -42,6 +42,9 @@ import org.apache.gravitino.authorization.SecurableObject; import org.apache.gravitino.authorization.SecurableObjects; import org.apache.gravitino.client.GravitinoMetalake; +import org.apache.gravitino.integration.test.container.FixtureContainer; +import org.apache.gravitino.integration.test.container.RangerContainer; +import org.apache.gravitino.integration.test.container.RangerFixture; import org.apache.gravitino.integration.test.util.BaseIT; import org.apache.gravitino.integration.test.util.GravitinoITUtils; import org.apache.gravitino.rel.TableChange; @@ -54,6 +57,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +@RangerFixture public abstract class RangerBaseE2EIT extends BaseIT { private static final Logger LOG = LoggerFactory.getLogger(RangerBaseE2EIT.class); public static final String catalogName = GravitinoITUtils.genRandomName("catalog").toLowerCase(); @@ -105,6 +109,8 @@ public abstract class RangerBaseE2EIT extends BaseIT { protected static final String SQL_DROP_TABLE = String.format("DROP TABLE %s", tableName); + @FixtureContainer protected RangerContainer ranger; + protected static void generateRangerSparkSecurityXML(String modeName) throws IOException { String templatePath = String.join( diff --git a/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerFilesetIT.java b/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerFilesetIT.java index be3b87f0b43..b0f0e3ae36f 100644 --- a/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerFilesetIT.java +++ b/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerFilesetIT.java @@ -52,8 +52,10 @@ import org.apache.gravitino.client.GravitinoMetalake; import org.apache.gravitino.file.Fileset; import org.apache.gravitino.file.FilesetChange; +import org.apache.gravitino.integration.test.container.FixtureContainer; import org.apache.gravitino.integration.test.container.HiveContainer; import org.apache.gravitino.integration.test.container.RangerContainer; +import org.apache.gravitino.integration.test.container.RangerFixture; import org.apache.gravitino.integration.test.util.BaseIT; import org.apache.gravitino.integration.test.util.GravitinoITUtils; import org.apache.hadoop.conf.Configuration; @@ -73,6 +75,7 @@ import org.slf4j.LoggerFactory; @Tag("gravitino-docker-test") +@RangerFixture public class RangerFilesetIT extends BaseIT { private static final Logger LOG = LoggerFactory.getLogger(RangerFilesetIT.class); @@ -85,6 +88,7 @@ public class RangerFilesetIT extends BaseIT { private FileSystem fileSystem; private GravitinoMetalake metalake; private Catalog catalog; + @FixtureContainer private RangerContainer ranger; @BeforeAll public void startIntegrationTest() throws Exception { @@ -100,13 +104,11 @@ public void startIntegrationTest() throws Exception { registerCustomConfigs(configs); super.startIntegrationTest(); - RangerITEnv.init(metalakeName, false); + RangerITEnv.init(ranger, metalakeName, false); RangerITEnv.startHiveRangerContainer(); RANGER_ADMIN_URL = - String.format( - "http://%s:%d", - containerSuite.getRangerContainer().getContainerIpAddress(), RANGER_SERVER_PORT); + String.format("http://%s:%d", ranger.getContainerIpAddress(), RANGER_SERVER_PORT); Configuration conf = new Configuration(); conf.set("fs.defaultFS", defaultBaseLocation()); diff --git a/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerHiveE2EIT.java b/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerHiveE2EIT.java index fc1ad726ccc..558eca5478a 100644 --- a/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerHiveE2EIT.java +++ b/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerHiveE2EIT.java @@ -65,7 +65,7 @@ public void startIntegrationTest() throws Exception { registerCustomConfigs(configs); super.startIntegrationTest(); - RangerITEnv.init(RangerBaseE2EIT.metalakeName, true); + RangerITEnv.init(ranger, RangerBaseE2EIT.metalakeName, true); RangerITEnv.startHiveRangerContainer(); HIVE_METASTORE_URIS = diff --git a/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerHiveHdfsE2EIT.java b/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerHiveHdfsE2EIT.java index ea107947c25..03e48b6b776 100644 --- a/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerHiveHdfsE2EIT.java +++ b/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerHiveHdfsE2EIT.java @@ -94,7 +94,7 @@ public void startIntegrationTest() throws Exception { registerCustomConfigs(configs); super.startIntegrationTest(); - RangerITEnv.init(RangerBaseE2EIT.metalakeName, false); + RangerITEnv.init(ranger, RangerBaseE2EIT.metalakeName, false); RangerITEnv.startHiveRangerContainer(); DEFAULT_FS = diff --git a/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerHiveIT.java b/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerHiveIT.java index 5828d6cbbce..12a5b0fc1a8 100644 --- a/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerHiveIT.java +++ b/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerHiveIT.java @@ -63,6 +63,7 @@ import org.apache.gravitino.connector.authorization.BaseAuthorization; import org.apache.gravitino.exceptions.AuthorizationPluginException; import org.apache.gravitino.integration.test.container.RangerContainer; +import org.apache.gravitino.integration.test.container.RangerFixture; import org.apache.gravitino.integration.test.util.GravitinoITUtils; import org.apache.gravitino.meta.AuditInfo; import org.apache.gravitino.meta.GroupEntity; @@ -83,6 +84,7 @@ import org.slf4j.LoggerFactory; @Tag("gravitino-docker-test") +@RangerFixture public class RangerHiveIT { private static final Logger LOG = LoggerFactory.getLogger(RangerHiveIT.class); @@ -92,8 +94,8 @@ public class RangerHiveIT { AuditInfo.builder().withCreator("test").withCreateTime(Instant.now()).build(); @BeforeAll - public static void setup() { - RangerITEnv.init(RangerITEnv.currentFunName(), true); + public static void setup(RangerContainer ranger) { + RangerITEnv.init(ranger, RangerITEnv.currentFunName(), true); rangerAuthHivePlugin = RangerITEnv.rangerAuthHivePlugin; rangerHelper = RangerITEnv.rangerHelper; diff --git a/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerITEnv.java b/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerITEnv.java index 997ba63aca5..d81b0a98803 100644 --- a/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerITEnv.java +++ b/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerITEnv.java @@ -92,14 +92,14 @@ public class RangerITEnv { protected static RangerHelper rangerHelper; protected static RangerHelper rangerHDFSHelper; public static String RANGER_ADMIN_URL = null; + private static RangerContainer rangerContainer; - public static void init(String metalakeName, boolean allowAnyoneAccessHDFS) { - containerSuite.startRangerContainer(); + public static void init( + RangerContainer ranger, String metalakeName, boolean allowAnyoneAccessHDFS) { + rangerContainer = ranger; RANGER_ADMIN_URL = - String.format( - "http://%s:%d", - containerSuite.getRangerContainer().getContainerIpAddress(), RANGER_SERVER_PORT); - rangerClient = containerSuite.getRangerContainer().rangerClient; + String.format("http://%s:%d", rangerContainer.getContainerIpAddress(), RANGER_SERVER_PORT); + rangerClient = rangerContainer.rangerClient; rangerAuthHivePlugin = new RangerAuthorizationHadoopSQLPlugin( @@ -108,8 +108,7 @@ public static void init(String metalakeName, boolean allowAnyoneAccessHDFS) { RangerAuthorizationProperties.RANGER_ADMIN_URL, String.format( "http://%s:%d", - containerSuite.getRangerContainer().getContainerIpAddress(), - RangerContainer.RANGER_SERVER_PORT), + rangerContainer.getContainerIpAddress(), RangerContainer.RANGER_SERVER_PORT), RangerAuthorizationProperties.RANGER_AUTH_TYPE, RangerContainer.authType, RangerAuthorizationProperties.RANGER_USERNAME, @@ -135,7 +134,7 @@ public static void init(String metalakeName, boolean allowAnyoneAccessHDFS) { RangerAuthorizationProperties.RANGER_ADMIN_URL, String.format( "http://%s:%d", - containerSuite.getRangerContainer().getContainerIpAddress(), + rangerContainer.getContainerIpAddress(), RangerContainer.RANGER_SERVER_PORT), RangerAuthorizationProperties.RANGER_AUTH_TYPE, RangerContainer.authType, @@ -199,8 +198,7 @@ public static void startHiveRangerContainer() { RangerContainer.DOCKER_ENV_RANGER_SERVER_URL, String.format( "http://%s:%d", - containerSuite.getRangerContainer().getContainerIpAddress(), - RangerContainer.RANGER_SERVER_PORT), + rangerContainer.getContainerIpAddress(), RangerContainer.RANGER_SERVER_PORT), RangerContainer.DOCKER_ENV_RANGER_HIVE_REPOSITORY_NAME, RangerITEnv.RANGER_HIVE_REPO_NAME, RangerContainer.DOCKER_ENV_RANGER_HDFS_REPOSITORY_NAME, diff --git a/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerIcebergE2EIT.java b/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerIcebergE2EIT.java index 78941a225d4..1acacf6b193 100644 --- a/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerIcebergE2EIT.java +++ b/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerIcebergE2EIT.java @@ -66,7 +66,7 @@ public void startIntegrationTest() throws Exception { registerCustomConfigs(configs); super.startIntegrationTest(); - RangerITEnv.init(RangerBaseE2EIT.metalakeName, true); + RangerITEnv.init(ranger, RangerBaseE2EIT.metalakeName, true); RangerITEnv.startHiveRangerContainer(); HIVE_METASTORE_URIS = diff --git a/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerPaimonE2EIT.java b/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerPaimonE2EIT.java index 015c444b795..26bee1143ae 100644 --- a/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerPaimonE2EIT.java +++ b/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerPaimonE2EIT.java @@ -65,7 +65,7 @@ public void startIntegrationTest() throws Exception { registerCustomConfigs(configs); super.startIntegrationTest(); - RangerITEnv.init(RangerBaseE2EIT.metalakeName, true); + RangerITEnv.init(ranger, RangerBaseE2EIT.metalakeName, true); RangerITEnv.startHiveRangerContainer(); HIVE_METASTORE_URIS = diff --git a/catalogs/catalog-glue/src/test/java/org/apache/gravitino/catalog/glue/integration/test/MotoGlueCatalogIT.java b/catalogs/catalog-glue/src/test/java/org/apache/gravitino/catalog/glue/integration/test/MotoGlueCatalogIT.java index 8fcc8a5d5f8..192d81c70d7 100644 --- a/catalogs/catalog-glue/src/test/java/org/apache/gravitino/catalog/glue/integration/test/MotoGlueCatalogIT.java +++ b/catalogs/catalog-glue/src/test/java/org/apache/gravitino/catalog/glue/integration/test/MotoGlueCatalogIT.java @@ -20,9 +20,10 @@ import java.util.Map; import org.apache.gravitino.catalog.glue.GlueConstants; +import org.apache.gravitino.integration.test.container.ContainerSuite; +import org.apache.gravitino.integration.test.container.FixtureContainer; import org.apache.gravitino.integration.test.container.GravitinoMotoContainer; -import org.junit.jupiter.api.AfterAll; -import org.junit.jupiter.api.BeforeAll; +import org.apache.gravitino.integration.test.container.MotoFixture; import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.TestInstance; @@ -31,34 +32,15 @@ * *

Requires Docker. Skipped by default when the {@code gravitino-docker-test} tag is excluded. * Override the container image with the {@code GRAVITINO_CI_MOTO_DOCKER_IMAGE} environment - * variable. + * variable. The shared Moto server starts through {@link ContainerSuite} and closes at run end. */ +@MotoFixture @Tag("gravitino-docker-test") @TestInstance(TestInstance.Lifecycle.PER_CLASS) class MotoGlueCatalogIT extends AbstractGlueCatalogIT { - private static final String FALLBACK_IMAGE = "motoserver/moto:5.1.4"; - - private GravitinoMotoContainer moto; - - @Override - @BeforeAll - void initOps() { - String image = GravitinoMotoContainer.DEFAULT_IMAGE; - if (image == null || image.isBlank()) { - image = FALLBACK_IMAGE; - } - moto = GravitinoMotoContainer.builder().withImage(image).build(); - moto.start(); - super.initOps(); - } - - @AfterAll - void stopContainer() { - if (moto != null) { - moto.close(); - } - } + // Filled when the instance is created, before the inherited @BeforeAll initOps() reads it. + @FixtureContainer private GravitinoMotoContainer moto; @Override protected Map catalogConfig() { diff --git a/catalogs/catalog-hive/src/test/java/org/apache/gravitino/catalog/hive/integration/test/CatalogHiveViewIT.java b/catalogs/catalog-hive/src/test/java/org/apache/gravitino/catalog/hive/integration/test/CatalogHiveViewIT.java index ef823785a91..dacb51b8580 100644 --- a/catalogs/catalog-hive/src/test/java/org/apache/gravitino/catalog/hive/integration/test/CatalogHiveViewIT.java +++ b/catalogs/catalog-hive/src/test/java/org/apache/gravitino/catalog/hive/integration/test/CatalogHiveViewIT.java @@ -18,7 +18,6 @@ */ package org.apache.gravitino.catalog.hive.integration.test; -import com.google.common.collect.ImmutableMap; import com.google.common.collect.Maps; import java.util.Collections; import java.util.Locale; @@ -31,8 +30,9 @@ import org.apache.gravitino.exceptions.NoSuchSchemaException; import org.apache.gravitino.exceptions.NoSuchViewException; import org.apache.gravitino.exceptions.ViewAlreadyExistsException; -import org.apache.gravitino.integration.test.container.ContainerSuite; +import org.apache.gravitino.integration.test.container.FixtureContainer; import org.apache.gravitino.integration.test.container.HiveContainer; +import org.apache.gravitino.integration.test.container.HiveFixture; import org.apache.gravitino.integration.test.util.BaseIT; import org.apache.gravitino.integration.test.util.GravitinoITUtils; import org.apache.gravitino.rel.Column; @@ -52,13 +52,14 @@ /** Integration tests for Hive catalog view CRUD operations. */ @Tag("gravitino-docker-test") @TestInstance(TestInstance.Lifecycle.PER_CLASS) +@HiveFixture public class CatalogHiveViewIT extends BaseIT { private static final String PROVIDER = "hive"; private static final String VIEW_COMMENT = "hive view comment"; private static final String HIVE_DIALECT = "hive"; - private final ContainerSuite containerSuite = ContainerSuite.getInstance(); + @FixtureContainer private HiveContainer hive; private HiveServer2Sql hiveSql; private String metalakeName; private String catalogName; @@ -71,8 +72,6 @@ public class CatalogHiveViewIT extends BaseIT { @BeforeAll public void setup() throws Exception { super.startIntegrationTest(); - containerSuite.startHiveContainer( - ImmutableMap.of(HiveContainer.HIVE_RUNTIME_VERSION, HiveContainer.HIVE2)); metalakeName = GravitinoITUtils.genRandomName("hive_view_it_metalake"); catalogName = GravitinoITUtils.genRandomName("hive_view_it_catalog"); @@ -83,9 +82,7 @@ public void setup() throws Exception { String hmsUri = String.format( - "thrift://%s:%d", - containerSuite.getHiveContainer().getContainerIpAddress(), - HiveContainer.HIVE_METASTORE_PORT); + "thrift://%s:%d", hive.getContainerIpAddress(), HiveContainer.HIVE_METASTORE_PORT); Map catalogProperties = Maps.newHashMap(); catalogProperties.put(HiveConstants.METASTORE_URIS, hmsUri); @@ -95,7 +92,7 @@ public void setup() throws Exception { catalogName, Catalog.Type.RELATIONAL, PROVIDER, "comment", catalogProperties); viewCatalog = catalog.asViewCatalog(); catalog.asSchemas().createSchema(schemaName, "schema comment", Maps.newHashMap()); - hiveSql = new HiveServer2Sql(containerSuite.getHiveContainer()); + hiveSql = new HiveServer2Sql(hive); } @AfterEach diff --git a/catalogs/catalog-hive/src/test/java/org/apache/gravitino/catalog/hive/integration/test/ProxyCatalogHiveIT.java b/catalogs/catalog-hive/src/test/java/org/apache/gravitino/catalog/hive/integration/test/ProxyCatalogHiveIT.java index 43a5b7d8523..6a8d1e0afbc 100644 --- a/catalogs/catalog-hive/src/test/java/org/apache/gravitino/catalog/hive/integration/test/ProxyCatalogHiveIT.java +++ b/catalogs/catalog-hive/src/test/java/org/apache/gravitino/catalog/hive/integration/test/ProxyCatalogHiveIT.java @@ -41,8 +41,9 @@ import org.apache.gravitino.hive.HivePartition; import org.apache.gravitino.hive.HiveSchema; import org.apache.gravitino.hive.HiveTable; -import org.apache.gravitino.integration.test.container.ContainerSuite; +import org.apache.gravitino.integration.test.container.FixtureContainer; import org.apache.gravitino.integration.test.container.HiveContainer; +import org.apache.gravitino.integration.test.container.HiveFixture; import org.apache.gravitino.integration.test.util.BaseIT; import org.apache.gravitino.integration.test.util.GravitinoITUtils; import org.apache.gravitino.rel.Column; @@ -69,6 +70,7 @@ @Tag("gravitino-docker-test") @TestInstance(TestInstance.Lifecycle.PER_CLASS) +@HiveFixture public class ProxyCatalogHiveIT extends BaseIT { public static final String METALAKE_NAME = @@ -81,7 +83,7 @@ public class ProxyCatalogHiveIT extends BaseIT { private static final String HADOOP_USER_NAME = "HADOOP_USER_NAME"; private static final String HMS_CATALOG = ""; - private static final ContainerSuite containerSuite = ContainerSuite.getInstance(); + @FixtureContainer private static HiveContainer hive; private static GravitinoMetalake metalake; private static Catalog catalog; private static HiveClientPool hiveClientPool; @@ -107,12 +109,9 @@ public void startIntegrationTest() throws Exception { configs.put(Configs.AUTHENTICATORS.getKey(), AuthenticatorType.SIMPLE.name().toLowerCase()); registerCustomConfigs(configs); super.startIntegrationTest(); - containerSuite.startHiveContainer(); HIVE_METASTORE_URIS = String.format( - "thrift://%s:%d", - containerSuite.getHiveContainer().getContainerIpAddress(), - HiveContainer.HIVE_METASTORE_PORT); + "thrift://%s:%d", hive.getContainerIpAddress(), HiveContainer.HIVE_METASTORE_PORT); HiveConf hiveConf = new HiveConf(); hiveConf.set(HiveConf.ConfVars.METASTOREURIS.varname, HIVE_METASTORE_URIS); @@ -126,9 +125,7 @@ public void startIntegrationTest() throws Exception { conf.set( "fs.defaultFS", String.format( - "hdfs://%s:%d", - containerSuite.getHiveContainer().getContainerIpAddress(), - HiveContainer.HDFS_DEFAULTFS_PORT)); + "hdfs://%s:%d", hive.getContainerIpAddress(), HiveContainer.HDFS_DEFAULTFS_PORT)); hdfs = FileSystem.get(conf); JettyServerConfig jettyServerConfig = JettyServerConfig.fromConfig(serverConfig, WEBSERVER_CONF_PREFIX); @@ -175,7 +172,7 @@ public void testOperateSchema() throws Exception { "location", String.format( "hdfs://%s:%d/user/hive/warehouse/%s.db", - containerSuite.getHiveContainer().getContainerIpAddress(), + hive.getContainerIpAddress(), HiveContainer.HDFS_DEFAULTFS_PORT, anotherSchemaName.toLowerCase())); SupportsSchemas schemas = anotherCatalog.asSchemas(); @@ -197,9 +194,7 @@ public void testOperateSchema() throws Exception { "location", String.format( "hdfs://%s:%d/user/hive/warehouse/%s.db", - containerSuite.getHiveContainer().getContainerIpAddress(), - HiveContainer.HDFS_DEFAULTFS_PORT, - "new_schema")); + hive.getContainerIpAddress(), HiveContainer.HDFS_DEFAULTFS_PORT, "new_schema")); e = Assertions.assertThrows( RuntimeException.class, @@ -283,7 +278,7 @@ private static void createSchema(String schemaName, String comment) { "location", String.format( "hdfs://%s:%d/user/hive/warehouse/%s.db", - containerSuite.getHiveContainer().getContainerIpAddress(), + hive.getContainerIpAddress(), HiveContainer.HDFS_DEFAULTFS_PORT, schemaName.toLowerCase())); catalog.asSchemas().createSchema(schemaName, comment, properties); diff --git a/catalogs/hive-metastore-common/src/test/java/org/apache/gravitino/hive/client/TestHive2HMS.java b/catalogs/hive-metastore-common/src/test/java/org/apache/gravitino/hive/client/TestHive2HMS.java index b803b940728..01f646cf8dd 100644 --- a/catalogs/hive-metastore-common/src/test/java/org/apache/gravitino/hive/client/TestHive2HMS.java +++ b/catalogs/hive-metastore-common/src/test/java/org/apache/gravitino/hive/client/TestHive2HMS.java @@ -19,7 +19,6 @@ package org.apache.gravitino.hive.client; -import com.google.common.collect.ImmutableMap; import java.time.Instant; import java.util.HashMap; import java.util.List; @@ -39,9 +38,9 @@ import org.apache.gravitino.hive.HivePartition; import org.apache.gravitino.hive.HiveSchema; import org.apache.gravitino.hive.HiveTable; -import org.apache.gravitino.integration.test.container.ContainerSuite; +import org.apache.gravitino.integration.test.container.FixtureContainer; import org.apache.gravitino.integration.test.container.HiveContainer; -import org.apache.gravitino.integration.test.util.CloseContainerExtension; +import org.apache.gravitino.integration.test.container.HiveFixture; import org.apache.gravitino.integration.test.util.PrintFuncNameExtension; import org.apache.gravitino.meta.AuditInfo; import org.apache.gravitino.rel.Column; @@ -62,12 +61,11 @@ // This class is used for testing against Hive 2.x Metastore instances using Docker containers. @Tag("gravitino-docker-test") @TestInstance(TestInstance.Lifecycle.PER_CLASS) -@ExtendWith({PrintFuncNameExtension.class, CloseContainerExtension.class}) +@ExtendWith(PrintFuncNameExtension.class) +@HiveFixture public class TestHive2HMS { - protected final ContainerSuite containerSuite = ContainerSuite.getInstance(); - - protected HiveContainer hiveContainer; + @FixtureContainer protected HiveContainer hive; protected String testPrefix = "hive2"; protected String metastoreUri; protected String hdfsBasePath; @@ -76,17 +74,13 @@ public class TestHive2HMS { @BeforeAll public void startHiveContainer() { - containerSuite.startHiveContainer( - ImmutableMap.of(HiveContainer.HIVE_RUNTIME_VERSION, HiveContainer.HIVE2)); - hiveContainer = containerSuite.getHiveContainer(); metastoreUri = String.format( - "thrift://%s:%d", - hiveContainer.getContainerIpAddress(), HiveContainer.HIVE_METASTORE_PORT); + "thrift://%s:%d", hive.getContainerIpAddress(), HiveContainer.HIVE_METASTORE_PORT); hdfsBasePath = String.format( "hdfs://%s:%d/tmp/gravitino_test", - hiveContainer.getContainerIpAddress(), HiveContainer.HDFS_DEFAULTFS_PORT); + hive.getContainerIpAddress(), HiveContainer.HDFS_DEFAULTFS_PORT); hiveClient = new HiveClientFactory(createHiveProperties(), testPrefix).createHiveClient(); } diff --git a/catalogs/hive-metastore-common/src/test/java/org/apache/gravitino/hive/client/TestHive2HMSWithKerberos.java b/catalogs/hive-metastore-common/src/test/java/org/apache/gravitino/hive/client/TestHive2HMSWithKerberos.java index 53ef460da09..4e83d4e3557 100644 --- a/catalogs/hive-metastore-common/src/test/java/org/apache/gravitino/hive/client/TestHive2HMSWithKerberos.java +++ b/catalogs/hive-metastore-common/src/test/java/org/apache/gravitino/hive/client/TestHive2HMSWithKerberos.java @@ -25,7 +25,9 @@ import java.nio.file.Files; import java.util.Properties; import org.apache.commons.io.FileUtils; +import org.apache.gravitino.integration.test.container.HiveAuth; import org.apache.gravitino.integration.test.container.HiveContainer; +import org.apache.gravitino.integration.test.container.HiveFixture; import org.apache.hadoop.security.authentication.util.KerberosName; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; @@ -35,6 +37,7 @@ /** Kerberos-enabled Hive2 HMS tests. */ @Tag("gravitino-docker-test") @TestInstance(TestInstance.Lifecycle.PER_CLASS) +@HiveFixture(auth = HiveAuth.KERBEROS) public class TestHive2HMSWithKerberos extends TestHive2HMS { private static final String CLIENT_PRINCIPAL = "cli@HADOOPKRB"; @@ -47,17 +50,14 @@ public class TestHive2HMSWithKerberos extends TestHive2HMS { public void startHiveContainer() { testPrefix = "hive2_kerberos"; catalogName = ""; - containerSuite.startKerberosHiveContainer(); - hiveContainer = containerSuite.getKerberosHiveContainer(); metastoreUri = String.format( - "thrift://%s:%d", - hiveContainer.getContainerIpAddress(), HiveContainer.HIVE_METASTORE_PORT); + "thrift://%s:%d", hive.getContainerIpAddress(), HiveContainer.HIVE_METASTORE_PORT); hdfsBasePath = String.format( "hdfs://%s:%d/tmp/gravitino_test", - hiveContainer.getContainerIpAddress(), HiveContainer.HDFS_DEFAULTFS_PORT); + hive.getContainerIpAddress(), HiveContainer.HDFS_DEFAULTFS_PORT); prepareKerberosConfig(); @@ -70,16 +70,16 @@ protected void prepareKerberosConfig() { tempDir = Files.createTempDirectory(testPrefix).toFile(); tempDir.deleteOnExit(); - String ip = hiveContainer.getContainerIpAddress(); + String ip = hive.getContainerIpAddress(); // Copy client keytab (admin keytab is used for metastore client) keytabPath = new File(tempDir, "admin.keytab").getAbsolutePath(); - hiveContainer.getContainer().copyFileFromContainer("/etc/admin.keytab", keytabPath); + hive.getContainer().copyFileFromContainer("/etc/admin.keytab", keytabPath); // Copy and patch krb5.conf String tmpKrb5Path = new File(tempDir, "krb5.conf.tmp").getAbsolutePath(); krb5ConfPath = new File(tempDir, "krb5.conf").getAbsolutePath(); - hiveContainer.getContainer().copyFileFromContainer("/etc/krb5.conf", tmpKrb5Path); + hive.getContainer().copyFileFromContainer("/etc/krb5.conf", tmpKrb5Path); String content = FileUtils.readFileToString(new File(tmpKrb5Path), StandardCharsets.UTF_8); content = content.replace("kdc = localhost:88", "kdc = " + ip + ":88"); content = content.replace("admin_server = localhost", "admin_server = " + ip + ":749"); @@ -117,7 +117,7 @@ protected Properties createHiveProperties() { properties.setProperty("hive.metastore.sasl.enabled", "true"); properties.setProperty( "hive.metastore.kerberos.principal", - String.format("hive/%s@HADOOPKRB", hiveContainer.getHostName())); + String.format("hive/%s@HADOOPKRB", hive.getHostName())); properties.setProperty("authentication.kerberos.principal", CLIENT_PRINCIPAL); properties.setProperty("authentication.kerberos.keytab-uri", keytabPath); properties.setProperty("authentication.impersonation-enable", "true"); diff --git a/catalogs/hive-metastore-common/src/test/java/org/apache/gravitino/hive/client/TestHive3HMS.java b/catalogs/hive-metastore-common/src/test/java/org/apache/gravitino/hive/client/TestHive3HMS.java index a8b79e39135..63c9767ccb1 100644 --- a/catalogs/hive-metastore-common/src/test/java/org/apache/gravitino/hive/client/TestHive3HMS.java +++ b/catalogs/hive-metastore-common/src/test/java/org/apache/gravitino/hive/client/TestHive3HMS.java @@ -19,8 +19,9 @@ package org.apache.gravitino.hive.client; -import com.google.common.collect.ImmutableMap; import org.apache.gravitino.integration.test.container.HiveContainer; +import org.apache.gravitino.integration.test.container.HiveFixture; +import org.apache.gravitino.integration.test.container.HiveVersion; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.TestInstance; @@ -30,6 +31,7 @@ */ @Tag("gravitino-docker-test") @TestInstance(TestInstance.Lifecycle.PER_CLASS) +@HiveFixture(version = HiveVersion.HIVE3) public class TestHive3HMS extends TestHive2HMS { @BeforeAll @@ -38,18 +40,13 @@ public void startHiveContainer() { testPrefix = "hive3"; catalogName = "hive"; // Hive3 default catalog - containerSuite.startHiveContainer( - ImmutableMap.of(HiveContainer.HIVE_RUNTIME_VERSION, HiveContainer.HIVE3)); - hiveContainer = containerSuite.getHiveContainer(); - metastoreUri = String.format( - "thrift://%s:%d", - hiveContainer.getContainerIpAddress(), HiveContainer.HIVE_METASTORE_PORT); + "thrift://%s:%d", hive.getContainerIpAddress(), HiveContainer.HIVE_METASTORE_PORT); hdfsBasePath = String.format( "hdfs://%s:%d/tmp/gravitino_test", - hiveContainer.getContainerIpAddress(), HiveContainer.HDFS_DEFAULTFS_PORT); + hive.getContainerIpAddress(), HiveContainer.HDFS_DEFAULTFS_PORT); hiveClient = new HiveClientFactory(createHiveProperties(), testPrefix).createHiveClient(); } diff --git a/catalogs/hive-metastore-common/src/test/java/org/apache/gravitino/hive/client/TestHive3HMSWithCatalog.java b/catalogs/hive-metastore-common/src/test/java/org/apache/gravitino/hive/client/TestHive3HMSWithCatalog.java index a3b38347fe8..fd0a9834f05 100644 --- a/catalogs/hive-metastore-common/src/test/java/org/apache/gravitino/hive/client/TestHive3HMSWithCatalog.java +++ b/catalogs/hive-metastore-common/src/test/java/org/apache/gravitino/hive/client/TestHive3HMSWithCatalog.java @@ -40,7 +40,7 @@ public void startHiveContainer() { String catalogLocation = String.format( "hdfs://%s:%d/tmp/gravitino_test/catalogs/%s", - hiveContainer.getContainerIpAddress(), HiveContainer.HDFS_DEFAULTFS_PORT, catalogName); + hive.getContainerIpAddress(), HiveContainer.HDFS_DEFAULTFS_PORT, catalogName); // Ensure the catalog exists; only create if missing. if (!hiveClient.getCatalogs().contains(catalogName)) { diff --git a/catalogs/hive-metastore-common/src/test/java/org/apache/gravitino/hive/client/TestHive3HMSWithKerberos.java b/catalogs/hive-metastore-common/src/test/java/org/apache/gravitino/hive/client/TestHive3HMSWithKerberos.java index 9bb1eb048de..7b3660b139d 100644 --- a/catalogs/hive-metastore-common/src/test/java/org/apache/gravitino/hive/client/TestHive3HMSWithKerberos.java +++ b/catalogs/hive-metastore-common/src/test/java/org/apache/gravitino/hive/client/TestHive3HMSWithKerberos.java @@ -19,8 +19,10 @@ package org.apache.gravitino.hive.client; -import com.google.common.collect.ImmutableMap; +import org.apache.gravitino.integration.test.container.HiveAuth; import org.apache.gravitino.integration.test.container.HiveContainer; +import org.apache.gravitino.integration.test.container.HiveFixture; +import org.apache.gravitino.integration.test.container.HiveVersion; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.TestInstance; @@ -31,6 +33,7 @@ */ @Tag("gravitino-docker-test") @TestInstance(TestInstance.Lifecycle.PER_CLASS) +@HiveFixture(version = HiveVersion.HIVE3, auth = HiveAuth.KERBEROS) public class TestHive3HMSWithKerberos extends TestHive2HMSWithKerberos { @BeforeAll @@ -39,18 +42,13 @@ public void startHiveContainer() { testPrefix = "hive3_kerberos"; catalogName = "hive"; // Hive3 default catalog - containerSuite.startKerberosHiveContainer( - ImmutableMap.of(HiveContainer.HIVE_RUNTIME_VERSION, HiveContainer.HIVE3)); - hiveContainer = containerSuite.getKerberosHiveContainer(); - metastoreUri = String.format( - "thrift://%s:%d", - hiveContainer.getContainerIpAddress(), HiveContainer.HIVE_METASTORE_PORT); + "thrift://%s:%d", hive.getContainerIpAddress(), HiveContainer.HIVE_METASTORE_PORT); hdfsBasePath = String.format( "hdfs://%s:%d/tmp/gravitino_test", - hiveContainer.getContainerIpAddress(), HiveContainer.HDFS_DEFAULTFS_PORT); + hive.getContainerIpAddress(), HiveContainer.HDFS_DEFAULTFS_PORT); // Prepare Kerberos config (keytab/krb5) for client connections. prepareKerberosConfig(); diff --git a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/BaseContainer.java b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/BaseContainer.java index 11c7ec50e20..ff0cb8666b0 100644 --- a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/BaseContainer.java +++ b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/BaseContainer.java @@ -35,21 +35,24 @@ import java.util.Optional; import java.util.Set; import java.util.function.Consumer; +import javax.annotation.Nullable; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.testcontainers.DockerClientFactory; import org.testcontainers.containers.Container; import org.testcontainers.containers.GenericContainer; -import org.testcontainers.containers.JdbcDatabaseContainer; import org.testcontainers.containers.Network; import org.testcontainers.containers.output.OutputFrame; import org.testcontainers.containers.startupcheck.IsRunningStartupCheckStrategy; import org.testcontainers.containers.wait.strategy.Wait; +import org.testcontainers.containers.wait.strategy.WaitStrategy; import org.testcontainers.lifecycle.Startable; /** - * The BaseContainer is the base for all containers. It's contains the common methods and settings - * for all containers. You can extend this class to create your own container to integration test. + * The base for all integration test containers: it applies the common settings and manages the + * lifecycle. Gravitino's own images have no official Testcontainers module, so they start from an + * image name; services that have an official module supply that module instead, so they keep its + * defaults and readiness check. */ public abstract class BaseContainer implements Startable { public static final Logger LOG = LoggerFactory.getLogger(BaseContainer.class); @@ -68,6 +71,18 @@ public abstract class BaseContainer implements Startable { protected final GenericContainer container; + /** + * Builds the container from an image name. Used by Gravitino's own images, which have no official + * Testcontainers module. + * + * @param image the Docker image name + * @param hostName the host name and network alias of the container + * @param ports the ports to expose + * @param extraHosts additional host name to IP address mappings + * @param filesToMount host files to copy into the container, keyed by container path + * @param envVars the environment variables of the container + * @param network the network to join, if any + */ protected BaseContainer( String image, String hostName, @@ -86,7 +101,19 @@ protected BaseContainer( network); } - /** Wraps a container a subclass has already created, such as a Testcontainers module. */ + /** + * Wraps a container the subclass has already built, such as an official Testcontainers module or + * a {@link GenericContainer} with its own wait strategy. A subclass that relies on the readiness + * check of the supplied container should also override {@link #waitStrategy()}. + * + * @param container the container to configure and manage + * @param hostName the host name and network alias of the container + * @param ports the ports to expose + * @param extraHosts additional host name to IP address mappings + * @param filesToMount host files to copy into the container, keyed by container path + * @param envVars the environment variables of the container + * @param network the network to join, if any + */ protected BaseContainer( GenericContainer container, String hostName, @@ -113,6 +140,10 @@ protected BaseContainer( setupContainer(); } + /** + * Applies the common settings and then the readiness check. Runs from the constructor, so an + * override must call {@code super.setupContainer()} and must not read subclass fields. + */ protected void setupContainer() { // Add exposed ports in the container for (int port : this.ports) { @@ -128,16 +159,29 @@ protected void setupContainer() { // can look up the corresponding IP address by host name. // This method fixes an error that occurs when HDFS looks up hostnames from DNS. extraHosts.forEach((hostName, ipAddress) -> container.withExtraHost(hostName, ipAddress)); - container - .withCreateContainerCmdModifier(c -> c.withHostName(hostName)) - .withStartupCheckStrategy(new IsRunningStartupCheckStrategy()); - // A Testcontainers JDBC module has its own readiness check: PostgreSQL's waits for a log line - // through the wait strategy, which a port wait would replace; MySQL's queries the database. - if (!(container instanceof JdbcDatabaseContainer)) { - container.waitingFor(Wait.forListeningPort()); - } - container.withStartupTimeout(Duration.ofMinutes(5)); + container.withCreateContainerCmdModifier(c -> c.withHostName(hostName)); network.ifPresent(net -> container.withNetwork(net).withNetworkAliases(hostName)); + + WaitStrategy waitStrategy = waitStrategy(); + if (waitStrategy != null) { + container + .withStartupCheckStrategy(new IsRunningStartupCheckStrategy()) + .waitingFor(waitStrategy) + .withStartupTimeout(Duration.ofMinutes(5)); + } + } + + /** + * Returns how to tell that the container is ready. The default waits for the exposed ports to + * listen. Override it to return a service-specific strategy, or return {@code null} to keep the + * startup check, wait strategy and timeout the supplied container already has, as an official + * module does. Called from the constructor, so an override must not read subclass fields. + * + * @return the wait strategy, or {@code null} to keep the container's own + */ + @Nullable + protected WaitStrategy waitStrategy() { + return Wait.forListeningPort(); } // This method is used to set the log output of the container. diff --git a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/ClickHouseContainer.java b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/ClickHouseContainer.java index 72f2fd355cb..7eac3b22874 100644 --- a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/ClickHouseContainer.java +++ b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/ClickHouseContainer.java @@ -41,7 +41,7 @@ public class ClickHouseContainer extends BaseContainer { public static final Logger LOG = LoggerFactory.getLogger(ClickHouseContainer.class); - public static final String DEFAULT_IMAGE = "clickhouse:24.8.14"; + public static final String DEFAULT_IMAGE = TestImages.CLICKHOUSE; public static final String HOST_NAME = "gravitino-ci-clickhouse"; public static final int CLICKHOUSE_PORT = 8123; public static final int CLICKHOUSE_NATIVE_PORT = 9000; diff --git a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/ContainerExtension.java b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/ContainerExtension.java index 9f5582463b9..a4b02028b12 100644 --- a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/ContainerExtension.java +++ b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/ContainerExtension.java @@ -47,6 +47,10 @@ *

This is dependency injection built on JUnit's own extension points ({@link ParameterResolver} * and {@link TestInstancePostProcessor}), so tests need no DI framework. * + *

Fixtures on superclasses apply too, but a class's fixture replaces a superclass's fixture of + * the same annotation type, so {@code @HiveFixture(version = HIVE3)} on a subclass of a Hive2 test + * runs Hive3 only. + * *

Each container starts once per run and is shared by every class that asks for the same spec. * The containers close when the run ends, through {@link CloseContainerExtension}. */ @@ -127,8 +131,8 @@ private static List> matching(Class wanted, Class testCla } // Fixtures on the class and its superclasses, nearest class first, one per annotation type. - // A subclass that re-declares a fixture, for example with another version, replaces the - // superclass's; keeping both would start two containers and make injection by type ambiguous. + // Hierarchies such as TestHive3HMS extends TestHive2HMS re-declare a fixture with another + // version; keeping both would start two containers and make injection by type ambiguous. static List> specs(Class testClass) { List> specs = new ArrayList<>(); Set> seen = new HashSet<>(); diff --git a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/ContainerSlot.java b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/ContainerSlot.java index b70c415a396..6a2e439df7a 100644 --- a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/ContainerSlot.java +++ b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/ContainerSlot.java @@ -30,6 +30,10 @@ * heavy service, such as Hive versions, that tests never need together and that would not fit side * by side on a CI runner. * + *

A start that throws leaves the slot empty, so the next request starts again instead of being + * handed a container that never became ready. The slot cannot close that container: the starter + * creates it and never returns it, so the starter closes it, as {@code ContainerSuite} does. + * * @param what makes two requests different, such as a Hive version * @param the container type */ @@ -47,6 +51,8 @@ synchronized C getOrStart(K requestedKey, Supplier starter) { container.close(); clear(); } + // Assigned only after the starter returns: a failed start must not be remembered as the + // running container. C started = starter.get(); container = started; key = requestedKey; diff --git a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/ContainerSuite.java b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/ContainerSuite.java index 1b152878ff2..3a50781363e 100644 --- a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/ContainerSuite.java +++ b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/ContainerSuite.java @@ -102,6 +102,8 @@ public class ContainerSuite implements Closeable { private static volatile RustFSContainer rustFSContainer; + private static volatile GravitinoMotoContainer motoContainer; + /** * We can share the same Hive container as Hive container with S3 contains the following * differences: 1. Configuration of S3 and corresponding environment variables 2. The Hive @@ -379,7 +381,7 @@ public void startMySQLVersion5Container(TestDatabaseName testDatabaseName) { // Start MySQL container MySQLContainer.Builder mysqlBuilder = MySQLContainer.builder() - .withImage("mysql:5.7") + .withImage(TestImages.MYSQL_5_7) .withHostName("gravitino-ci-mysql-v5") .withDurability(Durability.fromSystemProperty()) .withExposePorts(ImmutableSet.of(MySQLContainer.MYSQL_PORT)) @@ -504,6 +506,27 @@ public void startLocalStackContainer() { } } + /** Starts the shared Moto server, the AWS mock behind the Glue catalog tests. */ + public void startMotoContainer() { + if (motoContainer == null) { + synchronized (ContainerSuite.class) { + if (motoContainer == null) { + initIfNecessary(); + GravitinoMotoContainer.Builder builder = + GravitinoMotoContainer.builder().withNetwork(network); + GravitinoMotoContainer container = closer.register(builder.build()); + try { + start(container); + } catch (Exception e) { + LOG.error("Failed to start Moto container", e); + throw new RuntimeException("Failed to start Moto container", e); + } + motoContainer = container; + } + } + } + } + public void startStarRocksContainer() { if (starRocksContainer == null) { synchronized (ContainerSuite.class) { @@ -533,7 +556,7 @@ public void startClickHouseContainer(TestDatabaseName testDatabaseName) { // Start ClickHouse container ClickHouseContainer.Builder clickHouseBuilder = ClickHouseContainer.builder() - .withImage(resolveClickHouseImage()) + .withImage(TestImages.CLICKHOUSE) .withFilesToMount(resolveClickHouseTestFiles()) .withHostName("gravitino-ci-clickhouse") .withEnvVars( @@ -567,7 +590,7 @@ public void startClickHouseClusterContainer( for (int node = 1; node <= CLICKHOUSE_CLUSTER_SIZE; node++) { ClickHouseContainer.Builder clickHouseBuilder = ClickHouseContainer.builder() - .withImage(resolveClickHouseImage()) + .withImage(TestImages.CLICKHOUSE) .withFilesToMount(resolveClickHouseTestFiles()) .withHostName(CLICKHOUSE_CLUSTER_HOST_PREFIX + node) .withEnvVars( @@ -667,6 +690,11 @@ public GravitinoLocalStackContainer getLocalStackContainer() { return gravitinoLocalStackContainer; } + /** Returns the shared Moto server, or null before {@link #startMotoContainer()} runs. */ + public GravitinoMotoContainer getMotoContainer() { + return motoContainer; + } + /** Starts the shared RustFS object store for S3 credential-vending tests. */ public void startRustFSContainer() { if (rustFSContainer == null) { @@ -801,12 +829,19 @@ private static void start(BaseContainer container) { // Logs one greppable line per start, including the container's own readiness check, so CI logs // show where container time goes. Failed starts are logged too, with ok=false. + // + // A failed start is closed here because Testcontainers leaves the container running when its + // readiness check times out, and the caller never stores it, so the next start* call would start + // another one beside it. The closer still holds it; closing it again at run end is a no-op. static void startTimed(BaseContainer container) { long startNanos = System.nanoTime(); boolean ok = false; try { container.start(); ok = true; + } catch (Throwable e) { + closeFailedStart(container, e); + throw e; } finally { LOG.info( "[CONTAINER-TIMING] container={} image={} hostName={} ok={} startMs={}", @@ -818,6 +853,23 @@ static void startTimed(BaseContainer container) { } } + private static void closeFailedStart(BaseContainer container, Throwable startFailure) { + LOG.warn( + "Closing {} (image={}, hostName={}) after its start failed", + container.getClass().getSimpleName(), + container.getContainer().getDockerImageName(), + container.hostName); + try { + container.close(); + } catch (Throwable closeFailure) { + // The start failure is the one the test must report; the close failure only rides along. + LOG.warn("Failed to close {}", container.getClass().getSimpleName(), closeFailure); + if (closeFailure != startFailure) { + startFailure.addSuppressed(closeFailure); + } + } + } + // Hive2 and Hive3 run from one image; HIVE_RUNTIME_VERSION picks the version, Hive2 by default. static String hiveVersion(Map env) { return env.getOrDefault(HiveContainer.HIVE_RUNTIME_VERSION, HiveContainer.HIVE2); @@ -981,16 +1033,10 @@ private static void resetState() { zooKeeperContainer = null; gravitinoLocalStackContainer = null; rustFSContainer = null; + motoContainer = null; hiveContainerWithS3 = null; } - private static String resolveClickHouseImage() { - String configuredImage = System.getenv(CLICKHOUSE_TEST_IMAGE_ENV); - return configuredImage == null || configuredImage.isBlank() - ? ClickHouseContainer.DEFAULT_IMAGE - : configuredImage.trim(); - } - private static Map resolveClickHouseTestFiles() { // The feature-specific ClickHouse test image is 25.8+, where Time types require this setting. String configuredImage = System.getenv(CLICKHOUSE_TEST_IMAGE_ENV); diff --git a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/DatabaseContainer.java b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/DatabaseContainer.java index bece1defb1e..dfd4b5a5ec4 100644 --- a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/DatabaseContainer.java +++ b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/DatabaseContainer.java @@ -28,11 +28,13 @@ import java.util.Set; import java.util.stream.Collectors; import java.util.stream.IntStream; +import javax.annotation.Nullable; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.testcontainers.containers.Container; import org.testcontainers.containers.JdbcDatabaseContainer; import org.testcontainers.containers.Network; +import org.testcontainers.containers.wait.strategy.WaitStrategy; /** * A database test container backed by a Testcontainers JDBC module, which owns the credentials and @@ -102,6 +104,14 @@ protected void applyDurability(String dataDir, String... command) { } /** The user tests connect as. */ + // A Testcontainers JDBC module has its own readiness check: PostgreSQL's waits for a log line, + // which a port wait would replace; MySQL's queries the database. + @Override + @Nullable + protected WaitStrategy waitStrategy() { + return null; + } + public String getUsername() { return db.getUsername(); } diff --git a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/DorisContainer.java b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/DorisContainer.java index 4603f94b450..6f3565f9a75 100644 --- a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/DorisContainer.java +++ b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/DorisContainer.java @@ -53,7 +53,7 @@ public class DorisContainer extends BaseContainer { public static final Logger LOG = LoggerFactory.getLogger(DorisContainer.class); - public static final String DEFAULT_IMAGE = System.getenv("GRAVITINO_CI_DORIS_DOCKER_IMAGE"); + public static final String DEFAULT_IMAGE = TestImages.DORIS; public static final String HOST_NAME = "gravitino-ci-doris"; public static final String USER_NAME = "root"; public static final String PASSWORD = "root"; @@ -275,7 +275,7 @@ private boolean changePassword() { public static class Builder extends BaseContainer.Builder { private Builder() { - this.image = (DEFAULT_IMAGE != null && !DEFAULT_IMAGE.isEmpty()) ? DEFAULT_IMAGE : null; + this.image = DEFAULT_IMAGE; this.hostName = HOST_NAME; this.exposePorts = ImmutableSet.of(FE_HTTP_PORT, FE_MYSQL_PORT); } diff --git a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/DorisImageName.java b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/DorisImageName.java index 0135345af44..89634681979 100644 --- a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/DorisImageName.java +++ b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/DorisImageName.java @@ -32,11 +32,16 @@ * @see ContainerSuite#startDorisContainer(DorisImageName) */ public enum DorisImageName { - VERSION_1_2("apache/gravitino-ci:doris-0.1.5"), + /** + * The Gravitino CI all-in-one Doris 1.2 image. It follows {@code + * GRAVITINO_CI_DORIS_DOCKER_IMAGE}, so an override of that variable also changes what this + * constant starts. + */ + VERSION_1_2(TestImages.DORIS), /** The official all-in-one Doris 2.1.0 image. */ - VERSION_2_1("apache/doris:doris-all-in-one-2.1.0"), - VERSION_3_0("apache/doris:fe-3.0.6.2"), - VERSION_4_0("apache/doris:fe-4.0.6"); + VERSION_2_1(TestImages.DORIS_2_1), + VERSION_3_0(TestImages.DORIS_3_0), + VERSION_4_0(TestImages.DORIS_4_0); private final String imageName; diff --git a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/GravitinoLocalStackContainer.java b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/GravitinoLocalStackContainer.java index e6749380c43..df1c875a03e 100644 --- a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/GravitinoLocalStackContainer.java +++ b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/GravitinoLocalStackContainer.java @@ -29,8 +29,10 @@ import java.util.Set; import java.util.regex.Pattern; import java.util.stream.Collectors; +import javax.annotation.Nullable; import org.testcontainers.containers.Network; import org.testcontainers.containers.wait.strategy.Wait; +import org.testcontainers.containers.wait.strategy.WaitStrategy; import software.amazon.awssdk.auth.credentials.AwsBasicCredentials; import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider; import software.amazon.awssdk.regions.Region; @@ -47,7 +49,7 @@ */ public class GravitinoLocalStackContainer extends BaseContainer { - public static final String DEFAULT_IMAGE = System.getenv("GRAVITINO_CI_LOCALSTACK_DOCKER_IMAGE"); + public static final String DEFAULT_IMAGE = TestImages.LOCALSTACK; public static final String HOST_NAME = "gravitino-ci-localstack"; public static final int PORT = 4566; @@ -79,13 +81,12 @@ public static Builder builder() { } @Override - protected void setupContainer() { - super.setupContainer(); - container.waitingFor( - Wait.forHttp("/_localstack/health") - .forPort(PORT) - .forStatusCode(200) - .forResponsePredicate(GravitinoLocalStackContainer::reportsS3Ready)); + @Nullable + protected WaitStrategy waitStrategy() { + return Wait.forHttp("/_localstack/health") + .forPort(PORT) + .forStatusCode(200) + .forResponsePredicate(GravitinoLocalStackContainer::reportsS3Ready); } @Override diff --git a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/GravitinoMotoContainer.java b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/GravitinoMotoContainer.java index efaf29a524c..4760bb19782 100644 --- a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/GravitinoMotoContainer.java +++ b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/GravitinoMotoContainer.java @@ -27,7 +27,7 @@ /** Container wrapping {@code motoserver/moto} — a free AWS mock that supports Glue and more. */ public class GravitinoMotoContainer extends BaseContainer { - public static final String DEFAULT_IMAGE = System.getenv("GRAVITINO_CI_MOTO_DOCKER_IMAGE"); + public static final String DEFAULT_IMAGE = TestImages.MOTO; public static final String HOST_NAME = "gravitino-ci-moto"; public static final int PORT = 5000; @@ -62,11 +62,6 @@ public Builder() { @Override public GravitinoMotoContainer build() { - if (image == null || image.isBlank()) { - throw new IllegalStateException( - "GravitinoMotoContainer image must be set. " - + "Provide a non-blank image via withImage() or set the GRAVITINO_CI_MOTO_DOCKER_IMAGE environment variable."); - } return new GravitinoMotoContainer( image, hostName, exposePorts, extraHosts, filesToMount, envVars, network); } diff --git a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/HiveAuth.java b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/HiveAuth.java new file mode 100644 index 00000000000..ba147d18a4a --- /dev/null +++ b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/HiveAuth.java @@ -0,0 +1,32 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.integration.test.container; + +/** + * How clients authenticate to the Hive a {@link HiveFixture} runs. + * + *

Ranger is not here yet: it needs a Ranger container started first, which arrives with a Ranger + * dependency spec. + */ +public enum HiveAuth { + /** No authentication. */ + NONE, + /** Kerberos, from the Kerberos Hive image with its own KDC. */ + KERBEROS +} diff --git a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/HiveContainer.java b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/HiveContainer.java index 2fb04b4768f..93f054bb189 100644 --- a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/HiveContainer.java +++ b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/HiveContainer.java @@ -44,9 +44,8 @@ public class HiveContainer extends BaseContainer { public static final Logger LOG = LoggerFactory.getLogger(HiveContainer.class); - public static final String DEFAULT_IMAGE = System.getenv("GRAVITINO_CI_HIVE_DOCKER_IMAGE"); - public static final String KERBEROS_IMAGE = - System.getenv("GRAVITINO_CI_KERBEROS_HIVE_DOCKER_IMAGE"); + public static final String DEFAULT_IMAGE = TestImages.HIVE; + public static final String KERBEROS_IMAGE = TestImages.KERBEROS_HIVE; public static final String HOST_NAME = "gravitino-ci-hive"; public static final String HADOOP_USER_NAME = "HADOOP_USER_NAME"; diff --git a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/HiveFixture.java b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/HiveFixture.java new file mode 100644 index 00000000000..5174356ee2c --- /dev/null +++ b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/HiveFixture.java @@ -0,0 +1,56 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.integration.test.container; + +import java.lang.annotation.Documented; +import java.lang.annotation.ElementType; +import java.lang.annotation.Inherited; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; +import org.junit.jupiter.api.extension.ExtendWith; + +/** + * Declares that a test class needs a Hive server. Test and lifecycle methods receive it as a {@link + * HiveContainer} parameter, and fields marked {@link FixtureContainer} receive it too. + * + *

A subclass may declare its own {@code HiveFixture}; it replaces the superclass's. + */ +@Documented +@Inherited +@Target(ElementType.TYPE) +@Retention(RetentionPolicy.RUNTIME) +@ExtendWith(ContainerExtension.class) +@ContainerFixture(HiveSpec.Factory.class) +public @interface HiveFixture { + + /** + * Returns the Hive version to run. + * + * @return the Hive version; Hive2 by default + */ + HiveVersion version() default HiveVersion.HIVE2; + + /** + * Returns how clients authenticate to Hive. + * + * @return the authentication mode; none by default + */ + HiveAuth auth() default HiveAuth.NONE; +} diff --git a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/HiveSpec.java b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/HiveSpec.java new file mode 100644 index 00000000000..04a106f821b --- /dev/null +++ b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/HiveSpec.java @@ -0,0 +1,59 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.integration.test.container; + +import java.util.Map; + +/** + * A Hive server of one version and authentication mode. + * + *

Plain and Kerberos Hive run side by side, but each holds one version at a time: asking for a + * different version stops the running Hive of that mode and starts the requested one (see {@link + * ContainerSlot}). Tests of one version therefore share a container, and a version switch costs a + * restart, as with direct {@link ContainerSuite} calls. + */ +public record HiveSpec(HiveVersion version, HiveAuth auth) implements ContainerSpec { + + @Override + public Class containerType() { + return HiveContainer.class; + } + + @Override + public HiveContainer getOrStart(ContainerSuite suite) { + Map env = Map.of(HiveContainer.HIVE_RUNTIME_VERSION, version.runtimeVersion()); + return switch (auth) { + case NONE -> { + suite.startHiveContainer(env); + yield suite.getHiveContainer(); + } + case KERBEROS -> { + suite.startKerberosHiveContainer(env); + yield suite.getKerberosHiveContainer(); + } + }; + } + + static final class Factory implements ContainerFixture.SpecFactory { + @Override + public HiveSpec specFor(HiveFixture fixture) { + return new HiveSpec(fixture.version(), fixture.auth()); + } + } +} diff --git a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/HiveVersion.java b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/HiveVersion.java new file mode 100644 index 00000000000..5d0a4e4bfa9 --- /dev/null +++ b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/HiveVersion.java @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.integration.test.container; + +/** The Hive version a {@link HiveFixture} runs. Both versions run from the same image. */ +public enum HiveVersion { + /** Hive 2, the image default. */ + HIVE2(HiveContainer.HIVE2), + /** Hive 3. */ + HIVE3(HiveContainer.HIVE3); + + private final String runtimeVersion; + + HiveVersion(String runtimeVersion) { + this.runtimeVersion = runtimeVersion; + } + + /** + * Returns the value of the image's {@link HiveContainer#HIVE_RUNTIME_VERSION} variable. + * + * @return the runtime version, such as {@code hive2} + */ + public String runtimeVersion() { + return runtimeVersion; + } +} diff --git a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/KafkaContainer.java b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/KafkaContainer.java index d926b95d613..a4d9f0c0b22 100644 --- a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/KafkaContainer.java +++ b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/KafkaContainer.java @@ -39,8 +39,7 @@ public class KafkaContainer extends BaseContainer { public static final int DEFAULT_BROKER_PORT = 9092; - private static final String DEFAULT_KAFKA_IMAGE = - System.getenv("GRAVITINO_CI_KAFKA_DOCKER_IMAGE"); + private static final String DEFAULT_KAFKA_IMAGE = TestImages.KAFKA; private static final String DEFAULT_HOST_NAME = "gravitino-ci-kafka"; private static final String KAFKA_LOGS_DIR = "/opt/kafka/logs"; diff --git a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/MotoFixture.java b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/MotoFixture.java new file mode 100644 index 00000000000..28a96a4efa0 --- /dev/null +++ b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/MotoFixture.java @@ -0,0 +1,47 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.integration.test.container; + +import java.lang.annotation.Documented; +import java.lang.annotation.ElementType; +import java.lang.annotation.Inherited; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; +import org.junit.jupiter.api.extension.ExtendWith; + +/** + * Declares that a test class needs the Moto AWS mock. Test and lifecycle methods receive it as a + * {@link GravitinoMotoContainer} parameter, and fields marked {@link FixtureContainer} receive it + * before any {@code @BeforeAll} method runs. + * + * 

{@code
+ * @MotoFixture
+ * class MotoGlueCatalogIT {
+ *   @FixtureContainer private GravitinoMotoContainer moto;
+ * }
+ * }
+ */ +@Documented +@Inherited +@Target(ElementType.TYPE) +@Retention(RetentionPolicy.RUNTIME) +@ExtendWith(ContainerExtension.class) +@ContainerFixture(MotoSpec.Factory.class) +public @interface MotoFixture {} diff --git a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/MotoSpec.java b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/MotoSpec.java new file mode 100644 index 00000000000..7cf7c3df29e --- /dev/null +++ b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/MotoSpec.java @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.integration.test.container; + +/** The Moto AWS mock. There is one configuration, so the spec has no fields. */ +public record MotoSpec() implements ContainerSpec { + + @Override + public Class containerType() { + return GravitinoMotoContainer.class; + } + + @Override + public GravitinoMotoContainer getOrStart(ContainerSuite suite) { + suite.startMotoContainer(); + return suite.getMotoContainer(); + } + + static final class Factory implements ContainerFixture.SpecFactory { + @Override + public MotoSpec specFor(MotoFixture fixture) { + return new MotoSpec(); + } + } +} diff --git a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/MySQLContainer.java b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/MySQLContainer.java index b2069ede3d2..4ff44476b3f 100644 --- a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/MySQLContainer.java +++ b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/MySQLContainer.java @@ -38,7 +38,7 @@ public class MySQLContainer extends DatabaseContainer { public static final Logger LOG = LoggerFactory.getLogger(MySQLContainer.class); - public static final String DEFAULT_IMAGE = "mysql:8.0"; + public static final String DEFAULT_IMAGE = TestImages.MYSQL; public static final String HOST_NAME = "gravitino-ci-mysql"; public static final int MYSQL_PORT = 3306; public static final String USER_NAME = "root"; diff --git a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/OceanBaseContainer.java b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/OceanBaseContainer.java index 955c6c16f52..8915769a42f 100644 --- a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/OceanBaseContainer.java +++ b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/OceanBaseContainer.java @@ -42,7 +42,7 @@ public class OceanBaseContainer extends BaseContainer { public static final Logger LOG = LoggerFactory.getLogger(OceanBaseContainer.class); - public static final String DEFAULT_IMAGE = "oceanbase/oceanbase-ce:4.2.1-lts"; + public static final String DEFAULT_IMAGE = TestImages.OCEANBASE; public static final String HOST_NAME = "gravitino-ci-oceanbase"; public static final int OCEANBASE_PORT = 2881; public static final String USER_NAME = "root@test"; diff --git a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/PGImageName.java b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/PGImageName.java index 1a67d15a266..b194ae9764e 100644 --- a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/PGImageName.java +++ b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/PGImageName.java @@ -19,11 +19,11 @@ package org.apache.gravitino.integration.test.container; public enum PGImageName { - VERSION_12("postgres:12"), - VERSION_13("postgres:13"), - VERSION_14("postgres:14"), - VERSION_15("postgres:15"), - VERSION_16("postgres:16"); + VERSION_12(TestImages.POSTGRESQL_12), + VERSION_13(TestImages.POSTGRESQL_13), + VERSION_14(TestImages.POSTGRESQL_14), + VERSION_15(TestImages.POSTGRESQL_15), + VERSION_16(TestImages.POSTGRESQL_16); private final String imageName; diff --git a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/RangerContainer.java b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/RangerContainer.java index 54b2afc0c77..fe059f5129c 100644 --- a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/RangerContainer.java +++ b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/RangerContainer.java @@ -35,7 +35,7 @@ public class RangerContainer extends BaseContainer { public static final Logger LOG = LoggerFactory.getLogger(RangerContainer.class); - public static final String DEFAULT_IMAGE = System.getenv("GRAVITINO_CI_RANGER_DOCKER_IMAGE"); + public static final String DEFAULT_IMAGE = TestImages.RANGER; public static final String HOST_NAME = "gravitino-ci-ranger"; public static final int RANGER_SERVER_PORT = 6080; public RangerClient rangerClient; diff --git a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/RangerFixture.java b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/RangerFixture.java new file mode 100644 index 00000000000..895baa944e2 --- /dev/null +++ b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/RangerFixture.java @@ -0,0 +1,47 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.integration.test.container; + +import java.lang.annotation.Documented; +import java.lang.annotation.ElementType; +import java.lang.annotation.Inherited; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; +import org.junit.jupiter.api.extension.ExtendWith; + +/** + * Declares that a test class needs the Ranger admin server. Test and lifecycle methods receive it + * as a {@link RangerContainer} parameter. + * + * 
{@code
+ * @RangerFixture
+ * class RangerExampleIT {
+ *   @BeforeAll
+ *   static void setup(RangerContainer ranger) { ... }
+ * }
+ * }
+ */ +@Documented +@Inherited +@Target(ElementType.TYPE) +@Retention(RetentionPolicy.RUNTIME) +@ExtendWith(ContainerExtension.class) +@ContainerFixture(RangerSpec.Factory.class) +public @interface RangerFixture {} diff --git a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/RangerSpec.java b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/RangerSpec.java new file mode 100644 index 00000000000..419eafb454c --- /dev/null +++ b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/RangerSpec.java @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.integration.test.container; + +/** The Ranger admin server. There is one configuration, so the spec has no fields. */ +public record RangerSpec() implements ContainerSpec { + + @Override + public Class containerType() { + return RangerContainer.class; + } + + @Override + public RangerContainer getOrStart(ContainerSuite suite) { + suite.startRangerContainer(); + return suite.getRangerContainer(); + } + + static final class Factory implements ContainerFixture.SpecFactory { + @Override + public RangerSpec specFor(RangerFixture fixture) { + return new RangerSpec(); + } + } +} diff --git a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/RustFSContainer.java b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/RustFSContainer.java index aa25009a9ce..901903f8fd5 100644 --- a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/RustFSContainer.java +++ b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/RustFSContainer.java @@ -23,17 +23,12 @@ import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; -import java.io.IOException; -import java.io.InputStream; import java.net.URI; -import java.nio.charset.StandardCharsets; import java.time.Duration; import java.util.Map; import java.util.Optional; import java.util.Set; import java.util.concurrent.TimeUnit; -import java.util.regex.Matcher; -import java.util.regex.Pattern; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.testcontainers.containers.Network; @@ -51,7 +46,7 @@ public class RustFSContainer extends BaseContainer { private static final Logger LOG = LoggerFactory.getLogger(RustFSContainer.class); /** The image pin consumed from the Docker Compose manifest tracked by image updates. */ - public static final String DEFAULT_IMAGE = loadImage(); + public static final String DEFAULT_IMAGE = TestImages.RUSTFS; /** The container hostname on the shared integration-test network. */ public static final String HOST_NAME = "gravitino-ci-rustfs"; @@ -149,33 +144,6 @@ public String getS3Endpoint() { return format("http://%s:%d", container.getHost(), getMappedPort(PORT)); } - static String imageFromCompose(String compose) { - // This single-service manifest is deliberately limited to one unquoted image reference. - // Fail closed if its structure changes, rather than silently using a different image pin. - Matcher matcher = Pattern.compile("(?m)^\\s+image: (\\S+)\\s*$").matcher(compose); - if (!matcher.find()) { - throw new IllegalArgumentException("RustFS Compose manifest must contain an image"); - } - String image = matcher.group(1); - if (matcher.find() || !image.matches("rustfs/rustfs:[^@\\s]+@sha256:[0-9a-f]{64}")) { - throw new IllegalArgumentException( - "RustFS Compose manifest must contain one version and digest pin"); - } - return image; - } - - private static String loadImage() { - try (InputStream stream = - RustFSContainer.class.getResourceAsStream("/docker-compose-rustfs.yml")) { - if (stream == null) { - throw new IllegalStateException("Missing docker-compose-rustfs.yml test resource"); - } - return imageFromCompose(new String(stream.readAllBytes(), StandardCharsets.UTF_8)); - } catch (IOException e) { - throw new IllegalStateException("Unable to read RustFS image pin", e); - } - } - private S3Client createS3Client() { return S3Client.builder() .endpointOverride(URI.create(getS3Endpoint())) diff --git a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/StarRocksContainer.java b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/StarRocksContainer.java index 5fb546b1eed..02803cc120e 100644 --- a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/StarRocksContainer.java +++ b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/StarRocksContainer.java @@ -39,7 +39,7 @@ public class StarRocksContainer extends BaseContainer { public static final Logger LOG = LoggerFactory.getLogger(StarRocksContainer.class); - public static final String DEFAULT_IMAGE = "starrocks/allin1-ubuntu:3.3.22"; + public static final String DEFAULT_IMAGE = TestImages.STARROCKS; public static final String HOST_NAME = "gravitino-ci-starrocks"; public static final String USER_NAME = "root"; public static final String PASSWORD = ""; diff --git a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/TestBaseContainer.java b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/TestBaseContainer.java new file mode 100644 index 00000000000..932412acd6c --- /dev/null +++ b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/TestBaseContainer.java @@ -0,0 +1,141 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.integration.test.container; + +import com.github.dockerjava.api.command.CreateContainerCmd; +import com.github.dockerjava.api.model.HostConfig; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; +import java.time.Duration; +import java.util.Optional; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.mockito.Mockito; +import org.testcontainers.containers.GenericContainer; +import org.testcontainers.containers.Network; +import org.testcontainers.containers.wait.strategy.HostPortWaitStrategy; +import org.testcontainers.containers.wait.strategy.Wait; +import org.testcontainers.containers.wait.strategy.WaitStrategy; +import org.testcontainers.containers.wait.strategy.WaitStrategyTarget; +import org.testcontainers.utility.DockerImageName; + +/** Tests {@link BaseContainer} configuration without Docker; nothing here starts a container. */ +public class TestBaseContainer { + + private static final String HOST_NAME = "test-host"; + + @Test + void testSuppliedContainerIsConfiguredAndKept() { + ProbeContainer supplied = new ProbeContainer(); + Network network = Network.newNetwork(); + + BaseContainer base = new StubContainer(supplied, network); + + Assertions.assertSame(supplied, base.getContainer()); + Assertions.assertEquals(ImmutableList.of(8080), supplied.getExposedPorts()); + Assertions.assertEquals("v", supplied.getEnvMap().get("K")); + Assertions.assertEquals(ImmutableList.of("other-host:10.0.0.1"), supplied.getExtraHosts()); + Assertions.assertSame(network, supplied.getNetwork()); + Assertions.assertTrue(supplied.getNetworkAliases().contains(HOST_NAME)); + + CreateContainerCmd cmd = Mockito.mock(CreateContainerCmd.class); + HostConfig hostConfig = new HostConfig(); + Mockito.when(cmd.getHostConfig()).thenReturn(hostConfig); + supplied.getCreateContainerCmdModifiers().forEach(modifier -> modifier.modify(cmd)); + Mockito.verify(cmd).withHostName(HOST_NAME); + Assertions.assertEquals( + "20000 40000", hostConfig.getSysctls().get("net.ipv4.ip_local_port_range")); + } + + @Test + void testDefaultWaitStrategyReplacesTheSuppliedOne() { + // GenericContainer already defaults to a port check, so start from a different strategy to + // see that BaseContainer really applies its own. + WaitStrategy own = Wait.forLogMessage(".*ready.*", 1); + ProbeContainer supplied = new ProbeContainer(); + supplied.waitingFor(own); + + new StubContainer(supplied, null); + + Assertions.assertNotSame(own, supplied.currentWaitStrategy()); + Assertions.assertInstanceOf(HostPortWaitStrategy.class, supplied.currentWaitStrategy()); + } + + @Test + void testNullWaitStrategyKeepsTheSuppliedOne() { + RecordingWaitStrategy own = new RecordingWaitStrategy(); + ProbeContainer supplied = new ProbeContainer(); + supplied.waitingFor(own); + + new StubContainer(supplied, null) { + @Override + protected WaitStrategy waitStrategy() { + return null; + } + }; + + Assertions.assertSame(own, supplied.currentWaitStrategy()); + // GenericContainer forwards withStartupTimeout to the current wait strategy. + Assertions.assertNull(own.startupTimeout, "the module's own startup timeout was overridden"); + } + + /** Records the startup timeout it is given and never waits. */ + private static class RecordingWaitStrategy implements WaitStrategy { + private Duration startupTimeout; + + @Override + public void waitUntilReady(WaitStrategyTarget waitStrategyTarget) {} + + @Override + public WaitStrategy withStartupTimeout(Duration startupTimeout) { + this.startupTimeout = startupTimeout; + return this; + } + } + + /** Exposes the wait strategy, which GenericContainer keeps protected. */ + private static class ProbeContainer extends GenericContainer { + ProbeContainer() { + super(DockerImageName.parse("alpine:3.20")); + } + + WaitStrategy currentWaitStrategy() { + return getWaitStrategy(); + } + } + + private static class StubContainer extends BaseContainer { + StubContainer(GenericContainer container, Network network) { + super( + container, + HOST_NAME, + ImmutableSet.of(8080), + ImmutableMap.of("other-host", "10.0.0.1"), + ImmutableMap.of(), + ImmutableMap.of("K", "v"), + Optional.ofNullable(network)); + } + + @Override + protected boolean checkContainerStatus(int retryLimit) { + return true; + } + } +} diff --git a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/TestContainerExtension.java b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/TestContainerExtension.java index b8ec75289bb..aef0251905f 100644 --- a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/TestContainerExtension.java +++ b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/TestContainerExtension.java @@ -112,6 +112,17 @@ void testSuperclassFixtureApplies() { run(SubclassFixture.class).testEvents().assertStatistics(stats -> stats.succeeded(1).failed(0)); } + @Test + void testSubclassFixtureReplacesSuperclassFixtureOfSameType() { + run(OverridingFixture.class) + .testEvents() + .assertStatistics(stats -> stats.succeeded(1).failed(0)); + + Assertions.assertEquals( + List.of(new FakeSpec("second")), ContainerExtension.specs(OverridingFixture.class)); + Assertions.assertFalse(LOG.contains("start first"), LOG.toString()); + } + @Test void testTwoFixturesOfOneTypeFailClearly() { run(AmbiguousFixture.class) @@ -130,6 +141,9 @@ void testSpecsReadsEveryFixture() { ContainerExtension.specs(AmbiguousFixture.class)); Assertions.assertEquals( List.of(new KafkaSpec()), ContainerExtension.specs(KafkaFixtureOnly.class)); + Assertions.assertEquals( + List.of(new HiveSpec(HiveVersion.HIVE3, HiveAuth.KERBEROS)), + ContainerExtension.specs(HiveFixtureOnly.class)); } private static EngineExecutionResults run(Class fixture) { @@ -162,7 +176,7 @@ public FakeContainer getOrStart(ContainerSuite suite) { static final class FirstFactory implements ContainerFixture.SpecFactory { @Override public ContainerSpec specFor(FakeFirstFixture fixture) { - return new FakeSpec("first"); + return new FakeSpec(fixture.value()); } } @@ -177,7 +191,9 @@ public ContainerSpec specFor(FakeSecondFixture fixture) { @Retention(RetentionPolicy.RUNTIME) @ExtendWith(ContainerExtension.class) @ContainerFixture(FirstFactory.class) - @interface FakeFirstFixture {} + @interface FakeFirstFixture { + String value() default "first"; + } @Target(ElementType.TYPE) @Retention(RetentionPolicy.RUNTIME) @@ -271,6 +287,15 @@ void test(FakeContainer container) { } } + @Tag(FIXTURE) + @FakeFirstFixture("second") + static class OverridingFixture extends FixtureBase { + @Test + void test(FakeContainer container) { + Assertions.assertSame(SECOND, container); + } + } + @Tag(FIXTURE) @FakeFirstFixture @FakeSecondFixture @@ -282,4 +307,8 @@ void test(FakeContainer container) {} @Tag(FIXTURE) @KafkaFixture static class KafkaFixtureOnly {} + + @Tag(FIXTURE) + @HiveFixture(version = HiveVersion.HIVE3, auth = HiveAuth.KERBEROS) + static class HiveFixtureOnly {} } diff --git a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/TestContainerSlot.java b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/TestContainerSlot.java index 70791ff19ed..6341f7b2d60 100644 --- a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/TestContainerSlot.java +++ b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/TestContainerSlot.java @@ -61,4 +61,47 @@ void testClearForgetsTheContainer() { Assertions.assertNull(slot.get()); Assertions.assertSame(restarted, slot.getOrStart(HiveContainer.HIVE2, () -> restarted)); } + + @Test + void testFailedStartStoresNoContainerAndAllowsARetry() { + ContainerSlot slot = new ContainerSlot<>(); + IllegalStateException failure = new IllegalStateException("not ready"); + HiveContainer retried = Mockito.mock(HiveContainer.class); + + Assertions.assertSame( + failure, + Assertions.assertThrows( + IllegalStateException.class, + () -> + slot.getOrStart( + HiveContainer.HIVE2, + () -> { + throw failure; + }))); + + Assertions.assertNull(slot.get()); + Assertions.assertSame(retried, slot.getOrStart(HiveContainer.HIVE2, () -> retried)); + } + + @Test + void testFailedReplacementClosesThePreviousContainerOnce() { + ContainerSlot slot = new ContainerSlot<>(); + HiveContainer hive2 = Mockito.mock(HiveContainer.class); + slot.getOrStart(HiveContainer.HIVE2, () -> hive2); + + Assertions.assertThrows( + IllegalStateException.class, + () -> + slot.getOrStart( + HiveContainer.HIVE3, + () -> { + throw new IllegalStateException("not ready"); + })); + + Mockito.verify(hive2, Mockito.times(1)).close(); + Assertions.assertNull(slot.get()); + HiveContainer restarted = Mockito.mock(HiveContainer.class); + Assertions.assertSame(restarted, slot.getOrStart(HiveContainer.HIVE2, () -> restarted)); + Mockito.verify(hive2, Mockito.times(1)).close(); + } } diff --git a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/TestContainerSuite.java b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/TestContainerSuite.java index 9a8ce00bfb3..0f8c0319363 100644 --- a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/TestContainerSuite.java +++ b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/TestContainerSuite.java @@ -89,7 +89,16 @@ void testStartTimedStartsTheContainer() { } @Test - void testStartTimedRethrowsStartFailure() { + void testStartTimedKeepsAStartedContainerRunning() { + BaseContainer container = mockContainer(); + + ContainerSuite.startTimed(container); + + Mockito.verify(container, Mockito.never()).close(); + } + + @Test + void testStartTimedClosesAndRethrowsFailedStart() { BaseContainer container = mockContainer(); IllegalStateException failure = new IllegalStateException("not ready"); Mockito.doThrow(failure).when(container).start(); @@ -98,6 +107,39 @@ void testStartTimedRethrowsStartFailure() { failure, Assertions.assertThrows( IllegalStateException.class, () -> ContainerSuite.startTimed(container))); + Mockito.verify(container).close(); + } + + @Test + void testStartTimedReportsStartFailureWhenCloseFails() { + BaseContainer container = mockContainer(); + IllegalStateException failure = new IllegalStateException("not ready"); + IllegalStateException closeFailure = new IllegalStateException("cannot stop"); + Mockito.doThrow(failure).when(container).start(); + Mockito.doThrow(closeFailure).when(container).close(); + + IllegalStateException thrown = + Assertions.assertThrows( + IllegalStateException.class, () -> ContainerSuite.startTimed(container)); + + Assertions.assertSame(failure, thrown); + Assertions.assertArrayEquals(new Throwable[] {closeFailure}, thrown.getSuppressed()); + } + + @Test + void testStartTimedReportsStartFailureWhenCloseThrowsAnError() { + BaseContainer container = mockContainer(); + IllegalStateException failure = new IllegalStateException("not ready"); + AssertionError closeFailure = new AssertionError("cannot stop"); + Mockito.doThrow(failure).when(container).start(); + Mockito.doThrow(closeFailure).when(container).close(); + + IllegalStateException thrown = + Assertions.assertThrows( + IllegalStateException.class, () -> ContainerSuite.startTimed(container)); + + Assertions.assertSame(failure, thrown); + Assertions.assertArrayEquals(new Throwable[] {closeFailure}, thrown.getSuppressed()); } private static BaseContainer mockContainer() { diff --git a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/TestGravitinoLocalStackContainer.java b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/TestGravitinoLocalStackContainer.java index 1fcab41d0b0..377dc62947d 100644 --- a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/TestGravitinoLocalStackContainer.java +++ b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/TestGravitinoLocalStackContainer.java @@ -20,10 +20,18 @@ import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; +import org.testcontainers.containers.wait.strategy.HttpWaitStrategy; /** Tests the LocalStack readiness check without Docker. */ public class TestGravitinoLocalStackContainer { + @Test + void testWaitsForTheHealthEndpoint() { + GravitinoLocalStackContainer container = GravitinoLocalStackContainer.builder().build(); + + Assertions.assertInstanceOf(HttpWaitStrategy.class, container.waitStrategy()); + } + @Test void testS3MustBeAvailableOrRunning() { Assertions.assertTrue( diff --git a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/TestImages.java b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/TestImages.java new file mode 100644 index 00000000000..9bb5991efc9 --- /dev/null +++ b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/TestImages.java @@ -0,0 +1,157 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.integration.test.container; + +import java.io.IOException; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; +import java.util.function.Function; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** + * Every Docker image the integration tests pin, in one place, so a version bump is a one-line + * change and a reviewer can see the whole set at a glance. + * + *

Images that CI swaps out keep their {@code GRAVITINO_CI_*_DOCKER_IMAGE} environment variable + * as an override. The fallbacks here match the values {@code build.gradle.kts} passes to the test + * JVM, so a Gradle run picks the same image as before; when Gradle (or the shell) sets the + * variable, that value wins. The fallbacks let a test started from an IDE without those variables + * still find its image. + */ +public final class TestImages { + + /** Hive with HDFS, the default image for {@link HiveContainer}. */ + public static final String HIVE = + env("GRAVITINO_CI_HIVE_DOCKER_IMAGE", "apache/gravitino-ci:hive-0.1.20"); + + /** Hive with HDFS and a KDC, for Kerberos tests. */ + public static final String KERBEROS_HIVE = + env("GRAVITINO_CI_KERBEROS_HIVE_DOCKER_IMAGE", "apache/gravitino-ci:kerberos-hive-0.1.7"); + + /** Doris 1.2 all-in-one image, the default for {@link DorisContainer}. */ + public static final String DORIS = + env("GRAVITINO_CI_DORIS_DOCKER_IMAGE", "apache/gravitino-ci:doris-0.1.5"); + + /** Official Doris 2.1 all-in-one image. */ + public static final String DORIS_2_1 = "apache/doris:doris-all-in-one-2.1.0"; + + /** Official Doris 3.0 FE image; {@link DorisContainer} derives the matching BE image. */ + public static final String DORIS_3_0 = "apache/doris:fe-3.0.6.2"; + + /** Official Doris 4.0 FE image; {@link DorisContainer} derives the matching BE image. */ + public static final String DORIS_4_0 = "apache/doris:fe-4.0.6"; + + /** Trino with the Gravitino connector layout. */ + public static final String TRINO = + env("GRAVITINO_CI_TRINO_DOCKER_IMAGE", "apache/gravitino-ci:trino-0.1.6"); + + /** Ranger admin. */ + public static final String RANGER = + env("GRAVITINO_CI_RANGER_DOCKER_IMAGE", "apache/gravitino-ci:ranger-0.1.2"); + + /** Kafka in KRaft mode. */ + public static final String KAFKA = env("GRAVITINO_CI_KAFKA_DOCKER_IMAGE", "apache/kafka:3.7.0"); + + /** LocalStack for AWS service emulation. */ + public static final String LOCALSTACK = + env("GRAVITINO_CI_LOCALSTACK_DOCKER_IMAGE", "localstack/localstack:4.14.0"); + + /** Moto for AWS service emulation; Gradle does not set this variable. */ + public static final String MOTO = env("GRAVITINO_CI_MOTO_DOCKER_IMAGE", "motoserver/moto:5.1.4"); + + /** MySQL 8.0, the default for {@link MySQLContainer}. */ + public static final String MYSQL = "mysql:8.0"; + + /** MySQL 5.7, for tests that need the older server. */ + public static final String MYSQL_5_7 = "mysql:5.7"; + + /** PostgreSQL 12. */ + public static final String POSTGRESQL_12 = "postgres:12"; + + /** PostgreSQL 13, the default for {@link PostgreSQLContainer}. */ + public static final String POSTGRESQL_13 = "postgres:13"; + + /** PostgreSQL 14. */ + public static final String POSTGRESQL_14 = "postgres:14"; + + /** PostgreSQL 15. */ + public static final String POSTGRESQL_15 = "postgres:15"; + + /** PostgreSQL 16. */ + public static final String POSTGRESQL_16 = "postgres:16"; + + /** ClickHouse server; Gradle does not set this variable. */ + public static final String CLICKHOUSE = + env("GRAVITINO_CLICKHOUSE_TEST_IMAGE", "clickhouse:24.8.14"); + + /** OceanBase community edition. */ + public static final String OCEANBASE = "oceanbase/oceanbase-ce:4.2.1-lts"; + + /** StarRocks all-in-one (FE and BE in one container). */ + public static final String STARROCKS = "starrocks/allin1-ubuntu:3.3.22"; + + /** ZooKeeper. */ + public static final String ZOOKEEPER = "zookeeper:3.8.0"; + + /** + * RustFS, pinned by digest in {@code docker-compose-rustfs.yml} so the image-update tooling can + * bump it. Parsed here, not in {@link RustFSContainer}, so reading any image never loads the AWS + * SDK classes RustFSContainer needs. + */ + public static final String RUSTFS = loadRustFSImage(); + + private TestImages() {} + + static String rustFSImageFromCompose(String compose) { + // This single-service manifest is deliberately limited to one unquoted image reference. + // Fail closed if its structure changes, rather than silently using a different image pin. + Matcher matcher = Pattern.compile("(?m)^\\s+image: (\\S+)\\s*$").matcher(compose); + if (!matcher.find()) { + throw new IllegalArgumentException("RustFS Compose manifest must contain an image"); + } + String image = matcher.group(1); + if (matcher.find() || !image.matches("rustfs/rustfs:[^@\\s]+@sha256:[0-9a-f]{64}")) { + throw new IllegalArgumentException( + "RustFS Compose manifest must contain one version and digest pin"); + } + return image; + } + + private static String loadRustFSImage() { + try (InputStream stream = TestImages.class.getResourceAsStream("/docker-compose-rustfs.yml")) { + if (stream == null) { + throw new IllegalStateException("Missing docker-compose-rustfs.yml test resource"); + } + return rustFSImageFromCompose(new String(stream.readAllBytes(), StandardCharsets.UTF_8)); + } catch (IOException e) { + throw new IllegalStateException("Unable to read RustFS image pin", e); + } + } + + private static String env(String variable, String fallback) { + return resolve(System::getenv, variable, fallback); + } + + // Takes the lookup as a function so the precedence rule can be tested without the environment. + static String resolve(Function lookup, String variable, String fallback) { + String value = lookup.apply(variable); + return value == null || value.isBlank() ? fallback : value; + } +} diff --git a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/TestMotoSpec.java b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/TestMotoSpec.java new file mode 100644 index 00000000000..70c2021a02b --- /dev/null +++ b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/TestMotoSpec.java @@ -0,0 +1,45 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.integration.test.container; + +import java.util.List; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +public class TestMotoSpec { + + @MotoFixture + private static class MotoClass {} + + @Test + void testFactoryReturnsMotoSpec() { + MotoFixture fixture = MotoClass.class.getAnnotation(MotoFixture.class); + Assertions.assertEquals(new MotoSpec(), new MotoSpec.Factory().specFor(fixture)); + } + + @Test + void testContainerTypeIsMotoContainer() { + Assertions.assertEquals(GravitinoMotoContainer.class, new MotoSpec().containerType()); + } + + @Test + void testExtensionFindsMotoSpec() { + Assertions.assertEquals(List.of(new MotoSpec()), ContainerExtension.specs(MotoClass.class)); + } +} diff --git a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/TestRangerSpec.java b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/TestRangerSpec.java new file mode 100644 index 00000000000..5b8b932709f --- /dev/null +++ b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/TestRangerSpec.java @@ -0,0 +1,56 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.integration.test.container; + +import java.util.List; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.mockito.InOrder; +import org.mockito.Mockito; + +/** Tests {@link RangerSpec} and {@link RangerFixture} without Docker. */ +public class TestRangerSpec { + + @Test + void testFixtureMapsToTheRangerSpec() { + Assertions.assertEquals( + List.of(new RangerSpec()), ContainerExtension.specs(RangerFixtureOnly.class)); + } + + @Test + void testContainerType() { + Assertions.assertEquals(RangerContainer.class, new RangerSpec().containerType()); + } + + @Test + void testGetOrStartStartsThroughTheSuite() { + ContainerSuite suite = Mockito.mock(ContainerSuite.class); + RangerContainer ranger = Mockito.mock(RangerContainer.class); + Mockito.when(suite.getRangerContainer()).thenReturn(ranger); + + Assertions.assertSame(ranger, new RangerSpec().getOrStart(suite)); + + InOrder order = Mockito.inOrder(suite); + order.verify(suite).startRangerContainer(); + order.verify(suite).getRangerContainer(); + } + + @RangerFixture + static class RangerFixtureOnly {} +} diff --git a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/TestRustFSContainer.java b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/TestRustFSContainer.java index 50e0379a7c8..4067a33309d 100644 --- a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/TestRustFSContainer.java +++ b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/TestRustFSContainer.java @@ -34,18 +34,18 @@ void testImagePinFromPackagedResource() { void testImagePinParsing() { String manifest = "services:\n rustfs:\n image: " + RustFSContainer.DEFAULT_IMAGE + "\n"; Assertions.assertEquals( - RustFSContainer.DEFAULT_IMAGE, RustFSContainer.imageFromCompose(manifest)); + RustFSContainer.DEFAULT_IMAGE, TestImages.rustFSImageFromCompose(manifest)); Assertions.assertThrows( IllegalArgumentException.class, - () -> RustFSContainer.imageFromCompose(manifest + " image: another/image:latest\n")); + () -> TestImages.rustFSImageFromCompose(manifest + " image: another/image:latest\n")); Assertions.assertThrows( IllegalArgumentException.class, - () -> RustFSContainer.imageFromCompose("services:\n rustfs:\n")); + () -> TestImages.rustFSImageFromCompose("services:\n rustfs:\n")); Assertions.assertThrows( IllegalArgumentException.class, - () -> RustFSContainer.imageFromCompose(" image: rustfs/rustfs:latest\n")); + () -> TestImages.rustFSImageFromCompose(" image: rustfs/rustfs:latest\n")); Assertions.assertThrows( IllegalArgumentException.class, - () -> RustFSContainer.imageFromCompose(" image: rustfs/rustfs:1.0.0@sha256:bad\n")); + () -> TestImages.rustFSImageFromCompose(" image: rustfs/rustfs:1.0.0@sha256:bad\n")); } } diff --git a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/TestTestImages.java b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/TestTestImages.java new file mode 100644 index 00000000000..d9d36f87e72 --- /dev/null +++ b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/TestTestImages.java @@ -0,0 +1,77 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.integration.test.container; + +import java.lang.reflect.Field; +import java.lang.reflect.Modifier; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +/** + * Checks that every image in {@link TestImages} is set and pinned to a version, and that an + * environment override wins over the fallback. + */ +public class TestTestImages { + + @Test + void testEveryImageIsPinned() throws IllegalAccessException { + List images = new ArrayList<>(); + for (Field field : TestImages.class.getDeclaredFields()) { + int modifiers = field.getModifiers(); + if (Modifier.isPublic(modifiers) + && Modifier.isStatic(modifiers) + && field.getType() == String.class) { + String image = (String) field.get(null); + Assertions.assertNotNull(image, field.getName()); + Assertions.assertFalse(image.isBlank(), field.getName() + " is blank"); + // A colon before the last slash is a registry port, not a tag. + Assertions.assertTrue( + image.lastIndexOf(':') > image.lastIndexOf('/'), + field.getName() + " has no tag: " + image); + Assertions.assertFalse( + image.endsWith(":latest") || image.contains(":latest@"), + field.getName() + " uses the latest tag: " + image); + images.add(image); + } + } + Assertions.assertFalse(images.isEmpty(), "TestImages declares no images"); + } + + @Test + void testSetVariableOverridesTheFallback() { + Assertions.assertEquals( + "custom:1.0", + TestImages.resolve(Map.of("IMAGE", "custom:1.0")::get, "IMAGE", "fallback:1.0")); + } + + @Test + void testUnsetVariableUsesTheFallback() { + Assertions.assertEquals( + "fallback:1.0", TestImages.resolve(Map.of()::get, "IMAGE", "fallback:1.0")); + } + + @Test + void testBlankVariableUsesTheFallback() { + Assertions.assertEquals( + "fallback:1.0", TestImages.resolve(Map.of("IMAGE", " ")::get, "IMAGE", "fallback:1.0")); + } +} diff --git a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/TrinoContainer.java b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/TrinoContainer.java index 47f93aa04e8..0b4251e0537 100644 --- a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/TrinoContainer.java +++ b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/TrinoContainer.java @@ -49,7 +49,7 @@ public class TrinoContainer extends BaseContainer { public static final Logger LOG = LoggerFactory.getLogger(TrinoContainer.class); - public static final String DEFAULT_IMAGE = System.getenv("GRAVITINO_CI_TRINO_DOCKER_IMAGE"); + public static final String DEFAULT_IMAGE = TestImages.TRINO; public static final String HOST_NAME = "gravitino-ci-trino"; public static final int TRINO_PORT = 8080; public static final int TRINO_HTTPS_PORT = 8443; diff --git a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/ZooKeeperContainer.java b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/ZooKeeperContainer.java index 73125773dac..d6935b4e87a 100644 --- a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/ZooKeeperContainer.java +++ b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/ZooKeeperContainer.java @@ -29,7 +29,7 @@ public class ZooKeeperContainer extends BaseContainer { private static final Logger LOG = LoggerFactory.getLogger(ZooKeeperContainer.class); - public static final String DEFAULT_IMAGE = "zookeeper:3.8.0"; + public static final String DEFAULT_IMAGE = TestImages.ZOOKEEPER; public static final String HOST_NAME = "gravitino-ci-zookeeper"; public static final int ZK_PORT = 2181; From 7186e6fcec56af12edf8ade65aabef0a2a970f37 Mon Sep 17 00:00:00 2001 From: Nevin Zheng Date: Mon, 19 Oct 2026 15:45:44 -0700 Subject: [PATCH 11/14] [#14002] improvement(test): Move Hive callers to @HiveFixture; add ClickHouse fixtures and scope (part 3/6) - client-java, fileset, Hudi, Iceberg, Paimon, Iceberg REST, GVFS, Flink, Spark and Trino env tests use @HiveFixture. - ClickHouse (1 or 3 nodes) and ZooKeeper fixtures. - scope = RUN | CLASS | METHOD; HadoopUserAuthenticationIT gets a private Kerberos Hive. --- .../test/AuditCatalogClickHouseIT.java | 13 +- .../test/CatalogClickHouseClusterIT.java | 10 +- .../integration/test/CatalogClickHouseIT.java | 8 +- .../clickhouse/operations/TestClickHouse.java | 8 +- .../operations/TestClickHouseEngineIT.java | 10 +- .../integration/test/FilesetCatalogIT.java | 20 +- .../integration/test/HDFSKerberosIT.java | 25 +-- .../test/HadoopUserAuthenticationIT.java | 41 ++-- .../integration/test/HudiCatalogHMSIT.java | 15 +- .../test/HudiCatalogKerberosHiveIT.java | 28 ++- .../test/CatalogIcebergBaseIT.java | 10 +- .../test/CatalogIcebergHiveIT.java | 7 +- .../test/CatalogIcebergKerberosHiveIT.java | 34 ++- .../test/CatalogIcebergRestIT.java | 3 +- .../integration/test/CatalogPaimonBaseIT.java | 8 +- .../test/CatalogPaimonFileSystemIT.java | 8 +- .../integration/test/CatalogPaimonHiveIT.java | 12 +- .../integration/test/CatalogPaimonJdbcIT.java | 8 +- .../CatalogPaimonKerberosFilesystemIT.java | 26 +-- .../paimon/utils/TestCatalogUtils.java | 12 +- .../client/integration/test/CatalogIT.java | 13 +- .../client/integration/test/FunctionIT.java | 11 +- .../client/integration/test/PolicyIT.java | 11 +- .../client/integration/test/StatisticIT.java | 15 +- .../client/integration/test/TagIT.java | 11 +- .../authorization/CatalogAuthorizationIT.java | 13 +- .../authorization/FilesetAuthorizationIT.java | 11 +- .../MetadataObjectRoleAuthorizationIT.java | 10 +- .../authorization/OwnerAuthorizationIT.java | 12 +- .../test/authorization/OwnerIT.java | 10 +- .../PartitionAuthorizationIT.java | 12 +- .../authorization/PolicyAuthorizationIT.java | 12 +- .../authorization/SchemaAuthorizationIT.java | 13 +- .../StatisticAuthorizationIT.java | 13 +- .../authorization/TableAuthorizationIT.java | 12 +- .../TagOperationsAuthorizationIT.java | 12 +- .../test/GravitinoVirtualFileSystemIT.java | 12 +- .../integration/test/FlinkEnvIT.java | 18 +- .../integration/test/IcebergCacheIT.java | 19 +- .../test/IcebergRESTHiveCatalogIT.java | 17 +- .../test/IcebergRESTJdbcCatalogIT.java | 18 +- .../IcebergRestKerberosHiveCatalogIT.java | 26 ++- ...rosHiveWithUserImpersonationCatalogIT.java | 7 +- .../IcebergRestKerberosJdbcCatalogIT.java | 19 +- ...rosJdbcWithUserImpersonationCatalogIT.java | 8 +- .../test/IcebergRestKerberosTestEnv.java | 13 +- .../test/container/ClickHouseFixture.java | 65 ++++++ .../test/container/ClickHouseSpec.java | 105 +++++++++ .../test/container/ContainerExtension.java | 145 +++++++++--- .../test/container/ContainerFixture.java | 13 +- .../test/container/ContainerSpec.java | 14 ++ .../test/container/ContainerSuite.java | 68 ++++-- .../test/container/HiveFixture.java | 12 +- .../integration/test/container/HiveSpec.java | 34 ++- .../integration/test/container/Scope.java | 40 ++++ .../test/container/TestClickHouseSpec.java | 137 ++++++++++++ .../container/TestContainerExtension.java | 208 ++++++++++++++++++ .../test/container/TestContainerSuite.java | 28 +++ .../test/container/TestZooKeeperSpec.java | 59 +++++ .../test/container/ZooKeeperFixture.java | 39 ++++ .../test/container/ZooKeeperSpec.java | 41 ++++ .../src/test/resources/remote_servers.xml | 0 .../integration/test/SparkEnvIT.java | 18 +- .../integration/test/TrinoConnectorIT.java | 47 ++-- 64 files changed, 1294 insertions(+), 433 deletions(-) create mode 100644 integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/ClickHouseFixture.java create mode 100644 integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/ClickHouseSpec.java create mode 100644 integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/Scope.java create mode 100644 integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/TestClickHouseSpec.java create mode 100644 integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/TestZooKeeperSpec.java create mode 100644 integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/ZooKeeperFixture.java create mode 100644 integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/ZooKeeperSpec.java rename {catalogs-contrib/catalog-jdbc-clickhouse => integration-test-common}/src/test/resources/remote_servers.xml (100%) diff --git a/catalogs-contrib/catalog-jdbc-clickhouse/src/test/java/org/apache/gravitino/catalog/clickhouse/integration/test/AuditCatalogClickHouseIT.java b/catalogs-contrib/catalog-jdbc-clickhouse/src/test/java/org/apache/gravitino/catalog/clickhouse/integration/test/AuditCatalogClickHouseIT.java index 9416dc7b4cd..5a334fb8180 100644 --- a/catalogs-contrib/catalog-jdbc-clickhouse/src/test/java/org/apache/gravitino/catalog/clickhouse/integration/test/AuditCatalogClickHouseIT.java +++ b/catalogs-contrib/catalog-jdbc-clickhouse/src/test/java/org/apache/gravitino/catalog/clickhouse/integration/test/AuditCatalogClickHouseIT.java @@ -38,7 +38,8 @@ import org.apache.gravitino.catalog.jdbc.config.JdbcConfig; import org.apache.gravitino.client.GravitinoMetalake; import org.apache.gravitino.integration.test.container.ClickHouseContainer; -import org.apache.gravitino.integration.test.container.ContainerSuite; +import org.apache.gravitino.integration.test.container.ClickHouseFixture; +import org.apache.gravitino.integration.test.container.FixtureContainer; import org.apache.gravitino.integration.test.util.BaseIT; import org.apache.gravitino.integration.test.util.GravitinoITUtils; import org.apache.gravitino.integration.test.util.TestDatabaseName; @@ -53,8 +54,8 @@ import org.junit.jupiter.api.Test; @Tag("gravitino-docker-test") +@ClickHouseFixture(database = TestDatabaseName.CLICKHOUSE_AUDIT_CATALOG_CLICKHOUSE_IT) public class AuditCatalogClickHouseIT extends BaseIT { - private static final ContainerSuite containerSuite = ContainerSuite.getInstance(); public static final String metalakeName = GravitinoITUtils.genRandomName("audit_clickhouse_metalake"); private static final String expectUser = System.getProperty("user.name"); @@ -62,7 +63,7 @@ public class AuditCatalogClickHouseIT extends BaseIT { private static final String provider = "jdbc-clickhouse"; private static ClickHouseService clickhouseService; - private static ClickHouseContainer CLICKHOUSE_CONTAINER; + @FixtureContainer private static ClickHouseContainer CLICKHOUSE_CONTAINER; private static GravitinoMetalake metalake; @BeforeAll @@ -72,12 +73,8 @@ public void startIntegrationTest() throws Exception { registerCustomConfigs(configs); super.startIntegrationTest(); - containerSuite.startClickHouseContainer( - TestDatabaseName.CLICKHOUSE_AUDIT_CATALOG_CLICKHOUSE_IT); - CLICKHOUSE_CONTAINER = containerSuite.getClickHouseContainer(); TEST_DB_NAME = TestDatabaseName.CLICKHOUSE_AUDIT_CATALOG_CLICKHOUSE_IT; - clickhouseService = - new ClickHouseService(containerSuite.getClickHouseContainer(), TEST_DB_NAME); + clickhouseService = new ClickHouseService(CLICKHOUSE_CONTAINER, TEST_DB_NAME); createMetalake(); } diff --git a/catalogs-contrib/catalog-jdbc-clickhouse/src/test/java/org/apache/gravitino/catalog/clickhouse/integration/test/CatalogClickHouseClusterIT.java b/catalogs-contrib/catalog-jdbc-clickhouse/src/test/java/org/apache/gravitino/catalog/clickhouse/integration/test/CatalogClickHouseClusterIT.java index 6dd44dd0323..f6e1c9e1651 100644 --- a/catalogs-contrib/catalog-jdbc-clickhouse/src/test/java/org/apache/gravitino/catalog/clickhouse/integration/test/CatalogClickHouseClusterIT.java +++ b/catalogs-contrib/catalog-jdbc-clickhouse/src/test/java/org/apache/gravitino/catalog/clickhouse/integration/test/CatalogClickHouseClusterIT.java @@ -28,7 +28,6 @@ import static org.apache.gravitino.rel.Column.DEFAULT_VALUE_NOT_SET; import com.google.common.collect.Maps; -import java.nio.file.Paths; import java.sql.Connection; import java.sql.DriverManager; import java.sql.PreparedStatement; @@ -54,7 +53,9 @@ import org.apache.gravitino.catalog.jdbc.config.JdbcConfig; import org.apache.gravitino.client.GravitinoMetalake; import org.apache.gravitino.integration.test.container.ClickHouseContainer; +import org.apache.gravitino.integration.test.container.ClickHouseFixture; import org.apache.gravitino.integration.test.container.ContainerSuite; +import org.apache.gravitino.integration.test.container.FixtureContainer; import org.apache.gravitino.integration.test.util.BaseIT; import org.apache.gravitino.integration.test.util.GravitinoITUtils; import org.apache.gravitino.integration.test.util.TestDatabaseName; @@ -82,6 +83,7 @@ @Tag("gravitino-docker-test") @TestInstance(Lifecycle.PER_CLASS) +@ClickHouseFixture(nodes = 3, database = TestDatabaseName.CLICKHOUSE_CLUSTER_CLICKHOUSE_IT) public class CatalogClickHouseClusterIT extends BaseIT { private static final ContainerSuite containerSuite = ContainerSuite.getInstance(); private static final String provider = "jdbc-clickhouse"; @@ -105,16 +107,12 @@ public class CatalogClickHouseClusterIT extends BaseIT { private GravitinoMetalake metalake; private Catalog catalog; private ClickHouseService clickHouseService; - private ClickHouseContainer clickHouseClusterContainer; + @FixtureContainer private ClickHouseContainer clickHouseClusterContainer; private List clickHouseClusterContainers; private final TestDatabaseName TEST_DB_NAME = TestDatabaseName.CLICKHOUSE_CLUSTER_CLICKHOUSE_IT; @BeforeAll public void startup() { - String remoteServersConfig = - Paths.get("src", "test", "resources", "remote_servers.xml").toAbsolutePath().toString(); - containerSuite.startClickHouseClusterContainer(TEST_DB_NAME, remoteServersConfig); - clickHouseClusterContainer = containerSuite.getClickHouseClusterContainer(); clickHouseClusterContainers = containerSuite.getClickHouseClusterContainers(); Assertions.assertEquals(3, clickHouseClusterContainers.size()); diff --git a/catalogs-contrib/catalog-jdbc-clickhouse/src/test/java/org/apache/gravitino/catalog/clickhouse/integration/test/CatalogClickHouseIT.java b/catalogs-contrib/catalog-jdbc-clickhouse/src/test/java/org/apache/gravitino/catalog/clickhouse/integration/test/CatalogClickHouseIT.java index 801b0759674..cf6ba3ddd79 100644 --- a/catalogs-contrib/catalog-jdbc-clickhouse/src/test/java/org/apache/gravitino/catalog/clickhouse/integration/test/CatalogClickHouseIT.java +++ b/catalogs-contrib/catalog-jdbc-clickhouse/src/test/java/org/apache/gravitino/catalog/clickhouse/integration/test/CatalogClickHouseIT.java @@ -67,7 +67,9 @@ import org.apache.gravitino.exceptions.NoSuchSchemaException; import org.apache.gravitino.exceptions.NotFoundException; import org.apache.gravitino.integration.test.container.ClickHouseContainer; +import org.apache.gravitino.integration.test.container.ClickHouseFixture; import org.apache.gravitino.integration.test.container.ContainerSuite; +import org.apache.gravitino.integration.test.container.FixtureContainer; import org.apache.gravitino.integration.test.util.BaseIT; import org.apache.gravitino.integration.test.util.GravitinoITUtils; import org.apache.gravitino.integration.test.util.ITUtils; @@ -108,6 +110,7 @@ @Tag("gravitino-docker-test") @TestInstance(Lifecycle.PER_CLASS) +@ClickHouseFixture(database = TestDatabaseName.CLICKHOUSE_CATALOG_CLICKHOUSE_IT) public class CatalogClickHouseIT extends BaseIT { private static final ContainerSuite containerSuite = ContainerSuite.getInstance(); private static final String provider = "jdbc-clickhouse"; @@ -132,7 +135,7 @@ public class CatalogClickHouseIT extends BaseIT { private ClickHouseService clickhouseService; - private ClickHouseContainer CLICKHOUSE_CONTAINER; + @FixtureContainer private ClickHouseContainer CLICKHOUSE_CONTAINER; private TestDatabaseName TEST_DB_NAME; @@ -144,9 +147,6 @@ boolean supportColumnDefaultValueExpression() { public void startup() throws IOException, SQLException { TEST_DB_NAME = TestDatabaseName.CLICKHOUSE_CATALOG_CLICKHOUSE_IT; - containerSuite.startClickHouseContainer(TEST_DB_NAME); - CLICKHOUSE_CONTAINER = containerSuite.getClickHouseContainer(); - clickhouseService = new ClickHouseService(CLICKHOUSE_CONTAINER, TEST_DB_NAME); createMetalake(); createCatalog(); diff --git a/catalogs-contrib/catalog-jdbc-clickhouse/src/test/java/org/apache/gravitino/catalog/clickhouse/operations/TestClickHouse.java b/catalogs-contrib/catalog-jdbc-clickhouse/src/test/java/org/apache/gravitino/catalog/clickhouse/operations/TestClickHouse.java index 082b50b5b92..516d96db95c 100644 --- a/catalogs-contrib/catalog-jdbc-clickhouse/src/test/java/org/apache/gravitino/catalog/clickhouse/operations/TestClickHouse.java +++ b/catalogs-contrib/catalog-jdbc-clickhouse/src/test/java/org/apache/gravitino/catalog/clickhouse/operations/TestClickHouse.java @@ -30,19 +30,21 @@ import org.apache.gravitino.catalog.jdbc.config.JdbcConfig; import org.apache.gravitino.catalog.jdbc.utils.DataSourceUtils; import org.apache.gravitino.integration.test.container.ClickHouseContainer; +import org.apache.gravitino.integration.test.container.ClickHouseFixture; import org.apache.gravitino.integration.test.container.ContainerSuite; +import org.apache.gravitino.integration.test.container.FixtureContainer; import org.apache.gravitino.integration.test.util.TestDatabaseName; import org.junit.jupiter.api.BeforeAll; +@ClickHouseFixture(database = TestDatabaseName.CLICKHOUSE_CLICKHOUSE_ABSTRACT_IT) public class TestClickHouse extends TestJdbc { protected static final ContainerSuite containerSuite = ContainerSuite.getInstance(); protected static TestDatabaseName TEST_DB_NAME; + @FixtureContainer private static ClickHouseContainer clickHouseContainer; @BeforeAll public static void startup() throws Exception { - ContainerSuite containerSuite = ContainerSuite.getInstance(); TEST_DB_NAME = TestDatabaseName.CLICKHOUSE_CLICKHOUSE_ABSTRACT_IT; - containerSuite.startClickHouseContainer(TEST_DB_NAME); DataSource dataSource = DataSourceUtils.createDataSource(getClickHouseCatalogProperties()); DATABASE_OPERATIONS = new ClickHouseDatabaseOperations(); @@ -60,8 +62,6 @@ public static void startup() throws Exception { protected static Map getClickHouseCatalogProperties() throws SQLException { Map catalogProperties = Maps.newHashMap(); - ClickHouseContainer clickHouseContainer = containerSuite.getClickHouseContainer(); - String jdbcUrl = clickHouseContainer.getJdbcUrl(TEST_DB_NAME); catalogProperties.put(JdbcConfig.JDBC_URL.getKey(), jdbcUrl); diff --git a/catalogs-contrib/catalog-jdbc-clickhouse/src/test/java/org/apache/gravitino/catalog/clickhouse/operations/TestClickHouseEngineIT.java b/catalogs-contrib/catalog-jdbc-clickhouse/src/test/java/org/apache/gravitino/catalog/clickhouse/operations/TestClickHouseEngineIT.java index e1af276862a..5765ed1cae3 100644 --- a/catalogs-contrib/catalog-jdbc-clickhouse/src/test/java/org/apache/gravitino/catalog/clickhouse/operations/TestClickHouseEngineIT.java +++ b/catalogs-contrib/catalog-jdbc-clickhouse/src/test/java/org/apache/gravitino/catalog/clickhouse/operations/TestClickHouseEngineIT.java @@ -40,7 +40,8 @@ import org.apache.gravitino.catalog.jdbc.config.JdbcConfig; import org.apache.gravitino.catalog.jdbc.utils.DataSourceUtils; import org.apache.gravitino.integration.test.container.ClickHouseContainer; -import org.apache.gravitino.integration.test.container.ContainerSuite; +import org.apache.gravitino.integration.test.container.ClickHouseFixture; +import org.apache.gravitino.integration.test.container.FixtureContainer; import org.apache.gravitino.integration.test.util.TestDatabaseName; import org.apache.gravitino.rel.expressions.distributions.Distributions; import org.apache.gravitino.rel.indexes.Index; @@ -86,21 +87,18 @@ */ @Tag("gravitino-docker-test") @TestInstance(Lifecycle.PER_CLASS) +@ClickHouseFixture(database = TestDatabaseName.CLICKHOUSE_ENGINE_IT) public class TestClickHouseEngineIT { - private static final ContainerSuite containerSuite = ContainerSuite.getInstance(); private static final TestDatabaseName TEST_DB_NAME = TestDatabaseName.CLICKHOUSE_ENGINE_IT; - private ClickHouseContainer clickHouseContainer; + @FixtureContainer private ClickHouseContainer clickHouseContainer; private ClickHouseDatabaseOperations databaseOperations; private ClickHouseTableOperations tableOperations; private String schemaName; @BeforeAll public void setup() throws Exception { - containerSuite.startClickHouseContainer(TEST_DB_NAME); - clickHouseContainer = containerSuite.getClickHouseContainer(); - Map catalogProperties = Maps.newHashMap(); catalogProperties.put(JdbcConfig.JDBC_URL.getKey(), clickHouseContainer.getJdbcUrl()); catalogProperties.put( diff --git a/catalogs/catalog-fileset/src/test/java/org/apache/gravitino/catalog/fileset/integration/test/FilesetCatalogIT.java b/catalogs/catalog-fileset/src/test/java/org/apache/gravitino/catalog/fileset/integration/test/FilesetCatalogIT.java index 2f38fe24c2e..5595ec0b73a 100644 --- a/catalogs/catalog-fileset/src/test/java/org/apache/gravitino/catalog/fileset/integration/test/FilesetCatalogIT.java +++ b/catalogs/catalog-fileset/src/test/java/org/apache/gravitino/catalog/fileset/integration/test/FilesetCatalogIT.java @@ -52,8 +52,9 @@ import org.apache.gravitino.exceptions.NoSuchFilesetException; import org.apache.gravitino.file.Fileset; import org.apache.gravitino.file.FilesetChange; -import org.apache.gravitino.integration.test.container.ContainerSuite; +import org.apache.gravitino.integration.test.container.FixtureContainer; import org.apache.gravitino.integration.test.container.HiveContainer; +import org.apache.gravitino.integration.test.container.HiveFixture; import org.apache.gravitino.integration.test.util.BaseIT; import org.apache.gravitino.integration.test.util.GravitinoITUtils; import org.apache.hadoop.conf.Configuration; @@ -76,9 +77,11 @@ import org.slf4j.LoggerFactory; @Tag("gravitino-docker-test") +@HiveFixture public class FilesetCatalogIT extends BaseIT { private static final Logger LOG = LoggerFactory.getLogger(FilesetCatalogIT.class); - protected static final ContainerSuite containerSuite = ContainerSuite.getInstance(); + + @FixtureContainer private HiveContainer hive; protected String metalakeName = GravitinoITUtils.genRandomName("CatalogFilesetIT_metalake"); protected String catalogName = GravitinoITUtils.genRandomName("CatalogFilesetIT_catalog"); @@ -92,9 +95,8 @@ public class FilesetCatalogIT extends BaseIT { protected CloseableHttpClient httpClient; - protected void startNecessaryContainer() { - containerSuite.startHiveContainer(); - } + // Hook for subclasses whose storage needs more setup than the Hive fixture gives. + protected void startNecessaryContainer() {} @BeforeAll public void setup() throws IOException { @@ -1226,9 +1228,7 @@ public void testCreateSchemaAndFilesetWithSpecialLocation() { String localCatalogName = GravitinoITUtils.genRandomName("local_catalog"); String hdfsLocation = String.format( - "hdfs://%s:%d", - containerSuite.getHiveContainer().getContainerIpAddress(), - HiveContainer.HDFS_DEFAULTFS_PORT); + "hdfs://%s:%d", hive.getContainerIpAddress(), HiveContainer.HDFS_DEFAULTFS_PORT); Map catalogProps = ImmutableMap.of("location", hdfsLocation); Catalog localCatalog = @@ -1285,7 +1285,7 @@ public void testCreateSchemaAndFilesetWithSpecialLocation() { protected String generateLocation(String filesetName) { return String.format( "hdfs://%s:%d/user/hadoop/%s/%s/%s", - containerSuite.getHiveContainer().getContainerIpAddress(), + hive.getContainerIpAddress(), HiveContainer.HDFS_DEFAULTFS_PORT, catalogName, schemaName, @@ -1355,7 +1355,7 @@ protected String defaultBaseLocation() { defaultBaseLocation = String.format( "hdfs://%s:%d/user/hadoop/%s.db", - containerSuite.getHiveContainer().getContainerIpAddress(), + hive.getContainerIpAddress(), HiveContainer.HDFS_DEFAULTFS_PORT, schemaName.toLowerCase()); } diff --git a/catalogs/catalog-fileset/src/test/java/org/apache/gravitino/catalog/fileset/integration/test/HDFSKerberosIT.java b/catalogs/catalog-fileset/src/test/java/org/apache/gravitino/catalog/fileset/integration/test/HDFSKerberosIT.java index f4b2ddd3c4c..6c077352990 100644 --- a/catalogs/catalog-fileset/src/test/java/org/apache/gravitino/catalog/fileset/integration/test/HDFSKerberosIT.java +++ b/catalogs/catalog-fileset/src/test/java/org/apache/gravitino/catalog/fileset/integration/test/HDFSKerberosIT.java @@ -25,8 +25,10 @@ import java.nio.file.Files; import java.security.PrivilegedAction; import org.apache.commons.io.FileUtils; -import org.apache.gravitino.integration.test.container.ContainerSuite; +import org.apache.gravitino.integration.test.container.FixtureContainer; +import org.apache.gravitino.integration.test.container.HiveAuth; import org.apache.gravitino.integration.test.container.HiveContainer; +import org.apache.gravitino.integration.test.container.HiveFixture; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; @@ -42,10 +44,12 @@ import org.slf4j.LoggerFactory; @Tag("gravitino-docker-test") +@HiveFixture(auth = HiveAuth.KERBEROS) public class HDFSKerberosIT { private static final Logger LOG = LoggerFactory.getLogger(HDFSKerberosIT.class); - private static final ContainerSuite containerSuite = ContainerSuite.getInstance(); + @FixtureContainer private static HiveContainer kerberosHive; + private static final String CLIENT_PRINCIPAL = "cli@HADOOPKRB"; private static UserGroupInformation clientUGI; @@ -53,28 +57,20 @@ public class HDFSKerberosIT { @BeforeAll public static void setup() throws IOException { - containerSuite.startKerberosHiveContainer(); - File baseDir = new File(System.getProperty("java.io.tmpdir")); File file = Files.createTempDirectory(baseDir.toPath(), "test").toFile(); file.deleteOnExit(); // Copy the keytab and krb5.conf from the container keytabPath = file.getAbsolutePath() + "/client.keytab"; - containerSuite - .getKerberosHiveContainer() - .getContainer() - .copyFileFromContainer("/etc/admin.keytab", keytabPath); + kerberosHive.getContainer().copyFileFromContainer("/etc/admin.keytab", keytabPath); String krb5TmpPath = file.getAbsolutePath() + "/krb5.conf_tmp"; String krb5Path = file.getAbsolutePath() + "/krb5.conf"; - containerSuite - .getKerberosHiveContainer() - .getContainer() - .copyFileFromContainer("/etc/krb5.conf", krb5TmpPath); + kerberosHive.getContainer().copyFileFromContainer("/etc/krb5.conf", krb5TmpPath); // Modify the krb5.conf and change the kdc and admin_server to the container IP - String ip = containerSuite.getKerberosHiveContainer().getContainerIpAddress(); + String ip = kerberosHive.getContainerIpAddress(); String content = FileUtils.readFileToString(new File(krb5TmpPath), StandardCharsets.UTF_8); content = content.replace("kdc = localhost:88", "kdc = " + ip + ":88"); content = content.replace("admin_server = localhost", "admin_server = " + ip + ":749"); @@ -131,7 +127,6 @@ public void testKerberosHDFS() throws IOException { private static String defaultBaseLocation() { return String.format( "hdfs://%s:%d/user/", - containerSuite.getKerberosHiveContainer().getContainerIpAddress(), - HiveContainer.HDFS_DEFAULTFS_PORT); + kerberosHive.getContainerIpAddress(), HiveContainer.HDFS_DEFAULTFS_PORT); } } diff --git a/catalogs/catalog-fileset/src/test/java/org/apache/gravitino/catalog/fileset/integration/test/HadoopUserAuthenticationIT.java b/catalogs/catalog-fileset/src/test/java/org/apache/gravitino/catalog/fileset/integration/test/HadoopUserAuthenticationIT.java index af30205e626..a72e65870c0 100644 --- a/catalogs/catalog-fileset/src/test/java/org/apache/gravitino/catalog/fileset/integration/test/HadoopUserAuthenticationIT.java +++ b/catalogs/catalog-fileset/src/test/java/org/apache/gravitino/catalog/fileset/integration/test/HadoopUserAuthenticationIT.java @@ -45,8 +45,11 @@ import org.apache.gravitino.client.GravitinoMetalake; import org.apache.gravitino.client.KerberosTokenProvider; import org.apache.gravitino.file.Fileset; -import org.apache.gravitino.integration.test.container.ContainerSuite; +import org.apache.gravitino.integration.test.container.FixtureContainer; +import org.apache.gravitino.integration.test.container.HiveAuth; import org.apache.gravitino.integration.test.container.HiveContainer; +import org.apache.gravitino.integration.test.container.HiveFixture; +import org.apache.gravitino.integration.test.container.Scope; import org.apache.gravitino.integration.test.util.BaseIT; import org.apache.gravitino.integration.test.util.GravitinoITUtils; import org.apache.gravitino.integration.test.util.KerberosHdfs; @@ -60,12 +63,13 @@ import org.slf4j.LoggerFactory; import org.testcontainers.utility.MountableFile; +// A private Kerberos Hive: the test adds principals with kadmin and changes HDFS permissions, +// which would break the shared Kerberos Hive for later classes. @Tag("gravitino-docker-test") +@HiveFixture(auth = HiveAuth.KERBEROS, scope = Scope.CLASS) public class HadoopUserAuthenticationIT extends BaseIT { private static final Logger LOG = LoggerFactory.getLogger(HadoopUserAuthenticationIT.class); - private static final ContainerSuite containerSuite = ContainerSuite.getInstance(); - private static final String SDK_KERBEROS_PRINCIPAL_KEY = "client.kerberos.principal"; private static final String SDK_KERBEROS_KEYTAB_KEY = "client.kerberos.keytab"; @@ -95,7 +99,7 @@ public class HadoopUserAuthenticationIT extends BaseIT { private static KerberosHdfs hdfs; - private static HiveContainer kerberosHiveContainer; + @FixtureContainer private static HiveContainer kerberosHive; private static final String METALAKE_NAME = GravitinoITUtils.genRandomName("CatalogHadoop_metalake"); @@ -108,19 +112,16 @@ public class HadoopUserAuthenticationIT extends BaseIT { @BeforeAll public void startIntegrationTest() throws Exception { - containerSuite.startKerberosHiveContainer(); - kerberosHiveContainer = containerSuite.getKerberosHiveContainer(); - File baseDir = new File(System.getProperty("java.io.tmpdir")); File file = Files.createTempDirectory(baseDir.toPath(), "test").toFile(); file.deleteOnExit(); TMP_DIR = file.getAbsolutePath(); - HDFS_URL = String.format("hdfs://%s:9000", kerberosHiveContainer.getContainerIpAddress()); + HDFS_URL = String.format("hdfs://%s:9000", kerberosHive.getContainerIpAddress()); // Prepare kerberos related-config; prepareKerberosConfig(); - hdfs = KerberosHdfs.of(kerberosHiveContainer, TMP_DIR + HADOOP_CLIENT_KEYTAB); + hdfs = KerberosHdfs.of(kerberosHive, TMP_DIR + HADOOP_CLIENT_KEYTAB); // Config kerberos configuration for Gravitino server addKerberosConfig(); @@ -141,28 +142,28 @@ public static void stop() { private static void prepareKerberosConfig() throws IOException { // Keytab of the Gravitino SDK client - kerberosHiveContainer + kerberosHive .getContainer() .copyFileFromContainer("/gravitino_client.keytab", TMP_DIR + GRAVITINO_CLIENT_KEYTAB); createKeyTableForSchemaAndFileset(); // Keytab of the Gravitino server - kerberosHiveContainer + kerberosHive .getContainer() .copyFileFromContainer("/gravitino_server.keytab", TMP_DIR + GRAVITINO_SERVER_KEYTAB); // Keytab of Gravitino server to connector to HDFS - kerberosHiveContainer + kerberosHive .getContainer() .copyFileFromContainer("/etc/admin.keytab", TMP_DIR + HADOOP_CLIENT_KEYTAB); String tmpKrb5Path = TMP_DIR + "krb5.conf_tmp"; String krb5Path = TMP_DIR + "krb5.conf"; - kerberosHiveContainer.getContainer().copyFileFromContainer("/etc/krb5.conf", tmpKrb5Path); + kerberosHive.getContainer().copyFileFromContainer("/etc/krb5.conf", tmpKrb5Path); // Modify the krb5.conf and change the kdc and admin_server to the container IP - String ip = containerSuite.getKerberosHiveContainer().getContainerIpAddress(); + String ip = kerberosHive.getContainerIpAddress(); String content = FileUtils.readFileToString(new File(tmpKrb5Path), StandardCharsets.UTF_8); content = content.replace("kdc = localhost:88", "kdc = " + ip + ":88"); content = content.replace("admin_server = localhost", "admin_server = " + ip + ":749"); @@ -194,11 +195,11 @@ private static void createKeyTableForSchemaAndFileset() throws IOException { HADOOP_SCHEMA_PRINCIPAL, REALM), StandardCharsets.UTF_8); - kerberosHiveContainer + kerberosHive .getContainer() .copyFileToContainer( MountableFile.forHostPath(TMP_DIR + createSchemaShellFile), createSchemaShellFile); - kerberosHiveContainer.executeInContainer("bash", createSchemaShellFile); + kerberosHive.executeInContainer("bash", createSchemaShellFile); FileUtils.writeStringToFile( Paths.get(TMP_DIR + createFileSetShellFile).toFile(), @@ -212,16 +213,16 @@ private static void createKeyTableForSchemaAndFileset() throws IOException { HADOOP_FILESET_PRINCIPAL, REALM), StandardCharsets.UTF_8); - kerberosHiveContainer + kerberosHive .getContainer() .copyFileToContainer( MountableFile.forHostPath(TMP_DIR + createFileSetShellFile), createFileSetShellFile); - kerberosHiveContainer.executeInContainer("bash", createFileSetShellFile); + kerberosHive.executeInContainer("bash", createFileSetShellFile); - kerberosHiveContainer + kerberosHive .getContainer() .copyFileFromContainer(HADOOP_SCHEMA_KEYTAB, TMP_DIR + HADOOP_SCHEMA_KEYTAB); - kerberosHiveContainer + kerberosHive .getContainer() .copyFileFromContainer(HADOOP_FILESET_KEYTAB, TMP_DIR + HADOOP_FILESET_KEYTAB); } diff --git a/catalogs/catalog-lakehouse-hudi/src/test/java/org/apache/gravitino/catalog/lakehouse/hudi/integration/test/HudiCatalogHMSIT.java b/catalogs/catalog-lakehouse-hudi/src/test/java/org/apache/gravitino/catalog/lakehouse/hudi/integration/test/HudiCatalogHMSIT.java index e2cf741fb19..dbb7ac8691b 100644 --- a/catalogs/catalog-lakehouse-hudi/src/test/java/org/apache/gravitino/catalog/lakehouse/hudi/integration/test/HudiCatalogHMSIT.java +++ b/catalogs/catalog-lakehouse-hudi/src/test/java/org/apache/gravitino/catalog/lakehouse/hudi/integration/test/HudiCatalogHMSIT.java @@ -38,8 +38,9 @@ import org.apache.gravitino.SupportsSchemas; import org.apache.gravitino.client.GravitinoMetalake; import org.apache.gravitino.dto.rel.ColumnDTO; -import org.apache.gravitino.integration.test.container.ContainerSuite; +import org.apache.gravitino.integration.test.container.FixtureContainer; import org.apache.gravitino.integration.test.container.HiveContainer; +import org.apache.gravitino.integration.test.container.HiveFixture; import org.apache.gravitino.integration.test.util.BaseIT; import org.apache.gravitino.rel.Column; import org.apache.gravitino.rel.Table; @@ -60,8 +61,10 @@ import org.junit.jupiter.api.Test; @Tag("gravitino-docker-test") +@HiveFixture public class HudiCatalogHMSIT extends BaseIT { - private static final ContainerSuite containerSuite = ContainerSuite.getInstance(); + // Static because the static createHudiTables() reads it. + @FixtureContainer private static HiveContainer hive; private static String hmsURI; private static SparkSession sparkSession; @@ -77,12 +80,9 @@ public class HudiCatalogHMSIT extends BaseIT { @BeforeAll public void prepare() { - containerSuite.startHiveContainer(); hmsURI = String.format( - "thrift://%s:%d", - containerSuite.getHiveContainer().getContainerIpAddress(), - HiveContainer.HIVE_METASTORE_PORT); + "thrift://%s:%d", hive.getContainerIpAddress(), HiveContainer.HIVE_METASTORE_PORT); createHudiTables(); @@ -464,8 +464,7 @@ private static void createHudiTables() { "spark.sql.warehouse.dir", String.format( "hdfs://%s:%d/user/hive/warehouse-catalog-hudi", - containerSuite.getHiveContainer().getContainerIpAddress(), - HiveContainer.HDFS_DEFAULTFS_PORT)) + hive.getContainerIpAddress(), HiveContainer.HDFS_DEFAULTFS_PORT)) .config("spark.serializer", "org.apache.spark.serializer.KryoSerializer") .config("spark.sql.extensions", "org.apache.spark.sql.hudi.HoodieSparkSessionExtension") .config( diff --git a/catalogs/catalog-lakehouse-hudi/src/test/java/org/apache/gravitino/catalog/lakehouse/hudi/integration/test/HudiCatalogKerberosHiveIT.java b/catalogs/catalog-lakehouse-hudi/src/test/java/org/apache/gravitino/catalog/lakehouse/hudi/integration/test/HudiCatalogKerberosHiveIT.java index 6c6da59f2ed..73fd05504ad 100644 --- a/catalogs/catalog-lakehouse-hudi/src/test/java/org/apache/gravitino/catalog/lakehouse/hudi/integration/test/HudiCatalogKerberosHiveIT.java +++ b/catalogs/catalog-lakehouse-hudi/src/test/java/org/apache/gravitino/catalog/lakehouse/hudi/integration/test/HudiCatalogKerberosHiveIT.java @@ -37,8 +37,10 @@ import org.apache.gravitino.exceptions.NoSuchTableException; import org.apache.gravitino.hive.kerberos.AuthenticationConfig; import org.apache.gravitino.hive.kerberos.KerberosConfig; -import org.apache.gravitino.integration.test.container.ContainerSuite; +import org.apache.gravitino.integration.test.container.FixtureContainer; +import org.apache.gravitino.integration.test.container.HiveAuth; import org.apache.gravitino.integration.test.container.HiveContainer; +import org.apache.gravitino.integration.test.container.HiveFixture; import org.apache.gravitino.integration.test.util.BaseIT; import org.apache.gravitino.integration.test.util.GravitinoITUtils; import org.apache.gravitino.rel.TableCatalog; @@ -53,11 +55,10 @@ import org.slf4j.LoggerFactory; @Tag("gravitino-docker-test") +@HiveFixture(auth = HiveAuth.KERBEROS) public class HudiCatalogKerberosHiveIT extends BaseIT { private static final Logger LOG = LoggerFactory.getLogger(HudiCatalogKerberosHiveIT.class); - private static final ContainerSuite containerSuite = ContainerSuite.getInstance(); - private static final String SDK_KERBEROS_PRINCIPAL_KEY = "client.kerberos.principal"; private static final String SDK_KERBEROS_KEYTAB_KEY = "client.kerberos.keytab"; @@ -76,7 +77,7 @@ public class HudiCatalogKerberosHiveIT extends BaseIT { private static GravitinoAdminClient adminClient; - protected static HiveContainer kerberosHiveContainer; + @FixtureContainer protected static HiveContainer kerberosHive; static String METALAKE_NAME = GravitinoITUtils.genRandomName("test_metalake"); static String CATALOG_NAME = GravitinoITUtils.genRandomName("test_catalog"); @@ -85,9 +86,6 @@ public class HudiCatalogKerberosHiveIT extends BaseIT { @BeforeAll public void startIntegrationTest() throws Exception { - containerSuite.startKerberosHiveContainer(); - kerberosHiveContainer = containerSuite.getKerberosHiveContainer(); - File baseDir = new File(System.getProperty("java.io.tmpdir")); File file = Files.createTempDirectory(baseDir.toPath(), "test").toFile(); file.deleteOnExit(); @@ -96,7 +94,7 @@ public void startIntegrationTest() throws Exception { HIVE_METASTORE_URI = String.format( "thrift://%s:%d", - kerberosHiveContainer.getContainerIpAddress(), HiveContainer.HIVE_METASTORE_PORT); + kerberosHive.getContainerIpAddress(), HiveContainer.HIVE_METASTORE_PORT); // Prepare kerberos related-config; prepareKerberosConfig(); @@ -123,26 +121,26 @@ public void stop() { private static void prepareKerberosConfig() throws Exception { // Keytab of the Gravitino SDK client - kerberosHiveContainer + kerberosHive .getContainer() .copyFileFromContainer("/gravitino_client.keytab", TMP_DIR + GRAVITINO_CLIENT_KEYTAB); // Keytab of the Gravitino server - kerberosHiveContainer + kerberosHive .getContainer() .copyFileFromContainer("/gravitino_server.keytab", TMP_DIR + GRAVITINO_SERVER_KEYTAB); // Keytab of Gravitino server to connector to Hive - kerberosHiveContainer + kerberosHive .getContainer() .copyFileFromContainer("/etc/admin.keytab", TMP_DIR + HIVE_METASTORE_CLIENT_KEYTAB); String tmpKrb5Path = TMP_DIR + "/krb5.conf_tmp"; String krb5Path = TMP_DIR + "/krb5.conf"; - kerberosHiveContainer.getContainer().copyFileFromContainer("/etc/krb5.conf", tmpKrb5Path); + kerberosHive.getContainer().copyFileFromContainer("/etc/krb5.conf", tmpKrb5Path); // Modify the krb5.conf and change the kdc and admin_server to the container IP - String ip = containerSuite.getKerberosHiveContainer().getContainerIpAddress(); + String ip = kerberosHive.getContainerIpAddress(); String content = FileUtils.readFileToString(new File(tmpKrb5Path), java.nio.charset.StandardCharsets.UTF_8); content = content.replace("kdc = localhost:88", "kdc = " + ip + ":88"); @@ -203,7 +201,7 @@ public void testHudiCatalogWithKerberos() { properties.put("list-all-tables", "false"); properties.put( CATALOG_BYPASS_PREFIX + "hive.metastore.kerberos.principal", - "hive/_HOST@HADOOPKRB".replace("_HOST", kerberosHiveContainer.getHostName())); + "hive/_HOST@HADOOPKRB".replace("_HOST", kerberosHive.getHostName())); properties.put(CATALOG_BYPASS_PREFIX + "hive.metastore.sasl.enabled", "true"); properties.put(CATALOG_BYPASS_PREFIX + "hadoop.security.authentication", "kerberos"); @@ -213,7 +211,7 @@ public void testHudiCatalogWithKerberos() { "warehouse", String.format( "hdfs://%s:%d/user/hive/warehouse-catalog-hudi", - kerberosHiveContainer.getContainerIpAddress(), HiveContainer.HDFS_DEFAULTFS_PORT)); + kerberosHive.getContainerIpAddress(), HiveContainer.HDFS_DEFAULTFS_PORT)); Catalog catalog = gravitinoMetalake.createCatalog( CATALOG_NAME, Catalog.Type.RELATIONAL, "lakehouse-hudi", "comment", properties); diff --git a/catalogs/catalog-lakehouse-iceberg/src/test/java/org/apache/gravitino/catalog/lakehouse/iceberg/integration/test/CatalogIcebergBaseIT.java b/catalogs/catalog-lakehouse-iceberg/src/test/java/org/apache/gravitino/catalog/lakehouse/iceberg/integration/test/CatalogIcebergBaseIT.java index e7daf4f57b8..6d91fbbb808 100644 --- a/catalogs/catalog-lakehouse-iceberg/src/test/java/org/apache/gravitino/catalog/lakehouse/iceberg/integration/test/CatalogIcebergBaseIT.java +++ b/catalogs/catalog-lakehouse-iceberg/src/test/java/org/apache/gravitino/catalog/lakehouse/iceberg/integration/test/CatalogIcebergBaseIT.java @@ -61,7 +61,9 @@ import org.apache.gravitino.exceptions.ViewAlreadyExistsException; import org.apache.gravitino.iceberg.common.IcebergConfig; import org.apache.gravitino.iceberg.common.utils.IcebergCatalogUtil; -import org.apache.gravitino.integration.test.container.ContainerSuite; +import org.apache.gravitino.integration.test.container.FixtureContainer; +import org.apache.gravitino.integration.test.container.HiveContainer; +import org.apache.gravitino.integration.test.container.HiveFixture; import org.apache.gravitino.integration.test.util.BaseIT; import org.apache.gravitino.integration.test.util.GravitinoITUtils; import org.apache.gravitino.rel.Column; @@ -103,9 +105,12 @@ import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; +@HiveFixture public abstract class CatalogIcebergBaseIT extends BaseIT { - protected static final ContainerSuite containerSuite = ContainerSuite.getInstance(); + /** The Hive server whose metastore and HDFS the subclasses point the catalog at. */ + @FixtureContainer protected HiveContainer hive; + protected String WAREHOUSE; protected String URIS; protected String TYPE; @@ -139,7 +144,6 @@ public abstract class CatalogIcebergBaseIT extends BaseIT { public void startup() throws Exception { super.ignoreIcebergAuxRestService = false; super.startIntegrationTest(); - containerSuite.startHiveContainer(); initIcebergCatalogProperties(); createMetalake(); createCatalog(); diff --git a/catalogs/catalog-lakehouse-iceberg/src/test/java/org/apache/gravitino/catalog/lakehouse/iceberg/integration/test/CatalogIcebergHiveIT.java b/catalogs/catalog-lakehouse-iceberg/src/test/java/org/apache/gravitino/catalog/lakehouse/iceberg/integration/test/CatalogIcebergHiveIT.java index b78b7c2489a..c15d018dda2 100644 --- a/catalogs/catalog-lakehouse-iceberg/src/test/java/org/apache/gravitino/catalog/lakehouse/iceberg/integration/test/CatalogIcebergHiveIT.java +++ b/catalogs/catalog-lakehouse-iceberg/src/test/java/org/apache/gravitino/catalog/lakehouse/iceberg/integration/test/CatalogIcebergHiveIT.java @@ -36,15 +36,12 @@ public class CatalogIcebergHiveIT extends CatalogIcebergBaseIT { protected void initIcebergCatalogProperties() { URIS = String.format( - "thrift://%s:%d", - containerSuite.getHiveContainer().getContainerIpAddress(), - HiveContainer.HIVE_METASTORE_PORT); + "thrift://%s:%d", hive.getContainerIpAddress(), HiveContainer.HIVE_METASTORE_PORT); TYPE = "hive"; WAREHOUSE = String.format( "hdfs://%s:%d/user/hive/warehouse-catalog-iceberg/", - containerSuite.getHiveContainer().getContainerIpAddress(), - HiveContainer.HDFS_DEFAULTFS_PORT); + hive.getContainerIpAddress(), HiveContainer.HDFS_DEFAULTFS_PORT); } @Test diff --git a/catalogs/catalog-lakehouse-iceberg/src/test/java/org/apache/gravitino/catalog/lakehouse/iceberg/integration/test/CatalogIcebergKerberosHiveIT.java b/catalogs/catalog-lakehouse-iceberg/src/test/java/org/apache/gravitino/catalog/lakehouse/iceberg/integration/test/CatalogIcebergKerberosHiveIT.java index 21f41c4d1d4..8727d44d444 100644 --- a/catalogs/catalog-lakehouse-iceberg/src/test/java/org/apache/gravitino/catalog/lakehouse/iceberg/integration/test/CatalogIcebergKerberosHiveIT.java +++ b/catalogs/catalog-lakehouse-iceberg/src/test/java/org/apache/gravitino/catalog/lakehouse/iceberg/integration/test/CatalogIcebergKerberosHiveIT.java @@ -41,8 +41,10 @@ import org.apache.gravitino.client.GravitinoMetalake; import org.apache.gravitino.client.KerberosTokenProvider; import org.apache.gravitino.iceberg.common.IcebergConfig; -import org.apache.gravitino.integration.test.container.ContainerSuite; +import org.apache.gravitino.integration.test.container.FixtureContainer; +import org.apache.gravitino.integration.test.container.HiveAuth; import org.apache.gravitino.integration.test.container.HiveContainer; +import org.apache.gravitino.integration.test.container.HiveFixture; import org.apache.gravitino.integration.test.util.BaseIT; import org.apache.gravitino.integration.test.util.GravitinoITUtils; import org.apache.gravitino.integration.test.util.KerberosHdfs; @@ -64,12 +66,11 @@ import org.slf4j.LoggerFactory; @Tag("gravitino-docker-test") +@HiveFixture(auth = HiveAuth.KERBEROS) public class CatalogIcebergKerberosHiveIT extends BaseIT { private static final Logger LOG = LoggerFactory.getLogger(CatalogIcebergKerberosHiveIT.class); - private static final ContainerSuite containerSuite = ContainerSuite.getInstance(); - private static final String SDK_KERBEROS_PRINCIPAL_KEY = "client.kerberos.principal"; private static final String SDK_KERBEROS_KEYTAB_KEY = "client.kerberos.keytab"; @@ -84,7 +85,7 @@ public class CatalogIcebergKerberosHiveIT extends BaseIT { private static String TMP_DIR; - private static HiveContainer kerberosHiveContainer; + @FixtureContainer private static HiveContainer kerberosHive; private static GravitinoAdminClient adminClient; @@ -105,18 +106,15 @@ public class CatalogIcebergKerberosHiveIT extends BaseIT { @BeforeAll public void startIntegrationTest() { - containerSuite.startKerberosHiveContainer(); - kerberosHiveContainer = containerSuite.getKerberosHiveContainer(); - URIS = String.format( "thrift://%s:%d", - kerberosHiveContainer.getContainerIpAddress(), HiveContainer.HIVE_METASTORE_PORT); + kerberosHive.getContainerIpAddress(), HiveContainer.HIVE_METASTORE_PORT); TYPE = "hive"; WAREHOUSE = String.format( "hdfs://%s:%d/user/hive/warehouse-catalog-iceberg/", - kerberosHiveContainer.getContainerIpAddress(), HiveContainer.HDFS_DEFAULTFS_PORT); + kerberosHive.getContainerIpAddress(), HiveContainer.HDFS_DEFAULTFS_PORT); try { File baseDir = new File(System.getProperty("java.io.tmpdir")); @@ -126,7 +124,7 @@ public void startIntegrationTest() { // Prepare kerberos related-config; prepareKerberosConfig(); - hdfs = KerberosHdfs.of(kerberosHiveContainer, TMP_DIR + HIVE_METASTORE_CLIENT_KEYTAB); + hdfs = KerberosHdfs.of(kerberosHive, TMP_DIR + HIVE_METASTORE_CLIENT_KEYTAB); // Config kerberos configuration for Gravitino server addKerberosConfig(); @@ -154,26 +152,26 @@ public void stop() { private static void prepareKerberosConfig() throws Exception { // Keytab of the Gravitino SDK client - kerberosHiveContainer + kerberosHive .getContainer() .copyFileFromContainer("/gravitino_client.keytab", TMP_DIR + GRAVITINO_CLIENT_KEYTAB); // Keytab of the Gravitino server - kerberosHiveContainer + kerberosHive .getContainer() .copyFileFromContainer("/gravitino_server.keytab", TMP_DIR + GRAVITINO_SERVER_KEYTAB); // Keytab of Gravitino server to connector to Hive - kerberosHiveContainer + kerberosHive .getContainer() .copyFileFromContainer("/etc/admin.keytab", TMP_DIR + HIVE_METASTORE_CLIENT_KEYTAB); String tmpKrb5Path = TMP_DIR + "/krb5.conf_tmp"; String krb5Path = TMP_DIR + "/krb5.conf"; - kerberosHiveContainer.getContainer().copyFileFromContainer("/etc/krb5.conf", tmpKrb5Path); + kerberosHive.getContainer().copyFileFromContainer("/etc/krb5.conf", tmpKrb5Path); // Modify the krb5.conf and change the kdc and admin_server to the container IP - String ip = containerSuite.getKerberosHiveContainer().getContainerIpAddress(); + String ip = kerberosHive.getContainerIpAddress(); String content = FileUtils.readFileToString(new File(tmpKrb5Path), StandardCharsets.UTF_8); content = content.replace("kdc = localhost:88", "kdc = " + ip + ":88"); content = content.replace("admin_server = localhost", "admin_server = " + ip + ":749"); @@ -234,8 +232,7 @@ void testIcebergWithKerberosAndUserImpersonation() throws IOException { properties.put(PRINCIPAL_KEY, HIVE_METASTORE_CLIENT_PRINCIPAL); properties.put( CATALOG_BYPASS_PREFIX + "hive.metastore.kerberos.principal", - "hive/_HOST@HADOOPKRB" - .replace("_HOST", containerSuite.getKerberosHiveContainer().getHostName())); + "hive/_HOST@HADOOPKRB".replace("_HOST", kerberosHive.getHostName())); properties.put(CATALOG_BYPASS_PREFIX + "hive.metastore.sasl.enabled", "true"); properties.put(IcebergConfig.CATALOG_BACKEND.getKey(), TYPE); @@ -345,8 +342,7 @@ void testIcebergWithKerberos() { properties.put(PRINCIPAL_KEY, HIVE_METASTORE_CLIENT_PRINCIPAL); properties.put( CATALOG_BYPASS_PREFIX + "hive.metastore.kerberos.principal", - "hive/_HOST@HADOOPKRB" - .replace("_HOST", containerSuite.getKerberosHiveContainer().getHostName())); + "hive/_HOST@HADOOPKRB".replace("_HOST", kerberosHive.getHostName())); properties.put(CATALOG_BYPASS_PREFIX + "hive.metastore.sasl.enabled", "true"); properties.put(IcebergConfig.CATALOG_BACKEND.getKey(), TYPE); diff --git a/catalogs/catalog-lakehouse-iceberg/src/test/java/org/apache/gravitino/catalog/lakehouse/iceberg/integration/test/CatalogIcebergRestIT.java b/catalogs/catalog-lakehouse-iceberg/src/test/java/org/apache/gravitino/catalog/lakehouse/iceberg/integration/test/CatalogIcebergRestIT.java index f6cb49554b6..4b96e8f28d9 100644 --- a/catalogs/catalog-lakehouse-iceberg/src/test/java/org/apache/gravitino/catalog/lakehouse/iceberg/integration/test/CatalogIcebergRestIT.java +++ b/catalogs/catalog-lakehouse-iceberg/src/test/java/org/apache/gravitino/catalog/lakehouse/iceberg/integration/test/CatalogIcebergRestIT.java @@ -71,8 +71,7 @@ protected void initIcebergCatalogProperties() { WAREHOUSE = String.format( "hdfs://%s:%d/user/hive/warehouse-catalog-iceberg/", - containerSuite.getHiveContainer().getContainerIpAddress(), - HiveContainer.HDFS_DEFAULTFS_PORT); + hive.getContainerIpAddress(), HiveContainer.HDFS_DEFAULTFS_PORT); } // Checks an Iceberg REST warehouse misconfiguration is caught on create. See issue #11943. diff --git a/catalogs/catalog-lakehouse-paimon/src/test/java/org/apache/gravitino/catalog/lakehouse/paimon/integration/test/CatalogPaimonBaseIT.java b/catalogs/catalog-lakehouse-paimon/src/test/java/org/apache/gravitino/catalog/lakehouse/paimon/integration/test/CatalogPaimonBaseIT.java index e94faf01714..60f25d42723 100644 --- a/catalogs/catalog-lakehouse-paimon/src/test/java/org/apache/gravitino/catalog/lakehouse/paimon/integration/test/CatalogPaimonBaseIT.java +++ b/catalogs/catalog-lakehouse-paimon/src/test/java/org/apache/gravitino/catalog/lakehouse/paimon/integration/test/CatalogPaimonBaseIT.java @@ -51,7 +51,6 @@ import org.apache.gravitino.exceptions.NoSuchSchemaException; import org.apache.gravitino.exceptions.SchemaAlreadyExistsException; import org.apache.gravitino.exceptions.TableAlreadyExistsException; -import org.apache.gravitino.integration.test.container.ContainerSuite; import org.apache.gravitino.integration.test.container.MySQLContainer; import org.apache.gravitino.integration.test.util.BaseIT; import org.apache.gravitino.integration.test.util.GravitinoITUtils; @@ -93,7 +92,6 @@ public abstract class CatalogPaimonBaseIT extends BaseIT { - protected static final ContainerSuite containerSuite = ContainerSuite.getInstance(); protected static final TestDatabaseName TEST_DB_NAME = TestDatabaseName.PG_TEST_PAIMON_CATALOG_MULTIPLE_JDBC_LOAD; protected static MySQLContainer mySQLContainer; @@ -135,9 +133,9 @@ public void startup() { initSparkEnv(); } - protected void startNecessaryContainers() { - containerSuite.startHiveContainer(); - } + // Subclasses declare their containers as fixtures; this hook lets one prepare them, such as + // creating a bucket, before the catalog is created. + protected void startNecessaryContainers() {} @AfterAll public void stop() { diff --git a/catalogs/catalog-lakehouse-paimon/src/test/java/org/apache/gravitino/catalog/lakehouse/paimon/integration/test/CatalogPaimonFileSystemIT.java b/catalogs/catalog-lakehouse-paimon/src/test/java/org/apache/gravitino/catalog/lakehouse/paimon/integration/test/CatalogPaimonFileSystemIT.java index e699f8d396b..2966ab16da3 100644 --- a/catalogs/catalog-lakehouse-paimon/src/test/java/org/apache/gravitino/catalog/lakehouse/paimon/integration/test/CatalogPaimonFileSystemIT.java +++ b/catalogs/catalog-lakehouse-paimon/src/test/java/org/apache/gravitino/catalog/lakehouse/paimon/integration/test/CatalogPaimonFileSystemIT.java @@ -24,7 +24,9 @@ import org.apache.gravitino.Schema; import org.apache.gravitino.SupportsSchemas; import org.apache.gravitino.catalog.lakehouse.paimon.PaimonCatalogPropertiesMetadata; +import org.apache.gravitino.integration.test.container.FixtureContainer; import org.apache.gravitino.integration.test.container.HiveContainer; +import org.apache.gravitino.integration.test.container.HiveFixture; import org.apache.gravitino.integration.test.util.GravitinoITUtils; import org.apache.paimon.catalog.Catalog; import org.junit.jupiter.api.Assertions; @@ -32,8 +34,11 @@ import org.junit.jupiter.api.Test; @Tag("gravitino-docker-test") +@HiveFixture public class CatalogPaimonFileSystemIT extends CatalogPaimonBaseIT { + @FixtureContainer private HiveContainer hive; + @Override protected Map initPaimonCatalogProperties() { @@ -45,8 +50,7 @@ protected Map initPaimonCatalogProperties() { WAREHOUSE = String.format( "hdfs://%s:%d/user/hive/warehouse-catalog-paimon/", - containerSuite.getHiveContainer().getContainerIpAddress(), - HiveContainer.HDFS_DEFAULTFS_PORT); + hive.getContainerIpAddress(), HiveContainer.HDFS_DEFAULTFS_PORT); catalogProperties.put(PaimonCatalogPropertiesMetadata.GRAVITINO_CATALOG_BACKEND, TYPE); catalogProperties.put(PaimonCatalogPropertiesMetadata.WAREHOUSE, WAREHOUSE); diff --git a/catalogs/catalog-lakehouse-paimon/src/test/java/org/apache/gravitino/catalog/lakehouse/paimon/integration/test/CatalogPaimonHiveIT.java b/catalogs/catalog-lakehouse-paimon/src/test/java/org/apache/gravitino/catalog/lakehouse/paimon/integration/test/CatalogPaimonHiveIT.java index f943d971dcc..36f992083b6 100644 --- a/catalogs/catalog-lakehouse-paimon/src/test/java/org/apache/gravitino/catalog/lakehouse/paimon/integration/test/CatalogPaimonHiveIT.java +++ b/catalogs/catalog-lakehouse-paimon/src/test/java/org/apache/gravitino/catalog/lakehouse/paimon/integration/test/CatalogPaimonHiveIT.java @@ -26,7 +26,9 @@ import org.apache.gravitino.Schema; import org.apache.gravitino.SupportsSchemas; import org.apache.gravitino.catalog.lakehouse.paimon.PaimonCatalogPropertiesMetadata; +import org.apache.gravitino.integration.test.container.FixtureContainer; import org.apache.gravitino.integration.test.container.HiveContainer; +import org.apache.gravitino.integration.test.container.HiveFixture; import org.apache.gravitino.integration.test.util.GravitinoITUtils; import org.apache.gravitino.rel.Column; import org.apache.gravitino.rel.Representation; @@ -45,8 +47,11 @@ @Tag("gravitino-docker-test") @TestInstance(TestInstance.Lifecycle.PER_CLASS) +@HiveFixture public class CatalogPaimonHiveIT extends CatalogPaimonBaseIT { + @FixtureContainer private HiveContainer hive; + @Override protected Map initPaimonCatalogProperties() { Map catalogProperties = Maps.newHashMap(); @@ -57,13 +62,10 @@ protected Map initPaimonCatalogProperties() { WAREHOUSE = String.format( "hdfs://%s:%d/user/hive/warehouse-catalog-paimon/", - containerSuite.getHiveContainer().getContainerIpAddress(), - HiveContainer.HDFS_DEFAULTFS_PORT); + hive.getContainerIpAddress(), HiveContainer.HDFS_DEFAULTFS_PORT); URI = String.format( - "thrift://%s:%d", - containerSuite.getHiveContainer().getContainerIpAddress(), - HiveContainer.HIVE_METASTORE_PORT); + "thrift://%s:%d", hive.getContainerIpAddress(), HiveContainer.HIVE_METASTORE_PORT); catalogProperties.put(PaimonCatalogPropertiesMetadata.GRAVITINO_CATALOG_BACKEND, TYPE); catalogProperties.put(PaimonCatalogPropertiesMetadata.WAREHOUSE, WAREHOUSE); diff --git a/catalogs/catalog-lakehouse-paimon/src/test/java/org/apache/gravitino/catalog/lakehouse/paimon/integration/test/CatalogPaimonJdbcIT.java b/catalogs/catalog-lakehouse-paimon/src/test/java/org/apache/gravitino/catalog/lakehouse/paimon/integration/test/CatalogPaimonJdbcIT.java index 3eb14f132f5..ef6a05997a8 100644 --- a/catalogs/catalog-lakehouse-paimon/src/test/java/org/apache/gravitino/catalog/lakehouse/paimon/integration/test/CatalogPaimonJdbcIT.java +++ b/catalogs/catalog-lakehouse-paimon/src/test/java/org/apache/gravitino/catalog/lakehouse/paimon/integration/test/CatalogPaimonJdbcIT.java @@ -23,7 +23,9 @@ import org.apache.gravitino.Catalog; import org.apache.gravitino.catalog.lakehouse.paimon.PaimonCatalogPropertiesMetadata; import org.apache.gravitino.exceptions.ConnectionFailedException; +import org.apache.gravitino.integration.test.container.FixtureContainer; import org.apache.gravitino.integration.test.container.HiveContainer; +import org.apache.gravitino.integration.test.container.HiveFixture; import org.apache.gravitino.integration.test.util.GravitinoITUtils; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Tag; @@ -32,8 +34,11 @@ @Tag("gravitino-docker-test") @TestInstance(TestInstance.Lifecycle.PER_CLASS) +@HiveFixture public class CatalogPaimonJdbcIT extends CatalogPaimonBaseIT { + @FixtureContainer private HiveContainer hive; + @Override protected Map initPaimonCatalogProperties() { containerSuite.startMySQLContainer(TEST_DB_NAME); @@ -47,8 +52,7 @@ protected Map initPaimonCatalogProperties() { WAREHOUSE = String.format( "hdfs://%s:%d/user/hive/warehouse-catalog-paimon/", - containerSuite.getHiveContainer().getContainerIpAddress(), - HiveContainer.HDFS_DEFAULTFS_PORT); + hive.getContainerIpAddress(), HiveContainer.HDFS_DEFAULTFS_PORT); URI = mySQLContainer.getJdbcUrl(TEST_DB_NAME); jdbcUser = mySQLContainer.getUsername(); jdbcPassword = mySQLContainer.getPassword(); diff --git a/catalogs/catalog-lakehouse-paimon/src/test/java/org/apache/gravitino/catalog/lakehouse/paimon/integration/test/CatalogPaimonKerberosFilesystemIT.java b/catalogs/catalog-lakehouse-paimon/src/test/java/org/apache/gravitino/catalog/lakehouse/paimon/integration/test/CatalogPaimonKerberosFilesystemIT.java index 16f9a94abc9..a08cc424431 100644 --- a/catalogs/catalog-lakehouse-paimon/src/test/java/org/apache/gravitino/catalog/lakehouse/paimon/integration/test/CatalogPaimonKerberosFilesystemIT.java +++ b/catalogs/catalog-lakehouse-paimon/src/test/java/org/apache/gravitino/catalog/lakehouse/paimon/integration/test/CatalogPaimonKerberosFilesystemIT.java @@ -38,8 +38,10 @@ import org.apache.gravitino.client.GravitinoAdminClient; import org.apache.gravitino.client.GravitinoMetalake; import org.apache.gravitino.client.KerberosTokenProvider; -import org.apache.gravitino.integration.test.container.ContainerSuite; +import org.apache.gravitino.integration.test.container.FixtureContainer; +import org.apache.gravitino.integration.test.container.HiveAuth; import org.apache.gravitino.integration.test.container.HiveContainer; +import org.apache.gravitino.integration.test.container.HiveFixture; import org.apache.gravitino.integration.test.util.BaseIT; import org.apache.gravitino.integration.test.util.GravitinoITUtils; import org.apache.gravitino.integration.test.util.KerberosHdfs; @@ -61,13 +63,12 @@ import org.slf4j.LoggerFactory; @Tag("gravitino-docker-test") +@HiveFixture(auth = HiveAuth.KERBEROS) public class CatalogPaimonKerberosFilesystemIT extends BaseIT { private static final Logger LOG = LoggerFactory.getLogger(CatalogPaimonKerberosFilesystemIT.class); - private static final ContainerSuite containerSuite = ContainerSuite.getInstance(); - private static final String SDK_KERBEROS_PRINCIPAL_KEY = "client.kerberos.principal"; private static final String SDK_KERBEROS_KEYTAB_KEY = "client.kerberos.keytab"; @@ -82,7 +83,7 @@ public class CatalogPaimonKerberosFilesystemIT extends BaseIT { private static String TMP_DIR; - private static HiveContainer kerberosHiveContainer; + @FixtureContainer private static HiveContainer kerberosHive; private static GravitinoAdminClient adminClient; @@ -101,14 +102,11 @@ public class CatalogPaimonKerberosFilesystemIT extends BaseIT { @BeforeAll public void startIntegrationTest() { - containerSuite.startKerberosHiveContainer(); - kerberosHiveContainer = containerSuite.getKerberosHiveContainer(); - TYPE = "filesystem"; WAREHOUSE = String.format( "hdfs://%s:%d/user/hive/paimon_catalog_warehouse/", - kerberosHiveContainer.getContainerIpAddress(), HiveContainer.HDFS_DEFAULTFS_PORT); + kerberosHive.getContainerIpAddress(), HiveContainer.HDFS_DEFAULTFS_PORT); try { File baseDir = new File(System.getProperty("java.io.tmpdir")); @@ -118,7 +116,7 @@ public void startIntegrationTest() { // Prepare kerberos related-config; prepareKerberosConfig(); - hdfs = KerberosHdfs.of(kerberosHiveContainer, TMP_DIR + HDFS_CLIENT_KEYTAB); + hdfs = KerberosHdfs.of(kerberosHive, TMP_DIR + HDFS_CLIENT_KEYTAB); // Config kerberos configuration for Gravitino server addKerberosConfig(); @@ -145,26 +143,26 @@ public void stop() { private static void prepareKerberosConfig() throws Exception { // Keytab of the Gravitino SDK client - kerberosHiveContainer + kerberosHive .getContainer() .copyFileFromContainer("/gravitino_client.keytab", TMP_DIR + GRAVITINO_CLIENT_KEYTAB); // Keytab of the Gravitino server - kerberosHiveContainer + kerberosHive .getContainer() .copyFileFromContainer("/gravitino_server.keytab", TMP_DIR + GRAVITINO_SERVER_KEYTAB); // Keytab of Gravitino server to connector to Hive - kerberosHiveContainer + kerberosHive .getContainer() .copyFileFromContainer("/etc/admin.keytab", TMP_DIR + HDFS_CLIENT_KEYTAB); String tmpKrb5Path = TMP_DIR + "/krb5.conf_tmp"; String krb5Path = TMP_DIR + "/krb5.conf"; - kerberosHiveContainer.getContainer().copyFileFromContainer("/etc/krb5.conf", tmpKrb5Path); + kerberosHive.getContainer().copyFileFromContainer("/etc/krb5.conf", tmpKrb5Path); // Modify the krb5.conf and change the kdc and admin_server to the container IP - String ip = containerSuite.getKerberosHiveContainer().getContainerIpAddress(); + String ip = kerberosHive.getContainerIpAddress(); String content = FileUtils.readFileToString(new File(tmpKrb5Path), StandardCharsets.UTF_8); content = content.replace("kdc = localhost:88", "kdc = " + ip + ":88"); content = content.replace("admin_server = localhost", "admin_server = " + ip + ":749"); diff --git a/catalogs/catalog-lakehouse-paimon/src/test/java/org/apache/gravitino/catalog/lakehouse/paimon/utils/TestCatalogUtils.java b/catalogs/catalog-lakehouse-paimon/src/test/java/org/apache/gravitino/catalog/lakehouse/paimon/utils/TestCatalogUtils.java index 0554c482d03..727ac2a32d3 100644 --- a/catalogs/catalog-lakehouse-paimon/src/test/java/org/apache/gravitino/catalog/lakehouse/paimon/utils/TestCatalogUtils.java +++ b/catalogs/catalog-lakehouse-paimon/src/test/java/org/apache/gravitino/catalog/lakehouse/paimon/utils/TestCatalogUtils.java @@ -29,8 +29,9 @@ import java.util.function.Consumer; import org.apache.gravitino.catalog.lakehouse.paimon.PaimonCatalogBackend; import org.apache.gravitino.catalog.lakehouse.paimon.PaimonConfig; -import org.apache.gravitino.integration.test.container.ContainerSuite; +import org.apache.gravitino.integration.test.container.FixtureContainer; import org.apache.gravitino.integration.test.container.HiveContainer; +import org.apache.gravitino.integration.test.container.HiveFixture; import org.apache.paimon.catalog.Catalog; import org.apache.paimon.catalog.FileSystemCatalog; import org.apache.paimon.factories.FactoryException; @@ -41,8 +42,11 @@ /** Tests for {@link org.apache.gravitino.catalog.lakehouse.paimon.utils.CatalogUtils}. */ @Tag("gravitino-docker-test") +@HiveFixture public class TestCatalogUtils { + @FixtureContainer private static HiveContainer hive; + @Test void testLoadCatalogBackend() throws Exception { // Test load FileSystemCatalog for filesystem metastore. @@ -92,13 +96,9 @@ private static String generateUri(String metastore) { if (PaimonCatalogBackend.JDBC.name().equalsIgnoreCase(metastore)) { uri = "jdbc:h2:mem:testdb"; } else if (PaimonCatalogBackend.HIVE.name().equalsIgnoreCase(metastore)) { - ContainerSuite containerSuite = ContainerSuite.getInstance(); - containerSuite.startHiveContainer(); uri = String.format( - "thrift://%s:%d", - containerSuite.getHiveContainer().getContainerIpAddress(), - HiveContainer.HIVE_METASTORE_PORT); + "thrift://%s:%d", hive.getContainerIpAddress(), HiveContainer.HIVE_METASTORE_PORT); } return uri; } diff --git a/clients/client-java/src/test/java/org/apache/gravitino/client/integration/test/CatalogIT.java b/clients/client-java/src/test/java/org/apache/gravitino/client/integration/test/CatalogIT.java index 21e46b1ebea..056a83078bf 100644 --- a/clients/client-java/src/test/java/org/apache/gravitino/client/integration/test/CatalogIT.java +++ b/clients/client-java/src/test/java/org/apache/gravitino/client/integration/test/CatalogIT.java @@ -44,8 +44,9 @@ import org.apache.gravitino.file.Fileset; import org.apache.gravitino.file.FilesetCatalog; import org.apache.gravitino.file.FilesetChange; -import org.apache.gravitino.integration.test.container.ContainerSuite; +import org.apache.gravitino.integration.test.container.FixtureContainer; import org.apache.gravitino.integration.test.container.HiveContainer; +import org.apache.gravitino.integration.test.container.HiveFixture; import org.apache.gravitino.integration.test.util.BaseIT; import org.apache.gravitino.integration.test.util.GravitinoITUtils; import org.junit.jupiter.api.AfterAll; @@ -58,11 +59,12 @@ import org.slf4j.LoggerFactory; @Tag("gravitino-docker-test") +@HiveFixture public class CatalogIT extends BaseIT { - private static final Logger LOG = LoggerFactory.getLogger(CatalogIT.class); + @FixtureContainer private HiveContainer hive; - private static final ContainerSuite containerSuite = ContainerSuite.getInstance(); + private static final Logger LOG = LoggerFactory.getLogger(CatalogIT.class); private static final String metalakeName = GravitinoITUtils.genRandomName("catalog_it_metalake"); @@ -72,12 +74,9 @@ public class CatalogIT extends BaseIT { @BeforeAll public void startUp() { - containerSuite.startHiveContainer(); hmsUri = String.format( - "thrift://%s:%d", - containerSuite.getHiveContainer().getContainerIpAddress(), - HiveContainer.HIVE_METASTORE_PORT); + "thrift://%s:%d", hive.getContainerIpAddress(), HiveContainer.HIVE_METASTORE_PORT); Assertions.assertFalse(client.metalakeExists(metalakeName)); metalake = client.createMetalake(metalakeName, "metalake comment", Collections.emptyMap()); diff --git a/clients/client-java/src/test/java/org/apache/gravitino/client/integration/test/FunctionIT.java b/clients/client-java/src/test/java/org/apache/gravitino/client/integration/test/FunctionIT.java index 3bd3599f2c9..f7536008177 100644 --- a/clients/client-java/src/test/java/org/apache/gravitino/client/integration/test/FunctionIT.java +++ b/clients/client-java/src/test/java/org/apache/gravitino/client/integration/test/FunctionIT.java @@ -42,8 +42,9 @@ import org.apache.gravitino.function.FunctionParam; import org.apache.gravitino.function.FunctionParams; import org.apache.gravitino.function.FunctionType; -import org.apache.gravitino.integration.test.container.ContainerSuite; +import org.apache.gravitino.integration.test.container.FixtureContainer; import org.apache.gravitino.integration.test.container.HiveContainer; +import org.apache.gravitino.integration.test.container.HiveFixture; import org.apache.gravitino.integration.test.util.BaseIT; import org.apache.gravitino.integration.test.util.GravitinoITUtils; import org.apache.gravitino.rel.expressions.literals.Literals; @@ -57,9 +58,10 @@ import org.junit.jupiter.api.Test; @Tag("gravitino-docker-test") +@HiveFixture public class FunctionIT extends BaseIT { - private static final ContainerSuite containerSuite = ContainerSuite.getInstance(); + @FixtureContainer private HiveContainer hive; private static final String METALAKE_NAME = GravitinoITUtils.genRandomName("function_it_metalake"); @@ -73,12 +75,9 @@ public class FunctionIT extends BaseIT { @BeforeAll public void setUp() { - containerSuite.startHiveContainer(); hmsUri = String.format( - "thrift://%s:%d", - containerSuite.getHiveContainer().getContainerIpAddress(), - HiveContainer.HIVE_METASTORE_PORT); + "thrift://%s:%d", hive.getContainerIpAddress(), HiveContainer.HIVE_METASTORE_PORT); metalake = client.createMetalake(METALAKE_NAME, "metalake comment", Collections.emptyMap()); } diff --git a/clients/client-java/src/test/java/org/apache/gravitino/client/integration/test/PolicyIT.java b/clients/client-java/src/test/java/org/apache/gravitino/client/integration/test/PolicyIT.java index 37af4610562..02074d93103 100644 --- a/clients/client-java/src/test/java/org/apache/gravitino/client/integration/test/PolicyIT.java +++ b/clients/client-java/src/test/java/org/apache/gravitino/client/integration/test/PolicyIT.java @@ -38,8 +38,9 @@ import org.apache.gravitino.function.FunctionParam; import org.apache.gravitino.function.FunctionParams; import org.apache.gravitino.function.FunctionType; -import org.apache.gravitino.integration.test.container.ContainerSuite; +import org.apache.gravitino.integration.test.container.FixtureContainer; import org.apache.gravitino.integration.test.container.HiveContainer; +import org.apache.gravitino.integration.test.container.HiveFixture; import org.apache.gravitino.integration.test.util.BaseIT; import org.apache.gravitino.integration.test.util.GravitinoITUtils; import org.apache.gravitino.model.Model; @@ -61,9 +62,10 @@ import org.junit.jupiter.api.Test; @Tag("gravitino-docker-test") +@HiveFixture public class PolicyIT extends BaseIT { - private static final ContainerSuite containerSuite = ContainerSuite.getInstance(); + @FixtureContainer private HiveContainer hive; private static final String metalakeName = GravitinoITUtils.genRandomName("policy_it_metalake"); @@ -80,12 +82,9 @@ public class PolicyIT extends BaseIT { @BeforeAll public void setUp() { - containerSuite.startHiveContainer(); String hmsUri = String.format( - "thrift://%s:%d", - containerSuite.getHiveContainer().getContainerIpAddress(), - HiveContainer.HIVE_METASTORE_PORT); + "thrift://%s:%d", hive.getContainerIpAddress(), HiveContainer.HIVE_METASTORE_PORT); // Create metalake Assertions.assertFalse(client.metalakeExists(metalakeName)); diff --git a/clients/client-java/src/test/java/org/apache/gravitino/client/integration/test/StatisticIT.java b/clients/client-java/src/test/java/org/apache/gravitino/client/integration/test/StatisticIT.java index e3d3b255b14..e9956517839 100644 --- a/clients/client-java/src/test/java/org/apache/gravitino/client/integration/test/StatisticIT.java +++ b/clients/client-java/src/test/java/org/apache/gravitino/client/integration/test/StatisticIT.java @@ -30,8 +30,9 @@ import org.apache.gravitino.NameIdentifier; import org.apache.gravitino.client.GravitinoMetalake; import org.apache.gravitino.exceptions.IllegalStatisticNameException; -import org.apache.gravitino.integration.test.container.ContainerSuite; +import org.apache.gravitino.integration.test.container.FixtureContainer; import org.apache.gravitino.integration.test.container.HiveContainer; +import org.apache.gravitino.integration.test.container.HiveFixture; import org.apache.gravitino.integration.test.util.BaseIT; import org.apache.gravitino.integration.test.util.GravitinoITUtils; import org.apache.gravitino.rel.Column; @@ -63,8 +64,11 @@ import org.slf4j.LoggerFactory; @Tag("gravitino-docker-test") +@HiveFixture public class StatisticIT extends BaseIT { + @FixtureContainer private HiveContainer hive; + private static final Logger LOG = LoggerFactory.getLogger(StatisticIT.class); public static final String metalakeName = GravitinoITUtils.genRandomName("cataloghiveit_metalake"); @@ -79,7 +83,6 @@ public class StatisticIT extends BaseIT { public static final String HIVE_COL_NAME3 = "hive_col_name3"; private String HIVE_METASTORE_URIS; private final String provider = "hive"; - private final ContainerSuite containerSuite = ContainerSuite.getInstance(); private GravitinoMetalake metalake; private Catalog catalog; private Table table; @@ -284,13 +287,9 @@ public void testPartitionStatisticLifeCycle() { } protected void startNecessaryContainer() { - containerSuite.startHiveContainer(); - HIVE_METASTORE_URIS = String.format( - "thrift://%s:%d", - containerSuite.getHiveContainer().getContainerIpAddress(), - HiveContainer.HIVE_METASTORE_PORT); + "thrift://%s:%d", hive.getContainerIpAddress(), HiveContainer.HIVE_METASTORE_PORT); } private void createMetalake() { @@ -382,7 +381,7 @@ protected Map createSchemaProperties() { "location", String.format( "hdfs://%s:%d/user/hive/warehouse/%s.db", - containerSuite.getHiveContainer().getContainerIpAddress(), + hive.getContainerIpAddress(), HiveContainer.HDFS_DEFAULTFS_PORT, schemaName.toLowerCase())); return properties; diff --git a/clients/client-java/src/test/java/org/apache/gravitino/client/integration/test/TagIT.java b/clients/client-java/src/test/java/org/apache/gravitino/client/integration/test/TagIT.java index 80a6018ab00..c53c2e54c85 100644 --- a/clients/client-java/src/test/java/org/apache/gravitino/client/integration/test/TagIT.java +++ b/clients/client-java/src/test/java/org/apache/gravitino/client/integration/test/TagIT.java @@ -43,8 +43,9 @@ import org.apache.gravitino.function.FunctionParam; import org.apache.gravitino.function.FunctionParams; import org.apache.gravitino.function.FunctionType; -import org.apache.gravitino.integration.test.container.ContainerSuite; +import org.apache.gravitino.integration.test.container.FixtureContainer; import org.apache.gravitino.integration.test.container.HiveContainer; +import org.apache.gravitino.integration.test.container.HiveFixture; import org.apache.gravitino.integration.test.util.BaseIT; import org.apache.gravitino.integration.test.util.GravitinoITUtils; import org.apache.gravitino.model.Model; @@ -66,9 +67,10 @@ import org.junit.jupiter.api.Test; @org.junit.jupiter.api.Tag("gravitino-docker-test") +@HiveFixture public class TagIT extends BaseIT { - private static final ContainerSuite containerSuite = ContainerSuite.getInstance(); + @FixtureContainer private HiveContainer hive; private static final String metalakeName = GravitinoITUtils.genRandomName("tag_it_metalake"); @@ -87,12 +89,9 @@ public class TagIT extends BaseIT { @BeforeAll public void setUp() { - containerSuite.startHiveContainer(); String hmsUri = String.format( - "thrift://%s:%d", - containerSuite.getHiveContainer().getContainerIpAddress(), - HiveContainer.HIVE_METASTORE_PORT); + "thrift://%s:%d", hive.getContainerIpAddress(), HiveContainer.HIVE_METASTORE_PORT); // Create metalake Assertions.assertFalse(client.metalakeExists(metalakeName)); diff --git a/clients/client-java/src/test/java/org/apache/gravitino/client/integration/test/authorization/CatalogAuthorizationIT.java b/clients/client-java/src/test/java/org/apache/gravitino/client/integration/test/authorization/CatalogAuthorizationIT.java index 22dae4f4b11..d7545bddf49 100644 --- a/clients/client-java/src/test/java/org/apache/gravitino/client/integration/test/authorization/CatalogAuthorizationIT.java +++ b/clients/client-java/src/test/java/org/apache/gravitino/client/integration/test/authorization/CatalogAuthorizationIT.java @@ -34,8 +34,9 @@ import org.apache.gravitino.client.GravitinoMetalake; import org.apache.gravitino.dto.MetalakeDTO; import org.apache.gravitino.exceptions.ForbiddenException; -import org.apache.gravitino.integration.test.container.ContainerSuite; +import org.apache.gravitino.integration.test.container.FixtureContainer; import org.apache.gravitino.integration.test.container.HiveContainer; +import org.apache.gravitino.integration.test.container.HiveFixture; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.MethodOrderer; import org.junit.jupiter.api.Order; @@ -45,25 +46,23 @@ @Tag("gravitino-docker-test") @TestMethodOrder(MethodOrderer.OrderAnnotation.class) +@HiveFixture public class CatalogAuthorizationIT extends BaseRestApiAuthorizationIT { + @FixtureContainer private HiveContainer hive; + private String catalog1 = "catalog1"; private String catalog2 = "catalog2"; - private static final ContainerSuite containerSuite = ContainerSuite.getInstance(); - private static String hmsUri; @BeforeAll public void startIntegrationTest() throws Exception { - containerSuite.startHiveContainer(); super.startIntegrationTest(); hmsUri = String.format( - "thrift://%s:%d", - containerSuite.getHiveContainer().getContainerIpAddress(), - HiveContainer.HIVE_METASTORE_PORT); + "thrift://%s:%d", hive.getContainerIpAddress(), HiveContainer.HIVE_METASTORE_PORT); } @Test diff --git a/clients/client-java/src/test/java/org/apache/gravitino/client/integration/test/authorization/FilesetAuthorizationIT.java b/clients/client-java/src/test/java/org/apache/gravitino/client/integration/test/authorization/FilesetAuthorizationIT.java index 6d774b7b3c0..6b6dbd80e8b 100644 --- a/clients/client-java/src/test/java/org/apache/gravitino/client/integration/test/authorization/FilesetAuthorizationIT.java +++ b/clients/client-java/src/test/java/org/apache/gravitino/client/integration/test/authorization/FilesetAuthorizationIT.java @@ -50,8 +50,9 @@ import org.apache.gravitino.file.Fileset; import org.apache.gravitino.file.FilesetCatalog; import org.apache.gravitino.file.FilesetChange; -import org.apache.gravitino.integration.test.container.ContainerSuite; +import org.apache.gravitino.integration.test.container.FixtureContainer; import org.apache.gravitino.integration.test.container.HiveContainer; +import org.apache.gravitino.integration.test.container.HiveFixture; import org.apache.gravitino.integration.test.util.GravitinoITUtils; import org.apache.http.HttpHeaders; import org.apache.http.client.methods.CloseableHttpResponse; @@ -69,20 +70,20 @@ @Tag("gravitino-docker-test") @TestMethodOrder(MethodOrderer.OrderAnnotation.class) +@HiveFixture public class FilesetAuthorizationIT extends BaseRestApiAuthorizationIT { + @FixtureContainer private HiveContainer hive; + private static final String CATALOG = "catalog"; private static final String SCHEMA = "schema"; - private static final ContainerSuite containerSuite = ContainerSuite.getInstance(); - private String role = "role"; private String defaultBaseLocation; protected CloseableHttpClient httpClient; @BeforeAll public void startIntegrationTest() throws Exception { - containerSuite.startHiveContainer(); super.startIntegrationTest(); client .loadMetalake(METALAKE) @@ -443,7 +444,7 @@ private String defaultBaseLocation() { defaultBaseLocation = String.format( "hdfs://%s:%d/user/hive/%s", - containerSuite.getHiveContainer().getContainerIpAddress(), + hive.getContainerIpAddress(), HiveContainer.HDFS_DEFAULTFS_PORT, SCHEMA.toLowerCase()); } diff --git a/clients/client-java/src/test/java/org/apache/gravitino/client/integration/test/authorization/MetadataObjectRoleAuthorizationIT.java b/clients/client-java/src/test/java/org/apache/gravitino/client/integration/test/authorization/MetadataObjectRoleAuthorizationIT.java index b90f7ffc318..b73e7ca2eb7 100644 --- a/clients/client-java/src/test/java/org/apache/gravitino/client/integration/test/authorization/MetadataObjectRoleAuthorizationIT.java +++ b/clients/client-java/src/test/java/org/apache/gravitino/client/integration/test/authorization/MetadataObjectRoleAuthorizationIT.java @@ -28,14 +28,19 @@ import org.apache.gravitino.authorization.Privileges; import org.apache.gravitino.authorization.SecurableObjects; import org.apache.gravitino.client.GravitinoMetalake; +import org.apache.gravitino.integration.test.container.FixtureContainer; import org.apache.gravitino.integration.test.container.HiveContainer; +import org.apache.gravitino.integration.test.container.HiveFixture; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; @Tag("gravitino-docker-test") +@HiveFixture public class MetadataObjectRoleAuthorizationIT extends BaseRestApiAuthorizationIT { + @FixtureContainer private HiveContainer hive; + private String hmsUri; private static final String CATALOG = "catalog1"; @@ -43,13 +48,10 @@ public class MetadataObjectRoleAuthorizationIT extends BaseRestApiAuthorizationI @BeforeAll @Override public void startIntegrationTest() throws Exception { - containerSuite.startHiveContainer(); super.startIntegrationTest(); hmsUri = String.format( - "thrift://%s:%d", - containerSuite.getHiveContainer().getContainerIpAddress(), - HiveContainer.HIVE_METASTORE_PORT); + "thrift://%s:%d", hive.getContainerIpAddress(), HiveContainer.HIVE_METASTORE_PORT); Map properties = Maps.newHashMap(); properties.put("metastore.uris", hmsUri); client diff --git a/clients/client-java/src/test/java/org/apache/gravitino/client/integration/test/authorization/OwnerAuthorizationIT.java b/clients/client-java/src/test/java/org/apache/gravitino/client/integration/test/authorization/OwnerAuthorizationIT.java index a055cbce733..5e6cca617d9 100644 --- a/clients/client-java/src/test/java/org/apache/gravitino/client/integration/test/authorization/OwnerAuthorizationIT.java +++ b/clients/client-java/src/test/java/org/apache/gravitino/client/integration/test/authorization/OwnerAuthorizationIT.java @@ -38,8 +38,9 @@ import org.apache.gravitino.authorization.Privileges; import org.apache.gravitino.client.GravitinoMetalake; import org.apache.gravitino.exceptions.ForbiddenException; -import org.apache.gravitino.integration.test.container.ContainerSuite; +import org.apache.gravitino.integration.test.container.FixtureContainer; import org.apache.gravitino.integration.test.container.HiveContainer; +import org.apache.gravitino.integration.test.container.HiveFixture; import org.apache.gravitino.job.JobHandle; import org.apache.gravitino.job.JobTemplate; import org.apache.gravitino.job.ShellJobTemplate; @@ -57,11 +58,13 @@ @Tag("gravitino-docker-test") @TestMethodOrder(MethodOrderer.OrderAnnotation.class) +@HiveFixture public class OwnerAuthorizationIT extends BaseRestApiAuthorizationIT { + @FixtureContainer private HiveContainer hive; + private static final String CATALOG = "catalog"; private static final String SCHEMA = "schema"; - private static final ContainerSuite containerSuite = ContainerSuite.getInstance(); private static String hmsUri; private static final String role = "role"; private static final String TEMP_USER = "user3"; @@ -69,13 +72,10 @@ public class OwnerAuthorizationIT extends BaseRestApiAuthorizationIT { @BeforeAll public void startIntegrationTest() throws Exception { - containerSuite.startHiveContainer(); super.startIntegrationTest(); hmsUri = String.format( - "thrift://%s:%d", - containerSuite.getHiveContainer().getContainerIpAddress(), - HiveContainer.HIVE_METASTORE_PORT); + "thrift://%s:%d", hive.getContainerIpAddress(), HiveContainer.HIVE_METASTORE_PORT); Map properties = Maps.newHashMap(); properties.put("metastore.uris", hmsUri); client diff --git a/clients/client-java/src/test/java/org/apache/gravitino/client/integration/test/authorization/OwnerIT.java b/clients/client-java/src/test/java/org/apache/gravitino/client/integration/test/authorization/OwnerIT.java index 4de1ebd3ae8..1ab00e020ae 100644 --- a/clients/client-java/src/test/java/org/apache/gravitino/client/integration/test/authorization/OwnerIT.java +++ b/clients/client-java/src/test/java/org/apache/gravitino/client/integration/test/authorization/OwnerIT.java @@ -37,9 +37,9 @@ import org.apache.gravitino.exceptions.NoSuchMetadataObjectException; import org.apache.gravitino.exceptions.NotFoundException; import org.apache.gravitino.file.Fileset; -import org.apache.gravitino.integration.test.container.ContainerSuite; import org.apache.gravitino.integration.test.container.FixtureContainer; import org.apache.gravitino.integration.test.container.HiveContainer; +import org.apache.gravitino.integration.test.container.HiveFixture; import org.apache.gravitino.integration.test.container.KafkaContainer; import org.apache.gravitino.integration.test.container.KafkaFixture; import org.apache.gravitino.integration.test.util.BaseIT; @@ -56,12 +56,13 @@ @Tag("gravitino-docker-test") @KafkaFixture +@HiveFixture public class OwnerIT extends BaseIT { + @FixtureContainer private HiveContainer hive; @FixtureContainer private KafkaContainer kafka; private static final Logger LOG = LoggerFactory.getLogger(OwnerIT.class); - private static final ContainerSuite containerSuite = ContainerSuite.getInstance(); private static String hmsUri; private static String kafkaBootstrapServers; @@ -76,12 +77,9 @@ public void startIntegrationTest() throws Exception { registerCustomConfigs(configs); super.startIntegrationTest(); - containerSuite.startHiveContainer(); hmsUri = String.format( - "thrift://%s:%d", - containerSuite.getHiveContainer().getContainerIpAddress(), - HiveContainer.HIVE_METASTORE_PORT); + "thrift://%s:%d", hive.getContainerIpAddress(), HiveContainer.HIVE_METASTORE_PORT); kafkaBootstrapServers = String.format("%s:%d", kafka.getContainerIpAddress(), KafkaContainer.DEFAULT_BROKER_PORT); } diff --git a/clients/client-java/src/test/java/org/apache/gravitino/client/integration/test/authorization/PartitionAuthorizationIT.java b/clients/client-java/src/test/java/org/apache/gravitino/client/integration/test/authorization/PartitionAuthorizationIT.java index 4418e3f2263..93b33da5d6e 100644 --- a/clients/client-java/src/test/java/org/apache/gravitino/client/integration/test/authorization/PartitionAuthorizationIT.java +++ b/clients/client-java/src/test/java/org/apache/gravitino/client/integration/test/authorization/PartitionAuthorizationIT.java @@ -38,8 +38,9 @@ import org.apache.gravitino.authorization.SecurableObjects; import org.apache.gravitino.client.GravitinoMetalake; import org.apache.gravitino.exceptions.ForbiddenException; -import org.apache.gravitino.integration.test.container.ContainerSuite; +import org.apache.gravitino.integration.test.container.FixtureContainer; import org.apache.gravitino.integration.test.container.HiveContainer; +import org.apache.gravitino.integration.test.container.HiveFixture; import org.apache.gravitino.rel.Column; import org.apache.gravitino.rel.SupportsPartitions; import org.apache.gravitino.rel.Table; @@ -61,23 +62,22 @@ @Tag("gravitino-docker-test") @TestMethodOrder(MethodOrderer.OrderAnnotation.class) +@HiveFixture public class PartitionAuthorizationIT extends BaseRestApiAuthorizationIT { + @FixtureContainer private HiveContainer hive; + private static final String CATALOG = "catalog"; private static final String SCHEMA = "partitionSchema"; - private static final ContainerSuite containerSuite = ContainerSuite.getInstance(); private static String hmsUri; private static String role = "role"; @BeforeAll public void startIntegrationTest() throws Exception { - containerSuite.startHiveContainer(); super.startIntegrationTest(); hmsUri = String.format( - "thrift://%s:%d", - containerSuite.getHiveContainer().getContainerIpAddress(), - HiveContainer.HIVE_METASTORE_PORT); + "thrift://%s:%d", hive.getContainerIpAddress(), HiveContainer.HIVE_METASTORE_PORT); Map properties = Maps.newHashMap(); properties.put("metastore.uris", hmsUri); client diff --git a/clients/client-java/src/test/java/org/apache/gravitino/client/integration/test/authorization/PolicyAuthorizationIT.java b/clients/client-java/src/test/java/org/apache/gravitino/client/integration/test/authorization/PolicyAuthorizationIT.java index af524385fe4..6ec6b8a9efb 100644 --- a/clients/client-java/src/test/java/org/apache/gravitino/client/integration/test/authorization/PolicyAuthorizationIT.java +++ b/clients/client-java/src/test/java/org/apache/gravitino/client/integration/test/authorization/PolicyAuthorizationIT.java @@ -39,8 +39,9 @@ import org.apache.gravitino.authorization.SecurableObjects; import org.apache.gravitino.client.GravitinoMetalake; import org.apache.gravitino.exceptions.ForbiddenException; -import org.apache.gravitino.integration.test.container.ContainerSuite; +import org.apache.gravitino.integration.test.container.FixtureContainer; import org.apache.gravitino.integration.test.container.HiveContainer; +import org.apache.gravitino.integration.test.container.HiveFixture; import org.apache.gravitino.policy.PolicyChange; import org.apache.gravitino.policy.PolicyContents; import org.apache.gravitino.policy.PolicyList; @@ -61,23 +62,22 @@ @Tag("gravitino-docker-test") @TestMethodOrder(MethodOrderer.OrderAnnotation.class) @TestInstance(TestInstance.Lifecycle.PER_CLASS) +@HiveFixture public class PolicyAuthorizationIT extends BaseRestApiAuthorizationIT { + @FixtureContainer private HiveContainer hive; + private static final String CATALOG = "catalog"; private static final String SCHEMA = "schema"; - private static final ContainerSuite containerSuite = ContainerSuite.getInstance(); private static String hmsUri; private static String role = "role"; @BeforeAll public void startIntegrationTest() throws Exception { - containerSuite.startHiveContainer(); super.startIntegrationTest(); hmsUri = String.format( - "thrift://%s:%d", - containerSuite.getHiveContainer().getContainerIpAddress(), - HiveContainer.HIVE_METASTORE_PORT); + "thrift://%s:%d", hive.getContainerIpAddress(), HiveContainer.HIVE_METASTORE_PORT); Map properties = Maps.newHashMap(); properties.put("metastore.uris", hmsUri); client diff --git a/clients/client-java/src/test/java/org/apache/gravitino/client/integration/test/authorization/SchemaAuthorizationIT.java b/clients/client-java/src/test/java/org/apache/gravitino/client/integration/test/authorization/SchemaAuthorizationIT.java index 5fef28f15ca..1144f13ec10 100644 --- a/clients/client-java/src/test/java/org/apache/gravitino/client/integration/test/authorization/SchemaAuthorizationIT.java +++ b/clients/client-java/src/test/java/org/apache/gravitino/client/integration/test/authorization/SchemaAuthorizationIT.java @@ -39,8 +39,9 @@ import org.apache.gravitino.authorization.SecurableObjects; import org.apache.gravitino.client.GravitinoMetalake; import org.apache.gravitino.exceptions.ForbiddenException; -import org.apache.gravitino.integration.test.container.ContainerSuite; +import org.apache.gravitino.integration.test.container.FixtureContainer; import org.apache.gravitino.integration.test.container.HiveContainer; +import org.apache.gravitino.integration.test.container.HiveFixture; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.MethodOrderer; import org.junit.jupiter.api.Order; @@ -50,11 +51,12 @@ @Tag("gravitino-docker-test") @TestMethodOrder(MethodOrderer.OrderAnnotation.class) +@HiveFixture public class SchemaAuthorizationIT extends BaseRestApiAuthorizationIT { - private static final String CATALOG = "catalog"; + @FixtureContainer private HiveContainer hive; - private static final ContainerSuite containerSuite = ContainerSuite.getInstance(); + private static final String CATALOG = "catalog"; private static String hmsUri; @@ -62,13 +64,10 @@ public class SchemaAuthorizationIT extends BaseRestApiAuthorizationIT { @BeforeAll public void startIntegrationTest() throws Exception { - containerSuite.startHiveContainer(); super.startIntegrationTest(); hmsUri = String.format( - "thrift://%s:%d", - containerSuite.getHiveContainer().getContainerIpAddress(), - HiveContainer.HIVE_METASTORE_PORT); + "thrift://%s:%d", hive.getContainerIpAddress(), HiveContainer.HIVE_METASTORE_PORT); Map properties = Maps.newHashMap(); properties.put("metastore.uris", hmsUri); client diff --git a/clients/client-java/src/test/java/org/apache/gravitino/client/integration/test/authorization/StatisticAuthorizationIT.java b/clients/client-java/src/test/java/org/apache/gravitino/client/integration/test/authorization/StatisticAuthorizationIT.java index 5615076c5f6..9c5e55c6e02 100644 --- a/clients/client-java/src/test/java/org/apache/gravitino/client/integration/test/authorization/StatisticAuthorizationIT.java +++ b/clients/client-java/src/test/java/org/apache/gravitino/client/integration/test/authorization/StatisticAuthorizationIT.java @@ -35,8 +35,9 @@ import org.apache.gravitino.authorization.SecurableObjects; import org.apache.gravitino.client.GravitinoMetalake; import org.apache.gravitino.exceptions.ForbiddenException; -import org.apache.gravitino.integration.test.container.ContainerSuite; +import org.apache.gravitino.integration.test.container.FixtureContainer; import org.apache.gravitino.integration.test.container.HiveContainer; +import org.apache.gravitino.integration.test.container.HiveFixture; import org.apache.gravitino.rel.Column; import org.apache.gravitino.rel.Table; import org.apache.gravitino.rel.TableCatalog; @@ -58,22 +59,22 @@ @Tag("gravitino-docker-test") @TestMethodOrder(MethodOrderer.OrderAnnotation.class) +@HiveFixture public class StatisticAuthorizationIT extends BaseRestApiAuthorizationIT { + + @FixtureContainer private HiveContainer hive; + private static final String CATALOG = "catalog_stats"; private static final String SCHEMA = "schema_stats"; - private static final ContainerSuite containerSuite = ContainerSuite.getInstance(); private static String hmsUri; private static String role = "role_stats"; @BeforeAll public void startIntegrationTest() throws Exception { - containerSuite.startHiveContainer(); super.startIntegrationTest(); hmsUri = String.format( - "thrift://%s:%d", - containerSuite.getHiveContainer().getContainerIpAddress(), - HiveContainer.HIVE_METASTORE_PORT); + "thrift://%s:%d", hive.getContainerIpAddress(), HiveContainer.HIVE_METASTORE_PORT); Map properties = Maps.newHashMap(); properties.put("metastore.uris", hmsUri); GravitinoMetalake metalake = client.loadMetalake(METALAKE); diff --git a/clients/client-java/src/test/java/org/apache/gravitino/client/integration/test/authorization/TableAuthorizationIT.java b/clients/client-java/src/test/java/org/apache/gravitino/client/integration/test/authorization/TableAuthorizationIT.java index ebc7a8d39cf..792ab91af40 100644 --- a/clients/client-java/src/test/java/org/apache/gravitino/client/integration/test/authorization/TableAuthorizationIT.java +++ b/clients/client-java/src/test/java/org/apache/gravitino/client/integration/test/authorization/TableAuthorizationIT.java @@ -45,8 +45,9 @@ import org.apache.gravitino.client.GravitinoAdminClient; import org.apache.gravitino.client.GravitinoMetalake; import org.apache.gravitino.exceptions.ForbiddenException; -import org.apache.gravitino.integration.test.container.ContainerSuite; +import org.apache.gravitino.integration.test.container.FixtureContainer; import org.apache.gravitino.integration.test.container.HiveContainer; +import org.apache.gravitino.integration.test.container.HiveFixture; import org.apache.gravitino.rel.Column; import org.apache.gravitino.rel.Table; import org.apache.gravitino.rel.TableCatalog; @@ -62,23 +63,22 @@ @Tag("gravitino-docker-test") @TestMethodOrder(MethodOrderer.OrderAnnotation.class) +@HiveFixture public class TableAuthorizationIT extends BaseRestApiAuthorizationIT { + @FixtureContainer private HiveContainer hive; + private static final String CATALOG = "catalog"; private static final String SCHEMA = "schema"; - private static final ContainerSuite containerSuite = ContainerSuite.getInstance(); private static String hmsUri; private static String role = "role"; @BeforeAll public void startIntegrationTest() throws Exception { - containerSuite.startHiveContainer(); super.startIntegrationTest(); hmsUri = String.format( - "thrift://%s:%d", - containerSuite.getHiveContainer().getContainerIpAddress(), - HiveContainer.HIVE_METASTORE_PORT); + "thrift://%s:%d", hive.getContainerIpAddress(), HiveContainer.HIVE_METASTORE_PORT); Map properties = Maps.newHashMap(); properties.put("metastore.uris", hmsUri); client diff --git a/clients/client-java/src/test/java/org/apache/gravitino/client/integration/test/authorization/TagOperationsAuthorizationIT.java b/clients/client-java/src/test/java/org/apache/gravitino/client/integration/test/authorization/TagOperationsAuthorizationIT.java index 5c4b1fd7769..fc74ebce82a 100644 --- a/clients/client-java/src/test/java/org/apache/gravitino/client/integration/test/authorization/TagOperationsAuthorizationIT.java +++ b/clients/client-java/src/test/java/org/apache/gravitino/client/integration/test/authorization/TagOperationsAuthorizationIT.java @@ -41,8 +41,9 @@ import org.apache.gravitino.client.GravitinoMetalake; import org.apache.gravitino.dto.MetalakeDTO; import org.apache.gravitino.exceptions.ForbiddenException; -import org.apache.gravitino.integration.test.container.ContainerSuite; +import org.apache.gravitino.integration.test.container.FixtureContainer; import org.apache.gravitino.integration.test.container.HiveContainer; +import org.apache.gravitino.integration.test.container.HiveFixture; import org.apache.gravitino.rel.Column; import org.apache.gravitino.rel.TableCatalog; import org.apache.gravitino.rel.types.Types; @@ -60,23 +61,22 @@ @Tag("gravitino-docker-test") @TestMethodOrder(MethodOrderer.OrderAnnotation.class) +@HiveFixture public class TagOperationsAuthorizationIT extends BaseRestApiAuthorizationIT { + @FixtureContainer private HiveContainer hive; + private static final String CATALOG = "catalog"; private static final String SCHEMA = "schema"; - private static final ContainerSuite containerSuite = ContainerSuite.getInstance(); private static String hmsUri; private static String role = "role"; @BeforeAll public void startIntegrationTest() throws Exception { - containerSuite.startHiveContainer(); super.startIntegrationTest(); hmsUri = String.format( - "thrift://%s:%d", - containerSuite.getHiveContainer().getContainerIpAddress(), - HiveContainer.HIVE_METASTORE_PORT); + "thrift://%s:%d", hive.getContainerIpAddress(), HiveContainer.HIVE_METASTORE_PORT); Map properties = Maps.newHashMap(); properties.put("metastore.uris", hmsUri); client diff --git a/clients/filesystem-hadoop3/src/test/java/org/apache/gravitino/filesystem/hadoop/integration/test/GravitinoVirtualFileSystemIT.java b/clients/filesystem-hadoop3/src/test/java/org/apache/gravitino/filesystem/hadoop/integration/test/GravitinoVirtualFileSystemIT.java index 0b5c5995e19..1b4382f61fe 100644 --- a/clients/filesystem-hadoop3/src/test/java/org/apache/gravitino/filesystem/hadoop/integration/test/GravitinoVirtualFileSystemIT.java +++ b/clients/filesystem-hadoop3/src/test/java/org/apache/gravitino/filesystem/hadoop/integration/test/GravitinoVirtualFileSystemIT.java @@ -39,8 +39,9 @@ import org.apache.gravitino.client.GravitinoMetalake; import org.apache.gravitino.file.Fileset; import org.apache.gravitino.filesystem.hadoop.GravitinoVirtualFileSystemConfiguration; -import org.apache.gravitino.integration.test.container.ContainerSuite; +import org.apache.gravitino.integration.test.container.FixtureContainer; import org.apache.gravitino.integration.test.container.HiveContainer; +import org.apache.gravitino.integration.test.container.HiveFixture; import org.apache.gravitino.integration.test.util.BaseIT; import org.apache.gravitino.integration.test.util.GravitinoITUtils; import org.apache.hadoop.conf.Configuration; @@ -58,9 +59,10 @@ import org.slf4j.LoggerFactory; @Tag("gravitino-docker-test") +@HiveFixture public class GravitinoVirtualFileSystemIT extends BaseIT { private static final Logger LOG = LoggerFactory.getLogger(GravitinoVirtualFileSystemIT.class); - protected static final ContainerSuite containerSuite = ContainerSuite.getInstance(); + @FixtureContainer protected HiveContainer hive; protected String metalakeName = GravitinoITUtils.genRandomName("gvfs_it_metalake"); protected String catalogName = GravitinoITUtils.genRandomName("catalog"); protected String schemaName = GravitinoITUtils.genRandomName("schema"); @@ -72,7 +74,6 @@ public class GravitinoVirtualFileSystemIT extends BaseIT { @BeforeAll public void startUp() throws Exception { - containerSuite.startHiveContainer(); Assertions.assertFalse(client.metalakeExists(metalakeName)); metalake = client.createMetalake(metalakeName, "metalake comment", Collections.emptyMap()); Assertions.assertTrue(client.metalakeExists(metalakeName)); @@ -554,10 +555,7 @@ protected String genStorageLocation(String fileset) { private String baseHdfsPath() { return String.format( "hdfs://%s:%d/%s/%s", - containerSuite.getHiveContainer().getContainerIpAddress(), - HiveContainer.HDFS_DEFAULTFS_PORT, - catalogName, - schemaName); + hive.getContainerIpAddress(), HiveContainer.HDFS_DEFAULTFS_PORT, catalogName, schemaName); } protected Path genGvfsPath(String fileset) { diff --git a/flink-connector/flink-common/src/test/java/org/apache/gravitino/flink/connector/integration/test/FlinkEnvIT.java b/flink-connector/flink-common/src/test/java/org/apache/gravitino/flink/connector/integration/test/FlinkEnvIT.java index e0b0e7f2ea5..59343c63a47 100644 --- a/flink-connector/flink-common/src/test/java/org/apache/gravitino/flink/connector/integration/test/FlinkEnvIT.java +++ b/flink-connector/flink-common/src/test/java/org/apache/gravitino/flink/connector/integration/test/FlinkEnvIT.java @@ -40,8 +40,9 @@ import org.apache.gravitino.flink.connector.iceberg.IcebergPropertiesConstants; import org.apache.gravitino.flink.connector.integration.test.utils.TestUtils; import org.apache.gravitino.flink.connector.store.GravitinoCatalogStoreFactoryOptions; -import org.apache.gravitino.integration.test.container.ContainerSuite; +import org.apache.gravitino.integration.test.container.FixtureContainer; import org.apache.gravitino.integration.test.container.HiveContainer; +import org.apache.gravitino.integration.test.container.HiveFixture; import org.apache.gravitino.integration.test.util.BaseIT; import org.apache.hadoop.fs.FileSystem; import org.junit.jupiter.api.AfterAll; @@ -49,9 +50,10 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +@HiveFixture public abstract class FlinkEnvIT extends BaseIT { private static final Logger LOG = LoggerFactory.getLogger(FlinkEnvIT.class); - private static final ContainerSuite CONTAINER_SUITE = ContainerSuite.getInstance(); + @FixtureContainer private static HiveContainer hive; protected static final String icebergRestServiceName = "iceberg-rest"; @@ -153,17 +155,13 @@ private void initMetalake() { } private static void initHiveEnv() { - CONTAINER_SUITE.startHiveContainer(); hiveMetastoreUri = String.format( - "thrift://%s:%d", - CONTAINER_SUITE.getHiveContainer().getContainerIpAddress(), - HiveContainer.HIVE_METASTORE_PORT); + "thrift://%s:%d", hive.getContainerIpAddress(), HiveContainer.HIVE_METASTORE_PORT); warehouse = String.format( "hdfs://%s:%d/user/hive/warehouse", - CONTAINER_SUITE.getHiveContainer().getContainerIpAddress(), - HiveContainer.HDFS_DEFAULTFS_PORT); + hive.getContainerIpAddress(), HiveContainer.HDFS_DEFAULTFS_PORT); } private static void initHdfsEnv() { @@ -171,9 +169,7 @@ private static void initHdfsEnv() { conf.set( "fs.defaultFS", String.format( - "hdfs://%s:%d", - CONTAINER_SUITE.getHiveContainer().getContainerIpAddress(), - HiveContainer.HDFS_DEFAULTFS_PORT)); + "hdfs://%s:%d", hive.getContainerIpAddress(), HiveContainer.HDFS_DEFAULTFS_PORT)); try { hdfs = FileSystem.get(conf); } catch (IOException e) { diff --git a/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergCacheIT.java b/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergCacheIT.java index cbd36562edd..766bd501dad 100644 --- a/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergCacheIT.java +++ b/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergCacheIT.java @@ -30,8 +30,9 @@ import org.apache.gravitino.iceberg.common.cache.TableMetadataCache; import org.apache.gravitino.iceberg.common.ops.IcebergCatalogWrapper; import org.apache.gravitino.iceberg.integration.test.util.IcebergRESTServerManager; -import org.apache.gravitino.integration.test.container.ContainerSuite; +import org.apache.gravitino.integration.test.container.FixtureContainer; import org.apache.gravitino.integration.test.container.HiveContainer; +import org.apache.gravitino.integration.test.container.HiveFixture; import org.apache.gravitino.integration.test.util.GravitinoITUtils; import org.apache.gravitino.server.web.JettyServerConfig; import org.apache.iceberg.Schema; @@ -55,9 +56,10 @@ /** Docker IT for default table metadata cache in {@link IcebergCatalogWrapper}. */ @Tag("gravitino-docker-test") @TestInstance(Lifecycle.PER_CLASS) +@HiveFixture public class IcebergCacheIT { - private static final ContainerSuite CONTAINER_SUITE = ContainerSuite.getInstance(); + @FixtureContainer private HiveContainer hive; private static final Schema TABLE_SCHEMA = new Schema(Types.NestedField.required(1, "id", Types.IntegerType.get())); @@ -66,7 +68,6 @@ public class IcebergCacheIT { @BeforeAll void startDockerEnv() throws Exception { - CONTAINER_SUITE.startHiveContainer(); restServer = IcebergRESTServerManager.create(); Map serverConfigs = new HashMap<>(); serverConfigs.put( @@ -173,20 +174,16 @@ void testCustomSupportLocationOn() throws Exception { } } - private static String hiveMetastoreUri() { + private String hiveMetastoreUri() { return String.format( - "thrift://%s:%d", - CONTAINER_SUITE.getHiveContainer().getContainerIpAddress(), - HiveContainer.HIVE_METASTORE_PORT); + "thrift://%s:%d", hive.getContainerIpAddress(), HiveContainer.HIVE_METASTORE_PORT); } - private static String hdfsWarehouse(String suffix) { + private String hdfsWarehouse(String suffix) { return GravitinoITUtils.genRandomName( String.format( "hdfs://%s:%d/user/hive/warehouse-%s", - CONTAINER_SUITE.getHiveContainer().getContainerIpAddress(), - HiveContainer.HDFS_DEFAULTFS_PORT, - suffix)); + hive.getContainerIpAddress(), HiveContainer.HDFS_DEFAULTFS_PORT, suffix)); } private int restPort() { diff --git a/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTHiveCatalogIT.java b/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTHiveCatalogIT.java index 8e899ef89f1..aec22a39069 100644 --- a/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTHiveCatalogIT.java +++ b/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTHiveCatalogIT.java @@ -24,8 +24,9 @@ import java.util.Map; import org.apache.gravitino.catalog.lakehouse.iceberg.IcebergCatalogBackend; import org.apache.gravitino.iceberg.common.IcebergConfig; -import org.apache.gravitino.integration.test.container.ContainerSuite; +import org.apache.gravitino.integration.test.container.FixtureContainer; import org.apache.gravitino.integration.test.container.HiveContainer; +import org.apache.gravitino.integration.test.container.HiveFixture; import org.apache.gravitino.integration.test.util.GravitinoITUtils; import org.apache.iceberg.CatalogProperties; import org.apache.iceberg.Table; @@ -41,8 +42,9 @@ @Tag("gravitino-docker-test") @TestInstance(Lifecycle.PER_CLASS) +@HiveFixture public class IcebergRESTHiveCatalogIT extends IcebergRESTServiceIT { - protected static final ContainerSuite containerSuite = ContainerSuite.getInstance(); + @FixtureContainer protected HiveContainer hive; @Override protected boolean supportsNestedNamespaces() { @@ -54,9 +56,7 @@ public IcebergRESTHiveCatalogIT() { } @Override - void initEnv() { - containerSuite.startHiveContainer(); - } + void initEnv() {} @Override Map getCatalogConfig() { @@ -68,17 +68,14 @@ Map getCatalogConfig() { configMap.put( IcebergConfig.ICEBERG_CONFIG_PREFIX + IcebergConfig.CATALOG_URI.getKey(), String.format( - "thrift://%s:%d", - containerSuite.getHiveContainer().getContainerIpAddress(), - HiveContainer.HIVE_METASTORE_PORT)); + "thrift://%s:%d", hive.getContainerIpAddress(), HiveContainer.HIVE_METASTORE_PORT)); configMap.put( IcebergConfig.ICEBERG_CONFIG_PREFIX + IcebergConfig.CATALOG_WAREHOUSE.getKey(), GravitinoITUtils.genRandomName( String.format( "hdfs://%s:%d/user/hive/warehouse-hive", - containerSuite.getHiveContainer().getContainerIpAddress(), - HiveContainer.HDFS_DEFAULTFS_PORT))); + hive.getContainerIpAddress(), HiveContainer.HDFS_DEFAULTFS_PORT))); return configMap; } diff --git a/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTJdbcCatalogIT.java b/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTJdbcCatalogIT.java index 8678ff73362..03b68a0b99c 100644 --- a/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTJdbcCatalogIT.java +++ b/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTJdbcCatalogIT.java @@ -23,8 +23,9 @@ import org.apache.gravitino.catalog.lakehouse.iceberg.IcebergCatalogBackend; import org.apache.gravitino.catalog.lakehouse.iceberg.IcebergConstants; import org.apache.gravitino.iceberg.common.IcebergConfig; -import org.apache.gravitino.integration.test.container.ContainerSuite; +import org.apache.gravitino.integration.test.container.FixtureContainer; import org.apache.gravitino.integration.test.container.HiveContainer; +import org.apache.gravitino.integration.test.container.HiveFixture; import org.apache.gravitino.integration.test.util.GravitinoITUtils; import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.TestInstance; @@ -32,10 +33,13 @@ @Tag("gravitino-docker-test") @TestInstance(Lifecycle.PER_CLASS) +@HiveFixture public class IcebergRESTJdbcCatalogIT extends IcebergRESTServiceIT { - private static final ContainerSuite containerSuite = ContainerSuite.getInstance(); - private boolean hiveStarted = false; + @FixtureContainer private HiveContainer hive; + + // Cloud storage subclasses override initEnv() and keep their own warehouse instead of HDFS. + private boolean useHdfsWarehouse = false; public IcebergRESTJdbcCatalogIT() { catalogType = IcebergCatalogBackend.JDBC; @@ -43,8 +47,7 @@ public IcebergRESTJdbcCatalogIT() { @Override void initEnv() { - containerSuite.startHiveContainer(); - hiveStarted = true; + useHdfsWarehouse = true; } @Override @@ -78,14 +81,13 @@ protected Map getCatalogJdbcConfig() { configMap.put(IcebergConfig.ICEBERG_CONFIG_PREFIX + "jdbc.schema-version", "V1"); - if (hiveStarted) { + if (useHdfsWarehouse) { configMap.put( IcebergConfig.ICEBERG_CONFIG_PREFIX + IcebergConfig.CATALOG_WAREHOUSE.getKey(), GravitinoITUtils.genRandomName( String.format( "hdfs://%s:%d/user/hive/warehouse-jdbc-sqlite", - containerSuite.getHiveContainer().getContainerIpAddress(), - HiveContainer.HDFS_DEFAULTFS_PORT))); + hive.getContainerIpAddress(), HiveContainer.HDFS_DEFAULTFS_PORT))); } return configMap; } diff --git a/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRestKerberosHiveCatalogIT.java b/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRestKerberosHiveCatalogIT.java index 0b494d0861b..8693e445738 100644 --- a/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRestKerberosHiveCatalogIT.java +++ b/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRestKerberosHiveCatalogIT.java @@ -22,7 +22,10 @@ import java.util.Map; import org.apache.gravitino.catalog.lakehouse.iceberg.IcebergCatalogBackend; import org.apache.gravitino.iceberg.common.IcebergConfig; +import org.apache.gravitino.integration.test.container.FixtureContainer; +import org.apache.gravitino.integration.test.container.HiveAuth; import org.apache.gravitino.integration.test.container.HiveContainer; +import org.apache.gravitino.integration.test.container.HiveFixture; import org.apache.gravitino.integration.test.util.GravitinoITUtils; import org.apache.spark.SparkConf; import org.junit.jupiter.api.Tag; @@ -33,18 +36,23 @@ @Tag("gravitino-docker-test") @TestInstance(Lifecycle.PER_CLASS) +@HiveFixture(auth = HiveAuth.KERBEROS) public class IcebergRestKerberosHiveCatalogIT extends IcebergRESTHiveCatalogIT { protected static final String HIVE_METASTORE_CLIENT_PRINCIPAL = "cli@HADOOPKRB"; protected static String tempDir; + // Replaces the superclass's plain Hive fixture, so the inherited hive field holds this same + // container; the name says which Hive the readers below expect. + @FixtureContainer protected HiveContainer kerberosHive; + public IcebergRestKerberosHiveCatalogIT() { super(); } void initEnv() { - tempDir = IcebergRestKerberosTestEnv.init(containerSuite); + tempDir = IcebergRestKerberosTestEnv.init(kerberosHive); } @Override @@ -66,15 +74,13 @@ Map getCatalogConfig() { configMap.put("gravitino.iceberg-rest.hive.metastore.sasl.enabled", "true"); configMap.put( "gravitino.iceberg-rest.hive.metastore.kerberos.principal", - "hive/_HOST@HADOOPKRB" - .replace("_HOST", containerSuite.getKerberosHiveContainer().getHostName())); + "hive/_HOST@HADOOPKRB".replace("_HOST", kerberosHive.getHostName())); configMap.put("gravitino.iceberg-rest.hadoop.security.authentication", "kerberos"); configMap.put( "gravitino.iceberg-rest.dfs.namenode.kerberos.principal", - "hdfs/_HOST@HADOOPKRB" - .replace("_HOST", containerSuite.getKerberosHiveContainer().getHostName())); + "hdfs/_HOST@HADOOPKRB".replace("_HOST", kerberosHive.getHostName())); configMap.put( IcebergConfig.ICEBERG_CONFIG_PREFIX + IcebergConfig.CATALOG_BACKEND.getKey(), @@ -84,25 +90,21 @@ Map getCatalogConfig() { IcebergConfig.ICEBERG_CONFIG_PREFIX + IcebergConfig.CATALOG_URI.getKey(), String.format( "thrift://%s:%d", - containerSuite.getKerberosHiveContainer().getContainerIpAddress(), - HiveContainer.HIVE_METASTORE_PORT)); + kerberosHive.getContainerIpAddress(), HiveContainer.HIVE_METASTORE_PORT)); configMap.put( IcebergConfig.ICEBERG_CONFIG_PREFIX + IcebergConfig.CATALOG_WAREHOUSE.getKey(), GravitinoITUtils.genRandomName( String.format( "hdfs://%s:%d/user/hive/warehouse-hive", - containerSuite.getKerberosHiveContainer().getContainerIpAddress(), - HiveContainer.HDFS_DEFAULTFS_PORT))); + kerberosHive.getContainerIpAddress(), HiveContainer.HDFS_DEFAULTFS_PORT))); return configMap; } @Override protected void customizeSparkConf(SparkConf sparkConf) { IcebergRestKerberosTestEnv.configureSparkKerberos( - sparkConf, - HIVE_METASTORE_CLIENT_PRINCIPAL, - containerSuite.getKerberosHiveContainer().getHostName()); + sparkConf, HIVE_METASTORE_CLIENT_PRINCIPAL, kerberosHive.getHostName()); } // Spark writes data files to HDFS directly in these tests; keep them in embedded mode only. diff --git a/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRestKerberosHiveWithUserImpersonationCatalogIT.java b/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRestKerberosHiveWithUserImpersonationCatalogIT.java index b9bc4e0988b..362bfd324c5 100644 --- a/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRestKerberosHiveWithUserImpersonationCatalogIT.java +++ b/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRestKerberosHiveWithUserImpersonationCatalogIT.java @@ -48,8 +48,7 @@ void prepareSQLContext() { // change the ownership, the normal user will not have the permission to create table in Hive // as the /user/hive is owned by user `cli`, please see what's done in `initEnv` method in // superclass. - IcebergRestKerberosTestEnv.hdfs(containerSuite.getKerberosHiveContainer()) - .setOwner("/user/hive/", NORMAL_USER, true); + IcebergRestKerberosTestEnv.hdfs(kerberosHive).setOwner("/user/hive/", NORMAL_USER, true); super.prepareSQLContext(); } @@ -93,9 +92,7 @@ protected void initSparkEnv() { } IcebergRestKerberosTestEnv.configureSparkKerberos( - sparkConf, - HIVE_METASTORE_CLIENT_PRINCIPAL, - containerSuite.getKerberosHiveContainer().getHostName()); + sparkConf, HIVE_METASTORE_CLIENT_PRINCIPAL, kerberosHive.getHostName()); sparkSession = SparkSession.builder().master("local[1]").config(sparkConf).getOrCreate(); } diff --git a/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRestKerberosJdbcCatalogIT.java b/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRestKerberosJdbcCatalogIT.java index a97f7f59c91..e64aa00a6ba 100644 --- a/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRestKerberosJdbcCatalogIT.java +++ b/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRestKerberosJdbcCatalogIT.java @@ -23,8 +23,10 @@ import org.apache.gravitino.catalog.lakehouse.iceberg.IcebergCatalogBackend; import org.apache.gravitino.catalog.lakehouse.iceberg.IcebergConstants; import org.apache.gravitino.iceberg.common.IcebergConfig; -import org.apache.gravitino.integration.test.container.ContainerSuite; +import org.apache.gravitino.integration.test.container.FixtureContainer; +import org.apache.gravitino.integration.test.container.HiveAuth; import org.apache.gravitino.integration.test.container.HiveContainer; +import org.apache.gravitino.integration.test.container.HiveFixture; import org.apache.gravitino.integration.test.util.GravitinoITUtils; import org.apache.spark.SparkConf; import org.junit.jupiter.api.Tag; @@ -35,21 +37,22 @@ @Tag("gravitino-docker-test") @TestInstance(Lifecycle.PER_CLASS) +@HiveFixture(auth = HiveAuth.KERBEROS) public class IcebergRestKerberosJdbcCatalogIT extends IcebergRESTServiceIT { - private static final ContainerSuite containerSuite = ContainerSuite.getInstance(); - protected static final String HDFS_CLIENT_PRINCIPAL = "cli@HADOOPKRB"; private static String tempDir; + @FixtureContainer protected HiveContainer kerberosHive; + public IcebergRestKerberosJdbcCatalogIT() { catalogType = IcebergCatalogBackend.JDBC; } @Override void initEnv() { - tempDir = IcebergRestKerberosTestEnv.init(containerSuite); + tempDir = IcebergRestKerberosTestEnv.init(kerberosHive); } @Override @@ -71,8 +74,7 @@ Map getCatalogConfig() { configMap.put("gravitino.iceberg-rest.authentication.impersonation-enable", "false"); configMap.put( "gravitino.iceberg-rest.dfs.namenode.kerberos.principal", - "hdfs/_HOST@HADOOPKRB" - .replace("_HOST", containerSuite.getKerberosHiveContainer().getHostName())); + "hdfs/_HOST@HADOOPKRB".replace("_HOST", kerberosHive.getHostName())); configMap.put( IcebergConfig.ICEBERG_CONFIG_PREFIX + IcebergConfig.CATALOG_BACKEND.getKey(), @@ -95,15 +97,14 @@ Map getCatalogConfig() { GravitinoITUtils.genRandomName( String.format( "hdfs://%s:%d/user/hive/warehouse-jdbc-kerberos", - containerSuite.getKerberosHiveContainer().getContainerIpAddress(), - HiveContainer.HDFS_DEFAULTFS_PORT))); + kerberosHive.getContainerIpAddress(), HiveContainer.HDFS_DEFAULTFS_PORT))); return configMap; } @Override protected void customizeSparkConf(SparkConf sparkConf) { IcebergRestKerberosTestEnv.configureSparkKerberos( - sparkConf, HDFS_CLIENT_PRINCIPAL, containerSuite.getKerberosHiveContainer().getHostName()); + sparkConf, HDFS_CLIENT_PRINCIPAL, kerberosHive.getHostName()); } // Spark writes data files to HDFS directly in these tests; keep them in embedded mode only. diff --git a/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRestKerberosJdbcWithUserImpersonationCatalogIT.java b/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRestKerberosJdbcWithUserImpersonationCatalogIT.java index 2e7dc4875a6..fe68ebbca73 100644 --- a/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRestKerberosJdbcWithUserImpersonationCatalogIT.java +++ b/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRestKerberosJdbcWithUserImpersonationCatalogIT.java @@ -21,7 +21,6 @@ import java.util.HashMap; import java.util.Map; import javax.annotation.Nullable; -import org.apache.gravitino.integration.test.container.ContainerSuite; import org.apache.spark.SparkConf; import org.apache.spark.sql.SparkSession; import org.junit.jupiter.api.BeforeAll; @@ -38,15 +37,12 @@ public class IcebergRestKerberosJdbcWithUserImpersonationCatalogIT private static final String NORMAL_USER = "normal"; - private static final ContainerSuite containerSuite = ContainerSuite.getInstance(); - @BeforeAll void prepareSQLContext() { // Change the ownership of /user/hive to normal user for user impersonation test. If we do not // change the ownership, the normal user will not have the permission to create table on HDFS // warehouse as the /user/hive is owned by user `cli`. - IcebergRestKerberosTestEnv.hdfs(containerSuite.getKerberosHiveContainer()) - .setOwner("/user/hive/", NORMAL_USER, true); + IcebergRestKerberosTestEnv.hdfs(kerberosHive).setOwner("/user/hive/", NORMAL_USER, true); super.prepareSQLContext(); } @@ -90,7 +86,7 @@ protected void initSparkEnv() { } IcebergRestKerberosTestEnv.configureSparkKerberos( - sparkConf, HDFS_CLIENT_PRINCIPAL, containerSuite.getKerberosHiveContainer().getHostName()); + sparkConf, HDFS_CLIENT_PRINCIPAL, kerberosHive.getHostName()); sparkSession = SparkSession.builder().master("local[1]").config(sparkConf).getOrCreate(); } diff --git a/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRestKerberosTestEnv.java b/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRestKerberosTestEnv.java index 7bb9a5ef992..2a13b4bdbc6 100644 --- a/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRestKerberosTestEnv.java +++ b/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRestKerberosTestEnv.java @@ -26,7 +26,6 @@ import java.util.HashMap; import java.util.Map; import org.apache.commons.io.FileUtils; -import org.apache.gravitino.integration.test.container.ContainerSuite; import org.apache.gravitino.integration.test.container.HiveContainer; import org.apache.gravitino.integration.test.util.ITUtils; import org.apache.gravitino.integration.test.util.KerberosHdfs; @@ -125,25 +124,23 @@ public static boolean isDeployMode() { } /** - * Starts the Kerberos Hive container and configures the JVM krb5 settings for tests. + * Configures the JVM krb5 settings for tests against a running Kerberos Hive container. * - * @param containerSuite shared docker container suite + * @param kerberosHiveContainer the running Kerberos Hive container * @return temp directory containing krb5.conf and client keytab */ - public static String init(ContainerSuite containerSuite) { + public static String init(HiveContainer kerberosHiveContainer) { if (initRefCount == 0) { savedSystemProperties = saveSystemProperties(MANAGED_SYSTEM_PROPERTIES); } initRefCount++; - containerSuite.startKerberosHiveContainer(); try { File baseDir = new File(System.getProperty("java.io.tmpdir")); File file = Files.createTempDirectory(baseDir.toPath(), "test").toFile(); file.deleteOnExit(); String tempDir = file.getAbsolutePath(); - HiveContainer kerberosHiveContainer = containerSuite.getKerberosHiveContainer(); kerberosHiveContainer .getContainer() .copyFileFromContainer(KEYTAB_CONTAINER_PATH, tempDir + CLIENT_KEYTAB); @@ -181,7 +178,7 @@ public static String init(ContainerSuite containerSuite) { } /** - * Returns an HDFS client that logs in with the keytab {@link #init(ContainerSuite)} copied. + * Returns an HDFS client that logs in with the keytab {@link #init(HiveContainer)} copied. * * @param kerberosHiveContainer the running Kerberos Hive container * @return the client @@ -194,7 +191,7 @@ public static KerberosHdfs hdfs(HiveContainer kerberosHiveContainer) { return KerberosHdfs.of(kerberosHiveContainer, tempDir + CLIENT_KEYTAB); } - /** Restores JVM Kerberos system properties changed by {@link #init(ContainerSuite)}. */ + /** Restores JVM Kerberos system properties changed by {@link #init(HiveContainer)}. */ public static void reset() { if (initRefCount == 0) { return; diff --git a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/ClickHouseFixture.java b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/ClickHouseFixture.java new file mode 100644 index 00000000000..7ee4d194f3a --- /dev/null +++ b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/ClickHouseFixture.java @@ -0,0 +1,65 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.integration.test.container; + +import java.lang.annotation.Documented; +import java.lang.annotation.ElementType; +import java.lang.annotation.Inherited; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; +import org.apache.gravitino.integration.test.util.TestDatabaseName; +import org.junit.jupiter.api.extension.ExtendWith; + +/** + * Declares that a test class needs a ClickHouse server, or the three-node ClickHouse cluster with + * {@code nodes = 3}, with the test's database created on it. Test and lifecycle methods receive it + * as a {@link ClickHouseContainer} parameter, and fields marked {@link FixtureContainer} receive it + * too. For the cluster, that is the first node; {@link + * ContainerSuite#getClickHouseClusterContainers()} returns every node. + * + * 

{@code
+ * @ClickHouseFixture(database = TestDatabaseName.CLICKHOUSE_CATALOG_CLICKHOUSE_IT)
+ * class CatalogClickHouseIT extends BaseIT {
+ *   @FixtureContainer private static ClickHouseContainer clickHouse;
+ * }
+ * }
+ */ +@Documented +@Inherited +@Target(ElementType.TYPE) +@Retention(RetentionPolicy.RUNTIME) +@ExtendWith(ContainerExtension.class) +@ContainerFixture(ClickHouseSpec.Factory.class) +public @interface ClickHouseFixture { + + /** + * Returns the number of ClickHouse nodes. + * + * @return 1 for a single server (the default) or 3 for the cluster + */ + int nodes() default 1; + + /** + * Returns the database to create on the ClickHouse server before the test runs. + * + * @return the test's database + */ + TestDatabaseName database(); +} diff --git a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/ClickHouseSpec.java b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/ClickHouseSpec.java new file mode 100644 index 00000000000..2d49ff13093 --- /dev/null +++ b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/ClickHouseSpec.java @@ -0,0 +1,105 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.integration.test.container; + +import com.google.common.base.Supplier; +import com.google.common.base.Suppliers; +import java.io.IOException; +import java.io.InputStream; +import java.io.UncheckedIOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; +import java.util.Objects; +import org.apache.gravitino.integration.test.util.TestDatabaseName; + +/** + * A ClickHouse server ({@code nodes} 1) or the three-node ClickHouse cluster with ZooKeeper ({@code + * nodes} 3), with {@code database} created on it. + * + *

The database is created on the shared container and is not part of the container identity: two + * specs that differ only by database share one ClickHouse. The cluster size is fixed by {@link + * ContainerSuite}, so a variable size needs the cluster start to move out of {@code + * ContainerSuite}. + */ +public record ClickHouseSpec(int nodes, TestDatabaseName database) + implements ContainerSpec { + + private static final int SINGLE_NODE = 1; + // Must match ContainerSuite.CLICKHOUSE_CLUSTER_SIZE. + private static final int CLUSTER_NODES = 3; + + private static final String REMOTE_SERVERS_TEMPLATE = "/remote_servers.xml"; + + private static final Supplier REMOTE_SERVERS = + Suppliers.memoize(ClickHouseSpec::copyRemoteServersTemplate); + + /** + * Checks the node count. + * + * @throws IllegalArgumentException if {@code nodes} is neither 1 nor 3 + */ + public ClickHouseSpec { + if (nodes != SINGLE_NODE && nodes != CLUSTER_NODES) { + throw new IllegalArgumentException( + String.format( + "ClickHouse supports %d or %d nodes, not %d", SINGLE_NODE, CLUSTER_NODES, nodes)); + } + Objects.requireNonNull(database, "database"); + } + + @Override + public Class containerType() { + return ClickHouseContainer.class; + } + + @Override + public ClickHouseContainer getOrStart(ContainerSuite suite) { + if (nodes == SINGLE_NODE) { + suite.startClickHouseContainer(database); + return suite.getClickHouseContainer(); + } + suite.startClickHouseClusterContainer(database, REMOTE_SERVERS.get().toString()); + return suite.getClickHouseClusterContainer(); + } + + // ContainerSuite reads the template from a file path, and the resource may sit inside a jar. + // Copied on first use, not in a static initializer, so loading this class never touches the + // disk; a failed copy is not cached and the next start tries again. + private static Path copyRemoteServersTemplate() { + try (InputStream template = + Objects.requireNonNull( + ClickHouseSpec.class.getResourceAsStream(REMOTE_SERVERS_TEMPLATE), + REMOTE_SERVERS_TEMPLATE)) { + Path path = Files.createTempFile("gravitino-clickhouse-remote-servers-", ".xml"); + path.toFile().deleteOnExit(); + Files.copy(template, path, StandardCopyOption.REPLACE_EXISTING); + return path; + } catch (IOException e) { + throw new UncheckedIOException("Cannot copy " + REMOTE_SERVERS_TEMPLATE, e); + } + } + + static final class Factory implements ContainerFixture.SpecFactory { + @Override + public ClickHouseSpec specFor(ClickHouseFixture fixture) { + return new ClickHouseSpec(fixture.nodes(), fixture.database()); + } + } +} diff --git a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/ContainerExtension.java b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/ContainerExtension.java index a4b02028b12..be04a34c6bd 100644 --- a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/ContainerExtension.java +++ b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/ContainerExtension.java @@ -29,8 +29,10 @@ import javax.annotation.Nullable; import org.apache.gravitino.integration.test.util.CloseContainerExtension; import org.junit.jupiter.api.extension.BeforeAllCallback; +import org.junit.jupiter.api.extension.BeforeEachCallback; import org.junit.jupiter.api.extension.ExtensionConfigurationException; import org.junit.jupiter.api.extension.ExtensionContext; +import org.junit.jupiter.api.extension.ExtensionContext.Namespace; import org.junit.jupiter.api.extension.ParameterContext; import org.junit.jupiter.api.extension.ParameterResolutionException; import org.junit.jupiter.api.extension.ParameterResolver; @@ -51,33 +53,62 @@ * the same annotation type, so {@code @HiveFixture(version = HIVE3)} on a subclass of a Hive2 test * runs Hive3 only. * - *

Each container starts once per run and is shared by every class that asks for the same spec. - * The containers close when the run ends, through {@link CloseContainerExtension}. + *

By default ({@link Scope#RUN}) each container starts once per run and is shared by every class + * that asks for the same spec; the containers close when the run ends, through {@link + * CloseContainerExtension}. A {@link Scope#CLASS} container is private to the class and closes + * after it; a {@link Scope#METHOD} container is private to one test and closes after it, so it can + * only reach instance fields and the parameters of test and {@code @BeforeEach} methods. */ public class ContainerExtension - implements BeforeAllCallback, ParameterResolver, TestInstancePostProcessor { + implements BeforeAllCallback, BeforeEachCallback, ParameterResolver, TestInstancePostProcessor { + + private static final String PRIVATE_CONTAINER = "privateContainer"; private final CloseContainerExtension suiteCloser = new CloseContainerExtension(); + /** A fixture on the test class: what it asks for and how long its container lives. */ + record Declaration(ContainerSpec spec, Scope scope) {} + @Override public void beforeAll(ExtensionContext context) { suiteCloser.beforeAll(context); Class testClass = context.getRequiredTestClass(); - for (ContainerSpec spec : specs(testClass)) { - spec.getOrStart(ContainerSuite.getInstance()); + for (Declaration declaration : declarations(testClass)) { + if (declaration.scope() != Scope.METHOD) { + container(declaration, context); + } } - injectFields(testClass, null, ModifierSupport::isStatic); + injectFields(testClass, null, ModifierSupport::isStatic, context, false); + } + + @Override + public void beforeEach(ExtensionContext context) { + Class testClass = context.getRequiredTestClass(); + for (Declaration declaration : declarations(testClass)) { + if (declaration.scope() == Scope.METHOD) { + container(declaration, context); + } + } + // Each test gets a new METHOD container, so refill the fields that hold one. + injectFields( + testClass, context.getRequiredTestInstance(), ModifierSupport::isNotStatic, context, true); } // Runs when the instance is created, before beforeAll(), so instance fields are filled before an // overridden @BeforeAll method such as BaseIT.startIntegrationTest() runs. @Override public void postProcessTestInstance(Object testInstance, ExtensionContext context) { - injectFields(testInstance.getClass(), testInstance, ModifierSupport::isNotStatic); + injectFields( + testInstance.getClass(), testInstance, ModifierSupport::isNotStatic, context, false); } + // Fills the matching fields whose container is METHOD scoped (methodScoped) or not. private static void injectFields( - Class testClass, @Nullable Object testInstance, Predicate which) { + Class testClass, + @Nullable Object testInstance, + Predicate which, + ExtensionContext context, + boolean methodScoped) { for (Field field : AnnotationSupport.findAnnotatedFields( testClass, FixtureContainer.class, which, HierarchyTraversalMode.TOP_DOWN)) { @@ -87,9 +118,17 @@ private static void injectFields( "%s: @FixtureContainer field %s must not be final", testClass.getSimpleName(), field.getName())); } + Declaration declaration = + declarationFor(field.getType(), testClass, "field " + field.getName()); + if (declaration.scope() == Scope.METHOD && ModifierSupport.isStatic(field)) { + throw methodScopeOutsideTest(testClass, declaration.spec()); + } + if ((declaration.scope() == Scope.METHOD) != methodScoped) { + continue; + } try { field.setAccessible(true); - field.set(testInstance, resolve(field.getType(), testClass, "field " + field.getName())); + field.set(testInstance, container(declaration, context)); } catch (IllegalAccessException e) { throw new ExtensionConfigurationException("Cannot set field " + field.getName(), e); } @@ -103,61 +142,113 @@ public boolean supportsParameter(ParameterContext parameter, ExtensionContext co @Override public Object resolveParameter(ParameterContext parameter, ExtensionContext context) { - return resolve( - parameter.getParameter().getType(), - context.getRequiredTestClass(), - "parameter " + parameter.getParameter().getName()); + Declaration declaration = + declarationFor( + parameter.getParameter().getType(), + context.getRequiredTestClass(), + "parameter " + parameter.getParameter().getName()); + return container(declaration, context); } - private static Object resolve(Class wanted, Class testClass, String target) { - List> matches = matching(wanted, testClass); + private static Declaration declarationFor(Class wanted, Class testClass, String target) { + List matches = matching(wanted, testClass); if (matches.size() != 1) { throw new ParameterResolutionException( String.format( "%s: %s of type %s matches %d fixtures; declare exactly one fixture for it", testClass.getSimpleName(), target, wanted.getSimpleName(), matches.size())); } - return matches.get(0).getOrStart(ContainerSuite.getInstance()); + return matches.get(0); + } + + // A private container lives in the store of the class or test that owns it; it is AutoCloseable, + // so JUnit closes it with that context. Stores inherit from parent contexts, so a CLASS container + // also resolves inside the class's tests. + private static BaseContainer container(Declaration declaration, ExtensionContext context) { + ContainerSpec spec = declaration.spec(); + if (declaration.scope() == Scope.RUN) { + return spec.getOrStart(ContainerSuite.getInstance()); + } + ExtensionContext owner = + declaration.scope() == Scope.CLASS ? classContext(context) : methodContext(context, spec); + return owner + .getStore(Namespace.create(ContainerExtension.class, spec)) + .computeIfAbsent( + PRIVATE_CONTAINER, + key -> spec.startPrivate(ContainerSuite.getInstance()), + BaseContainer.class); + } + + // Instance post-processing and test methods may run in a method context; a CLASS container + // belongs to the class. + private static ExtensionContext classContext(ExtensionContext context) { + ExtensionContext current = context; + while (current.getTestMethod().isPresent()) { + current = current.getParent().orElseThrow(); + } + return current; + } + + private static ExtensionContext methodContext(ExtensionContext context, ContainerSpec spec) { + if (context.getTestMethod().isEmpty()) { + throw methodScopeOutsideTest(context.getRequiredTestClass(), spec); + } + return context; } - private static List> matching(Class wanted, Class testClass) { - List> matches = new ArrayList<>(); - for (ContainerSpec spec : specs(testClass)) { - if (wanted.isAssignableFrom(spec.containerType())) { - matches.add(spec); + private static ExtensionConfigurationException methodScopeOutsideTest( + Class testClass, ContainerSpec spec) { + return new ExtensionConfigurationException( + String.format( + "%s: %s has Scope.METHOD, so only instance fields and test and @BeforeEach methods" + + " can receive its container", + testClass.getSimpleName(), spec)); + } + + private static List matching(Class wanted, Class testClass) { + List matches = new ArrayList<>(); + for (Declaration declaration : declarations(testClass)) { + if (wanted.isAssignableFrom(declaration.spec().containerType())) { + matches.add(declaration); } } return matches; } + static List> specs(Class testClass) { + return declarations(testClass).stream().map(Declaration::spec).toList(); + } + // Fixtures on the class and its superclasses, nearest class first, one per annotation type. // Hierarchies such as TestHive3HMS extends TestHive2HMS re-declare a fixture with another // version; keeping both would start two containers and make injection by type ambiguous. - static List> specs(Class testClass) { - List> specs = new ArrayList<>(); + private static List declarations(Class testClass) { + List declarations = new ArrayList<>(); Set> seen = new HashSet<>(); for (Class c = testClass; c != null && c != Object.class; c = c.getSuperclass()) { for (Annotation annotation : c.getDeclaredAnnotations()) { ContainerFixture fixture = annotation.annotationType().getAnnotation(ContainerFixture.class); if (fixture != null && seen.add(annotation.annotationType())) { - specs.add(specFor(fixture, annotation)); + declarations.add(declare(fixture, annotation)); } } } - return specs; + return declarations; } @SuppressWarnings({"unchecked", "rawtypes"}) - private static ContainerSpec specFor(ContainerFixture fixture, Annotation annotation) { + private static Declaration declare(ContainerFixture fixture, Annotation annotation) { + ContainerFixture.SpecFactory factory; try { Constructor> constructor = fixture.value().getDeclaredConstructor(); constructor.setAccessible(true); - return ((ContainerFixture.SpecFactory) constructor.newInstance()).specFor(annotation); + factory = constructor.newInstance(); } catch (ReflectiveOperationException e) { throw new IllegalStateException( "Cannot create the spec factory " + fixture.value().getName(), e); } + return new Declaration(factory.specFor(annotation), factory.scopeFor(annotation)); } } diff --git a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/ContainerFixture.java b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/ContainerFixture.java index 8015d8fec94..de4edf4443f 100644 --- a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/ContainerFixture.java +++ b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/ContainerFixture.java @@ -27,7 +27,8 @@ /** * Marks an annotation as a container fixture, such as {@link KafkaFixture}, and names how to turn - * it into a {@link ContainerSpec}. {@link ContainerExtension} reads every fixture on a test class. + * it into a {@link ContainerSpec}. {@link ContainerExtension} reads the nearest fixture of each + * annotation type on a test class and its superclasses. */ @Documented @Target(ElementType.ANNOTATION_TYPE) @@ -55,5 +56,15 @@ interface SpecFactory { * @return the spec */ ContainerSpec specFor(A fixture); + + /** + * Returns how long the fixture's container lives. + * + * @param fixture the fixture annotation on the test class + * @return the scope; {@link Scope#RUN} unless the fixture has a scope attribute + */ + default Scope scopeFor(A fixture) { + return Scope.RUN; + } } } diff --git a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/ContainerSpec.java b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/ContainerSpec.java index 1a90757c5bd..34a58ccf078 100644 --- a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/ContainerSpec.java +++ b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/ContainerSpec.java @@ -45,4 +45,18 @@ public interface ContainerSpec { * @return the running container */ C getOrStart(ContainerSuite suite); + + /** + * Starts a new container for this spec that only the caller uses, for fixtures with {@link + * Scope#CLASS} or {@link Scope#METHOD}. The caller closes it; it never replaces the shared + * container. + * + * @param suite the shared container suite, which starts the container on its network + * @return the running private container + * @throws UnsupportedOperationException if this spec has no private containers yet + */ + default C startPrivate(ContainerSuite suite) { + throw new UnsupportedOperationException( + "scope CLASS/METHOD is not supported for " + getClass().getSimpleName() + " yet"); + } } diff --git a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/ContainerSuite.java b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/ContainerSuite.java index 3a50781363e..6327b94f61d 100644 --- a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/ContainerSuite.java +++ b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/ContainerSuite.java @@ -46,6 +46,9 @@ import java.util.NoSuchElementException; import java.util.Objects; import java.util.Set; +import java.util.function.Consumer; +import java.util.function.Function; +import javax.annotation.Nullable; import org.apache.gravitino.integration.test.container.DatabaseContainer.Durability; import org.apache.gravitino.integration.test.util.CloseableGroup; import org.apache.gravitino.integration.test.util.CommandExecutor; @@ -152,21 +155,13 @@ public Network getNetwork() { } public void startHiveContainer(Map env) { - ImmutableMap.Builder builder = ImmutableMap.builder(); - builder.putAll(env); - builder.put("HADOOP_USER_NAME", "anonymous"); - synchronized (ContainerSuite.class) { hiveContainer.getOrStart( hiveVersion(env), () -> { initIfNecessary(); - HiveContainer.Builder hiveBuilder = - HiveContainer.builder() - .withHostName("gravitino-ci-hive") - .withEnvVars(builder.build()) - .withNetwork(network); - HiveContainer container = closer.register(hiveBuilder.build()); + HiveContainer container = + closer.register(hiveBuilder(env).withNetwork(network).build()); start(container); return container; }); @@ -244,19 +239,58 @@ public void startKerberosHiveContainer(Map envVars) { hiveVersion(envVars), () -> { initIfNecessary(); - HiveContainer.Builder hiveBuilder = - HiveContainer.builder() - .withHostName("gravitino-ci-kerberos-hive") - .withKerberosEnabled(true) - .withEnvVars(envVars) - .withNetwork(network); - HiveContainer container = closer.register(hiveBuilder.build()); + HiveContainer container = + closer.register(kerberosHiveBuilder(envVars).withNetwork(network).build()); start(container); return container; }); } } + // The shared Hive and a private one (HiveSpec.startPrivate) are built the same way. + static HiveContainer.Builder hiveBuilder(Map env) { + return HiveContainer.builder() + .withHostName("gravitino-ci-hive") + .withEnvVars( + ImmutableMap.builder() + .putAll(env) + .put("HADOOP_USER_NAME", "anonymous") + .build()); + } + + static HiveContainer.Builder kerberosHiveBuilder(Map env) { + return HiveContainer.builder() + .withHostName("gravitino-ci-kerberos-hive") + .withKerberosEnabled(true) + .withEnvVars(env); + } + + /** + * Starts a container that only the caller uses, for fixtures with {@link Scope#CLASS} or {@link + * Scope#METHOD}. It starts like a shared container, on the shared network, but no field keeps it, + * so it never replaces a shared container. The caller closes it; the suite also closes it at the + * end of the run in case the caller does not. + * + * @param build builds the container on the given network, which is null when there is none + * @param the container type + * @return the running container + */ + public C startPrivate(Function build) { + initIfNecessary(); + return startPrivate(network, build, ContainerSuite::start); + } + + // Tests pass their own start so that no Docker and no disk cleanup run. + static C startPrivate( + @Nullable Network network, Function build, Consumer start) { + C container = build.apply(network); + synchronized (ContainerSuite.class) { + closer.register(container); + } + start.accept(container); + return container; + } + public void startKerberosHiveContainer() { startKerberosHiveContainer(ImmutableMap.of()); } diff --git a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/HiveFixture.java b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/HiveFixture.java index 5174356ee2c..86a28fc275b 100644 --- a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/HiveFixture.java +++ b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/HiveFixture.java @@ -30,7 +30,9 @@ * Declares that a test class needs a Hive server. Test and lifecycle methods receive it as a {@link * HiveContainer} parameter, and fields marked {@link FixtureContainer} receive it too. * - *

A subclass may declare its own {@code HiveFixture}; it replaces the superclass's. + *

A class gets one Hive: the annotation is not repeatable, and both authentication modes inject + * as {@link HiveContainer}. A subclass may declare its own {@code HiveFixture}; it replaces the + * superclass's. */ @Documented @Inherited @@ -53,4 +55,12 @@ * @return the authentication mode; none by default */ HiveAuth auth() default HiveAuth.NONE; + + /** + * Returns how long the Hive container lives. Use {@link Scope#CLASS} or {@link Scope#METHOD} for + * tests that change Hive in ways later tests would see, such as adding Kerberos principals. + * + * @return the scope; the whole run by default, shared with other classes + */ + Scope scope() default Scope.RUN; } diff --git a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/HiveSpec.java b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/HiveSpec.java index 04a106f821b..829088e0a71 100644 --- a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/HiveSpec.java +++ b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/HiveSpec.java @@ -24,9 +24,10 @@ * A Hive server of one version and authentication mode. * *

Plain and Kerberos Hive run side by side, but each holds one version at a time: asking for a - * different version stops the running Hive of that mode and starts the requested one (see {@link + * different version stops the running Hive of that mode and starts the requested one (see {@code * ContainerSlot}). Tests of one version therefore share a container, and a version switch costs a - * restart, as with direct {@link ContainerSuite} calls. + * restart, as with direct {@link ContainerSuite} calls. A private Hive ({@link Scope#CLASS} or + * {@link Scope#METHOD}) leaves the shared one alone. */ public record HiveSpec(HiveVersion version, HiveAuth auth) implements ContainerSpec { @@ -37,23 +38,46 @@ public Class containerType() { @Override public HiveContainer getOrStart(ContainerSuite suite) { - Map env = Map.of(HiveContainer.HIVE_RUNTIME_VERSION, version.runtimeVersion()); return switch (auth) { case NONE -> { - suite.startHiveContainer(env); + suite.startHiveContainer(env()); yield suite.getHiveContainer(); } case KERBEROS -> { - suite.startKerberosHiveContainer(env); + suite.startKerberosHiveContainer(env()); yield suite.getKerberosHiveContainer(); } }; } + // Keeps the shared Hive's host name: the Kerberos image derives its principals from it. On macOS + // the shared network then has two containers with one alias, which is fine because tests reach + // containers by IP; on Linux CI there is no shared network. + @Override + public HiveContainer startPrivate(ContainerSuite suite) { + return suite.startPrivate(network -> builder().withNetwork(network).build()); + } + + private HiveContainer.Builder builder() { + return switch (auth) { + case NONE -> ContainerSuite.hiveBuilder(env()); + case KERBEROS -> ContainerSuite.kerberosHiveBuilder(env()); + }; + } + + private Map env() { + return Map.of(HiveContainer.HIVE_RUNTIME_VERSION, version.runtimeVersion()); + } + static final class Factory implements ContainerFixture.SpecFactory { @Override public HiveSpec specFor(HiveFixture fixture) { return new HiveSpec(fixture.version(), fixture.auth()); } + + @Override + public Scope scopeFor(HiveFixture fixture) { + return fixture.scope(); + } } } diff --git a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/Scope.java b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/Scope.java new file mode 100644 index 00000000000..e647e24739c --- /dev/null +++ b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/Scope.java @@ -0,0 +1,40 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.integration.test.container; + +/** + * How long a fixture's container lives, and who shares it. + * + *

Use a narrower scope only for tests that change or break their container, such as tests that + * add Kerberos principals: a private container costs a full start. + */ +public enum Scope { + /** One container for the whole run, shared by every class that asks for the same spec. */ + RUN, + /** + * A private container for one test class, closed after the class. It never replaces the shared + * container. + */ + CLASS, + /** + * A private container for each test method, closed after the test. It never replaces the shared + * container. + */ + METHOD +} diff --git a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/TestClickHouseSpec.java b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/TestClickHouseSpec.java new file mode 100644 index 00000000000..2cacc78102d --- /dev/null +++ b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/TestClickHouseSpec.java @@ -0,0 +1,137 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.integration.test.container; + +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.List; +import org.apache.gravitino.integration.test.util.TestDatabaseName; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; +import org.mockito.ArgumentCaptor; +import org.mockito.InOrder; +import org.mockito.Mockito; + +/** Tests {@link ClickHouseSpec} and {@link ClickHouseFixture} without Docker. */ +public class TestClickHouseSpec { + + private static final TestDatabaseName DATABASE = + TestDatabaseName.CLICKHOUSE_CATALOG_CLICKHOUSE_IT; + + @Test + void testSingleNodeFixtureMapsToTheSpec() { + Assertions.assertEquals( + List.of(new ClickHouseSpec(1, DATABASE)), + ContainerExtension.specs(SingleNodeFixture.class)); + } + + @Test + void testClusterFixtureMapsToTheSpec() { + Assertions.assertEquals( + List.of(new ClickHouseSpec(3, TestDatabaseName.CLICKHOUSE_CLUSTER_CLICKHOUSE_IT)), + ContainerExtension.specs(ClusterFixture.class)); + } + + @Test + void testContainerType() { + Assertions.assertEquals( + ClickHouseContainer.class, new ClickHouseSpec(1, DATABASE).containerType()); + } + + @Test + void testSingleNodeStartsWithTheDatabase() { + ContainerSuite suite = Mockito.mock(ContainerSuite.class); + ClickHouseContainer clickHouse = Mockito.mock(ClickHouseContainer.class); + Mockito.when(suite.getClickHouseContainer()).thenReturn(clickHouse); + + Assertions.assertSame(clickHouse, new ClickHouseSpec(1, DATABASE).getOrStart(suite)); + + InOrder order = Mockito.inOrder(suite); + order.verify(suite).startClickHouseContainer(DATABASE); + order.verify(suite).getClickHouseContainer(); + Mockito.verify(suite, Mockito.never()) + .startClickHouseClusterContainer(Mockito.any(), Mockito.anyString()); + } + + @Test + void testClusterStartsWithTheDatabaseAndTheTemplate() throws Exception { + ContainerSuite suite = Mockito.mock(ContainerSuite.class); + ClickHouseContainer firstNode = Mockito.mock(ClickHouseContainer.class); + Mockito.when(suite.getClickHouseClusterContainer()).thenReturn(firstNode); + + Assertions.assertSame(firstNode, new ClickHouseSpec(3, DATABASE).getOrStart(suite)); + + ArgumentCaptor templatePath = ArgumentCaptor.forClass(String.class); + InOrder order = Mockito.inOrder(suite); + order + .verify(suite) + .startClickHouseClusterContainer(Mockito.eq(DATABASE), templatePath.capture()); + order.verify(suite).getClickHouseClusterContainer(); + Mockito.verify(suite, Mockito.never()).startClickHouseContainer(Mockito.any()); + + Path template = Paths.get(templatePath.getValue()); + Assertions.assertTrue(Files.isReadable(template)); + Assertions.assertTrue( + Files.readString(template, StandardCharsets.UTF_8).contains("${ZOOKEEPER_HOST}")); + } + + @Test + void testClusterTemplateIsCopiedOnce() { + ContainerSuite suite = Mockito.mock(ContainerSuite.class); + ClickHouseSpec spec = new ClickHouseSpec(3, DATABASE); + + spec.getOrStart(suite); + spec.getOrStart(suite); + + ArgumentCaptor templatePath = ArgumentCaptor.forClass(String.class); + Mockito.verify(suite, Mockito.times(2)) + .startClickHouseClusterContainer(Mockito.eq(DATABASE), templatePath.capture()); + Assertions.assertEquals(templatePath.getAllValues().get(0), templatePath.getAllValues().get(1)); + } + + @Test + void testOtherNodeCountsAreRejected() { + IllegalArgumentException e = + Assertions.assertThrows( + IllegalArgumentException.class, () -> new ClickHouseSpec(2, DATABASE)); + Assertions.assertTrue(e.getMessage().contains("1 or 3"), e.getMessage()); + } + + @Test + void testFixtureWithAnUnsupportedNodeCountIsRejected() { + Assertions.assertThrows( + IllegalArgumentException.class, () -> ContainerExtension.specs(TwoNodeFixture.class)); + } + + // The build excludes this tag, so Gradle never runs the fixture classes on their own. + @Tag("gravitino-test-fixture") + @ClickHouseFixture(database = TestDatabaseName.CLICKHOUSE_CATALOG_CLICKHOUSE_IT) + static class SingleNodeFixture {} + + @Tag("gravitino-test-fixture") + @ClickHouseFixture(nodes = 3, database = TestDatabaseName.CLICKHOUSE_CLUSTER_CLICKHOUSE_IT) + static class ClusterFixture {} + + @Tag("gravitino-test-fixture") + @ClickHouseFixture(nodes = 2, database = TestDatabaseName.CLICKHOUSE_CATALOG_CLICKHOUSE_IT) + static class TwoNodeFixture {} +} diff --git a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/TestContainerExtension.java b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/TestContainerExtension.java index aef0251905f..a18d68c7053 100644 --- a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/TestContainerExtension.java +++ b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/TestContainerExtension.java @@ -29,6 +29,7 @@ import java.lang.annotation.RetentionPolicy; import java.lang.annotation.Target; import java.util.ArrayList; +import java.util.Collections; import java.util.List; import java.util.Optional; import org.junit.jupiter.api.Assertions; @@ -55,10 +56,14 @@ public class TestContainerExtension { private static final List LOG = new ArrayList<>(); private static final FakeContainer FIRST = Mockito.mock(FakeContainer.class); private static final FakeContainer SECOND = Mockito.mock(FakeContainer.class); + private static final OtherContainer OTHER = Mockito.mock(OtherContainer.class); + // Every private container FakeSpec.startPrivate returned, in start order. + private static final List PRIVATE = new ArrayList<>(); @BeforeEach void clearLog() { LOG.clear(); + PRIVATE.clear(); } @Test @@ -146,6 +151,76 @@ void testSpecsReadsEveryFixture() { ContainerExtension.specs(HiveFixtureOnly.class)); } + @Test + void testClassScopeStartsOnePrivateContainerForTheClass() { + run(ClassScopeFixture.class) + .testEvents() + .assertStatistics(stats -> stats.succeeded(2).failed(0)); + + Assertions.assertEquals(List.of("startPrivate first"), LOG); + Assertions.assertEquals(1, PRIVATE.size()); + Mockito.verify(PRIVATE.get(0), Mockito.times(1)).close(); + } + + @Test + void testMethodScopeStartsOnePrivateContainerPerTest() { + run(MethodScopeFixture.class) + .testEvents() + .assertStatistics(stats -> stats.succeeded(2).failed(0)); + + Assertions.assertEquals( + List.of("startPrivate first", "test", "startPrivate first", "test"), LOG); + Assertions.assertEquals(2, PRIVATE.size()); + Assertions.assertNotSame(PRIVATE.get(0), PRIVATE.get(1)); + for (FakeContainer container : PRIVATE) { + Mockito.verify(container, Mockito.times(1)).close(); + } + } + + @Test + void testRunAndClassScopeFixturesResolveToDifferentContainers() { + run(MixedScopeFixture.class) + .testEvents() + .assertStatistics(stats -> stats.succeeded(1).failed(0)); + + Assertions.assertEquals(1, Collections.frequency(LOG, "startPrivate first"), LOG.toString()); + Assertions.assertFalse(LOG.contains("start first"), LOG.toString()); + } + + @Test + void testMethodScopeStaticFieldFailsClearly() { + run(MethodScopeStaticFieldFixture.class) + .allEvents() + .assertThatEvents() + .haveExactly( + 1, + event( + finishedWithFailure( + instanceOf(ExtensionConfigurationException.class), + message(m -> m.contains("Scope.METHOD"))))); + Assertions.assertTrue(PRIVATE.isEmpty()); + } + + @Test + void testMethodScopeBeforeAllParameterFailsClearly() { + run(MethodScopeBeforeAllFixture.class) + .allEvents() + .assertThatEvents() + .haveExactly(1, event(finishedWithFailure(message(m -> m.contains("Scope.METHOD"))))); + Assertions.assertTrue(PRIVATE.isEmpty()); + } + + @Test + void testStartPrivateIsUnsupportedByDefault() { + UnsupportedOperationException e = + Assertions.assertThrows( + UnsupportedOperationException.class, + () -> new KafkaSpec().startPrivate(ContainerSuite.getInstance())); + + Assertions.assertEquals( + "scope CLASS/METHOD is not supported for KafkaSpec yet", e.getMessage()); + } + private static EngineExecutionResults run(Class fixture) { return EngineTestKit.engine("junit-jupiter") .selectors(DiscoverySelectors.selectClass(fixture)) @@ -171,6 +246,34 @@ public FakeContainer getOrStart(ContainerSuite suite) { LOG.add("start " + name); return name.equals("first") ? FIRST : SECOND; } + + @Override + public FakeContainer startPrivate(ContainerSuite suite) { + LOG.add("startPrivate " + name); + FakeContainer container = Mockito.mock(FakeContainer.class); + PRIVATE.add(container); + return container; + } + } + + // Only ever mocked, so the constructor never runs. + abstract static class OtherContainer extends BaseContainer { + OtherContainer() { + super((String) null, null, null, null, null, null, Optional.empty()); + } + } + + record OtherSpec() implements ContainerSpec { + @Override + public Class containerType() { + return OtherContainer.class; + } + + @Override + public OtherContainer getOrStart(ContainerSuite suite) { + LOG.add("start other"); + return OTHER; + } } static final class FirstFactory implements ContainerFixture.SpecFactory { @@ -178,6 +281,18 @@ static final class FirstFactory implements ContainerFixture.SpecFactory specFor(FakeFirstFixture fixture) { return new FakeSpec(fixture.value()); } + + @Override + public Scope scopeFor(FakeFirstFixture fixture) { + return fixture.scope(); + } + } + + static final class OtherFactory implements ContainerFixture.SpecFactory { + @Override + public ContainerSpec specFor(FakeOtherFixture fixture) { + return new OtherSpec(); + } } static final class SecondFactory implements ContainerFixture.SpecFactory { @@ -193,6 +308,8 @@ public ContainerSpec specFor(FakeSecondFixture fixture) { @ContainerFixture(FirstFactory.class) @interface FakeFirstFixture { String value() default "first"; + + Scope scope() default Scope.RUN; } @Target(ElementType.TYPE) @@ -201,6 +318,12 @@ public ContainerSpec specFor(FakeSecondFixture fixture) { @ContainerFixture(SecondFactory.class) @interface FakeSecondFixture {} + @Target(ElementType.TYPE) + @Retention(RetentionPolicy.RUNTIME) + @ExtendWith(ContainerExtension.class) + @ContainerFixture(OtherFactory.class) + @interface FakeOtherFixture {} + @Tag(FIXTURE) @FakeFirstFixture @TestInstance(TestInstance.Lifecycle.PER_CLASS) @@ -311,4 +434,89 @@ static class KafkaFixtureOnly {} @Tag(FIXTURE) @HiveFixture(version = HiveVersion.HIVE3, auth = HiveAuth.KERBEROS) static class HiveFixtureOnly {} + + @Tag(FIXTURE) + @FakeFirstFixture(scope = Scope.CLASS) + static class ClassScopeFixture { + @FixtureContainer private static FakeContainer staticField; + @FixtureContainer private FakeContainer instanceField; + + @BeforeAll + static void beforeAll(FakeContainer parameter) { + Assertions.assertNotSame(FIRST, parameter); + Assertions.assertSame(staticField, parameter); + } + + @Test + void first(FakeContainer parameter) { + assertAllArePrivate(parameter); + } + + @Test + void second(FakeContainer parameter) { + assertAllArePrivate(parameter); + } + + private void assertAllArePrivate(FakeContainer parameter) { + Assertions.assertSame(PRIVATE.get(0), staticField); + Assertions.assertSame(staticField, instanceField); + Assertions.assertSame(staticField, parameter); + } + } + + // One instance for both tests, so the field must be refilled before each test. + @Tag(FIXTURE) + @FakeFirstFixture(scope = Scope.METHOD) + @TestInstance(TestInstance.Lifecycle.PER_CLASS) + static class MethodScopeFixture { + @FixtureContainer private FakeContainer instanceField; + + @Test + void first(FakeContainer parameter) { + assertCurrent(parameter); + } + + @Test + void second(FakeContainer parameter) { + assertCurrent(parameter); + } + + private void assertCurrent(FakeContainer parameter) { + Assertions.assertSame(PRIVATE.get(PRIVATE.size() - 1), instanceField); + Assertions.assertSame(instanceField, parameter); + LOG.add("test"); + } + } + + @Tag(FIXTURE) + @FakeFirstFixture(scope = Scope.CLASS) + @FakeOtherFixture + static class MixedScopeFixture { + @Test + void test(FakeContainer fake, OtherContainer other) { + Assertions.assertSame(PRIVATE.get(0), fake); + Assertions.assertSame(OTHER, other); + } + } + + @Tag(FIXTURE) + @FakeFirstFixture(scope = Scope.METHOD) + static class MethodScopeStaticFieldFixture { + @SuppressWarnings("UnusedVariable") + @FixtureContainer + private static FakeContainer staticField; + + @Test + void test() {} + } + + @Tag(FIXTURE) + @FakeFirstFixture(scope = Scope.METHOD) + static class MethodScopeBeforeAllFixture { + @BeforeAll + static void beforeAll(FakeContainer parameter) {} + + @Test + void test() {} + } } diff --git a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/TestContainerSuite.java b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/TestContainerSuite.java index 0f8c0319363..896d6c5934a 100644 --- a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/TestContainerSuite.java +++ b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/TestContainerSuite.java @@ -142,6 +142,34 @@ void testStartTimedReportsStartFailureWhenCloseThrowsAnError() { Assertions.assertArrayEquals(new Throwable[] {closeFailure}, thrown.getSuppressed()); } + @Test + void testStartPrivateStartsAndRegistersWithoutKeepingTheContainer() throws Exception { + BaseContainer container = mockContainer(); + + BaseContainer started = + ContainerSuite.startPrivate(null, network -> container, ContainerSuite::startTimed); + + Assertions.assertSame(container, started); + Mockito.verify(container, Mockito.times(1)).start(); + for (Field field : ContainerSuite.class.getDeclaredFields()) { + if (!Modifier.isStatic(field.getModifiers()) || field.getName().equals("closer")) { + continue; + } + field.setAccessible(true); + Object value = field.get(null); + Assertions.assertNotSame(container, value, field.getName()); + if (value instanceof Map) { + Assertions.assertFalse(((Map) value).containsValue(container), field.getName()); + } else if (value instanceof Collection) { + Assertions.assertFalse(((Collection) value).contains(container), field.getName()); + } + } + + // Registered with the closer, so the end of the run closes it even if the caller does not. + ContainerSuite.getInstance().close(); + Mockito.verify(container).close(); + } + private static BaseContainer mockContainer() { GenericContainer genericContainer = Mockito.mock(GenericContainer.class); Mockito.when(genericContainer.getDockerImageName()).thenReturn("test/image:1.0"); diff --git a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/TestZooKeeperSpec.java b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/TestZooKeeperSpec.java new file mode 100644 index 00000000000..f239624cad8 --- /dev/null +++ b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/TestZooKeeperSpec.java @@ -0,0 +1,59 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.integration.test.container; + +import java.util.List; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; +import org.mockito.InOrder; +import org.mockito.Mockito; + +/** Tests {@link ZooKeeperSpec} and {@link ZooKeeperFixture} without Docker. */ +public class TestZooKeeperSpec { + + @Test + void testFixtureMapsToTheZooKeeperSpec() { + Assertions.assertEquals( + List.of(new ZooKeeperSpec()), ContainerExtension.specs(ZooKeeperFixtureOnly.class)); + } + + @Test + void testContainerType() { + Assertions.assertEquals(ZooKeeperContainer.class, new ZooKeeperSpec().containerType()); + } + + @Test + void testGetOrStartStartsThroughTheSuite() { + ContainerSuite suite = Mockito.mock(ContainerSuite.class); + ZooKeeperContainer zooKeeper = Mockito.mock(ZooKeeperContainer.class); + Mockito.when(suite.getZooKeeperContainer()).thenReturn(zooKeeper); + + Assertions.assertSame(zooKeeper, new ZooKeeperSpec().getOrStart(suite)); + + InOrder order = Mockito.inOrder(suite); + order.verify(suite).startZooKeeperContainer(); + order.verify(suite).getZooKeeperContainer(); + } + + // The build excludes this tag, so Gradle never runs the fixture class on its own. + @Tag("gravitino-test-fixture") + @ZooKeeperFixture + static class ZooKeeperFixtureOnly {} +} diff --git a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/ZooKeeperFixture.java b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/ZooKeeperFixture.java new file mode 100644 index 00000000000..9f1cd540b77 --- /dev/null +++ b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/ZooKeeperFixture.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.integration.test.container; + +import java.lang.annotation.Documented; +import java.lang.annotation.ElementType; +import java.lang.annotation.Inherited; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; +import org.junit.jupiter.api.extension.ExtendWith; + +/** + * Declares that a test class needs the ZooKeeper server. Test and lifecycle methods receive it as a + * {@link ZooKeeperContainer} parameter, and fields marked {@link FixtureContainer} receive it too. + */ +@Documented +@Inherited +@Target(ElementType.TYPE) +@Retention(RetentionPolicy.RUNTIME) +@ExtendWith(ContainerExtension.class) +@ContainerFixture(ZooKeeperSpec.Factory.class) +public @interface ZooKeeperFixture {} diff --git a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/ZooKeeperSpec.java b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/ZooKeeperSpec.java new file mode 100644 index 00000000000..e567e60f542 --- /dev/null +++ b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/ZooKeeperSpec.java @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.integration.test.container; + +/** The ZooKeeper server. There is one configuration, so the spec has no fields. */ +public record ZooKeeperSpec() implements ContainerSpec { + + @Override + public Class containerType() { + return ZooKeeperContainer.class; + } + + @Override + public ZooKeeperContainer getOrStart(ContainerSuite suite) { + suite.startZooKeeperContainer(); + return suite.getZooKeeperContainer(); + } + + static final class Factory implements ContainerFixture.SpecFactory { + @Override + public ZooKeeperSpec specFor(ZooKeeperFixture fixture) { + return new ZooKeeperSpec(); + } + } +} diff --git a/catalogs-contrib/catalog-jdbc-clickhouse/src/test/resources/remote_servers.xml b/integration-test-common/src/test/resources/remote_servers.xml similarity index 100% rename from catalogs-contrib/catalog-jdbc-clickhouse/src/test/resources/remote_servers.xml rename to integration-test-common/src/test/resources/remote_servers.xml diff --git a/spark-connector/spark-common/src/test/java/org/apache/gravitino/spark/connector/integration/test/SparkEnvIT.java b/spark-connector/spark-common/src/test/java/org/apache/gravitino/spark/connector/integration/test/SparkEnvIT.java index e5191a7d2b9..cd7b1c7b005 100644 --- a/spark-connector/spark-common/src/test/java/org/apache/gravitino/spark/connector/integration/test/SparkEnvIT.java +++ b/spark-connector/spark-common/src/test/java/org/apache/gravitino/spark/connector/integration/test/SparkEnvIT.java @@ -37,8 +37,9 @@ import org.apache.gravitino.function.FunctionParam; import org.apache.gravitino.function.FunctionParams; import org.apache.gravitino.function.FunctionType; -import org.apache.gravitino.integration.test.container.ContainerSuite; +import org.apache.gravitino.integration.test.container.FixtureContainer; import org.apache.gravitino.integration.test.container.HiveContainer; +import org.apache.gravitino.integration.test.container.HiveFixture; import org.apache.gravitino.rel.types.Types; import org.apache.gravitino.spark.connector.GravitinoSparkConfig; import org.apache.gravitino.spark.connector.functions.StringLengthFunction; @@ -56,10 +57,11 @@ import org.slf4j.LoggerFactory; /** Setup Hive, Gravitino, Spark, Metalake environment to execute SparkSQL. */ +@HiveFixture public abstract class SparkEnvIT extends SparkUtilIT { private static final Logger LOG = LoggerFactory.getLogger(SparkEnvIT.class); - private static final ContainerSuite containerSuite = ContainerSuite.getInstance(); + @FixtureContainer private HiveContainer hive; protected static final String icebergRestServiceName = "iceberg-rest"; protected String hiveMetastoreUri = "thrift://127.0.0.1:9083"; @@ -236,17 +238,13 @@ private void initGravitinoEnv() { } private void initHiveEnv() { - containerSuite.startHiveContainer(); hiveMetastoreUri = String.format( - "thrift://%s:%d", - containerSuite.getHiveContainer().getContainerIpAddress(), - HiveContainer.HIVE_METASTORE_PORT); + "thrift://%s:%d", hive.getContainerIpAddress(), HiveContainer.HIVE_METASTORE_PORT); warehouse = String.format( "hdfs://%s:%d/user/hive/warehouse", - containerSuite.getHiveContainer().getContainerIpAddress(), - HiveContainer.HDFS_DEFAULTFS_PORT); + hive.getContainerIpAddress(), HiveContainer.HDFS_DEFAULTFS_PORT); } protected void initCatalogEnv() throws Exception {} @@ -292,9 +290,7 @@ private void initHdfsFileSystem() { conf.set( "fs.defaultFS", String.format( - "hdfs://%s:%d", - containerSuite.getHiveContainer().getContainerIpAddress(), - HiveContainer.HDFS_DEFAULTFS_PORT)); + "hdfs://%s:%d", hive.getContainerIpAddress(), HiveContainer.HDFS_DEFAULTFS_PORT)); try { hdfs = FileSystem.get(conf); } catch (IOException e) { diff --git a/trino-connector/integration-test/src/test/java/org/apache/gravitino/trino/connector/integration/test/TrinoConnectorIT.java b/trino-connector/integration-test/src/test/java/org/apache/gravitino/trino/connector/integration/test/TrinoConnectorIT.java index 442cc1b9572..3f4584c109b 100644 --- a/trino-connector/integration-test/src/test/java/org/apache/gravitino/trino/connector/integration/test/TrinoConnectorIT.java +++ b/trino-connector/integration-test/src/test/java/org/apache/gravitino/trino/connector/integration/test/TrinoConnectorIT.java @@ -40,7 +40,9 @@ import org.apache.gravitino.catalog.lakehouse.iceberg.IcebergConstants; import org.apache.gravitino.client.GravitinoMetalake; import org.apache.gravitino.integration.test.container.ContainerSuite; +import org.apache.gravitino.integration.test.container.FixtureContainer; import org.apache.gravitino.integration.test.container.HiveContainer; +import org.apache.gravitino.integration.test.container.HiveFixture; import org.apache.gravitino.integration.test.container.TrinoContainer; import org.apache.gravitino.integration.test.util.BaseIT; import org.apache.gravitino.integration.test.util.GravitinoITUtils; @@ -76,11 +78,14 @@ @Disabled @Deprecated @Tag("gravitino-docker-test") +@HiveFixture public class TrinoConnectorIT extends BaseIT { public static final Logger LOG = LoggerFactory.getLogger(TrinoConnectorIT.class); private static final ContainerSuite containerSuite = ContainerSuite.getInstance(); + @FixtureContainer private static HiveContainer hive; + public static String metalakeName = GravitinoITUtils.genRandomName("TrinoIT_metalake").toLowerCase(); public static String catalogName = @@ -114,15 +119,11 @@ public class TrinoConnectorIT extends BaseIT { public void startDockerContainer() throws TException, InterruptedException { String trinoConfDir = System.getenv("TRINO_CONF_DIR"); - containerSuite.startHiveContainer(); - // Initial Hive client HiveConf hiveConf = new HiveConf(); String hiveMetastoreUris = String.format( - "thrift://%s:%d", - containerSuite.getHiveContainer().getContainerIpAddress(), - HiveContainer.HIVE_METASTORE_PORT); + "thrift://%s:%d", hive.getContainerIpAddress(), HiveContainer.HIVE_METASTORE_PORT); hiveConf.set(HiveConf.ConfVars.METASTOREURIS.varname, hiveMetastoreUris); // Currently must first create metalake and catalog then start Trino container @@ -158,7 +159,7 @@ public static void createSchema() throws TException, InterruptedException { + ")", catalogName, databaseName, - containerSuite.getHiveContainer().getContainerIpAddress(), + hive.getContainerIpAddress(), HiveContainer.HDFS_DEFAULTFS_PORT, databaseName); containerSuite.getTrinoContainer().executeUpdateSQL(sql1); @@ -463,8 +464,7 @@ void testHiveSchemaCreatedByGravitino() { "metastore.uris", String.format( "thrift://%s:%s", - containerSuite.getHiveContainer().getContainerIpAddress(), - HiveContainer.HIVE_METASTORE_PORT)) + hive.getContainerIpAddress(), HiveContainer.HIVE_METASTORE_PORT)) .build()); catalog @@ -602,7 +602,7 @@ void testColumnTypeNotNullByTrino() { String catalogName = GravitinoITUtils.genRandomName("mysql_catalog").toLowerCase(); GravitinoMetalake createdMetalake = client.loadMetalake(metalakeName); grantRootAccessToHiveMysql(); - String hiveHost = containerSuite.getHiveContainer().getContainerIpAddress(); + String hiveHost = hive.getContainerIpAddress(); createdMetalake.createCatalog( catalogName, @@ -693,8 +693,7 @@ void testHiveTableCreatedByGravitino() { "metastore.uris", String.format( "thrift://%s:%s", - containerSuite.getHiveContainer().getContainerIpAddress(), - HiveContainer.HIVE_METASTORE_PORT)) + hive.getContainerIpAddress(), HiveContainer.HIVE_METASTORE_PORT)) .build()); Schema schema = @@ -824,8 +823,7 @@ void testHiveCatalogCreatedByGravitino() { "metastore.uris", String.format( "thrift://%s:%s", - containerSuite.getHiveContainer().getContainerIpAddress(), - HiveContainer.HIVE_METASTORE_PORT)) + hive.getContainerIpAddress(), HiveContainer.HIVE_METASTORE_PORT)) .put("trino.bypass.hive.immutable-partitions", "true") .put("trino.bypass.hive.target-max-file-size", "1GB") .put("trino.bypass.hive.create-empty-bucket-files", "true") @@ -866,8 +864,7 @@ void testWrongHiveCatalogProperty() { "metastore.uris", String.format( "thrift://%s:%s", - containerSuite.getHiveContainer().getContainerIpAddress(), - HiveContainer.HIVE_METASTORE_PORT)) + hive.getContainerIpAddress(), HiveContainer.HIVE_METASTORE_PORT)) .put("trino.bypass.hive.immutable-partitions", "true") // it should be like '1GB, 1MB', we make it wrong purposely. .put("trino.bypass.hive.target-max-file-size", "xxxx") @@ -898,7 +895,7 @@ void testIcebergTableAndSchemaCreatedByGravitino() { String tableName = GravitinoITUtils.genRandomName("table").toLowerCase(); GravitinoMetalake createdMetalake = client.loadMetalake(metalakeName); - String hiveContainerIp = containerSuite.getHiveContainer().getContainerIpAddress(); + String hiveContainerIp = hive.getContainerIpAddress(); Catalog catalog = createdMetalake.createCatalog( catalogName, @@ -1002,7 +999,7 @@ void testIcebergCatalogCreatedByGravitino() { String schemaName = GravitinoITUtils.genRandomName("iceberg_catalog").toLowerCase(); GravitinoMetalake createdMetalake = client.loadMetalake(metalakeName); grantRootAccessToHiveMysql(); - String hiveHost = containerSuite.getHiveContainer().getContainerIpAddress(); + String hiveHost = hive.getContainerIpAddress(); createdMetalake.createCatalog( catalogName, @@ -1075,7 +1072,7 @@ void testMySQLCatalogCreatedByGravitino() { String catalogName = GravitinoITUtils.genRandomName("mysql_catalog").toLowerCase(); GravitinoMetalake createdMetalake = client.loadMetalake(metalakeName); grantRootAccessToHiveMysql(); - String hiveHost = containerSuite.getHiveContainer().getContainerIpAddress(); + String hiveHost = hive.getContainerIpAddress(); createdMetalake.createCatalog( catalogName, @@ -1108,7 +1105,7 @@ void testMySQLTableCreatedByGravitino() { String tableName = GravitinoITUtils.genRandomName("mysql_table").toLowerCase(); GravitinoMetalake createdMetalake = client.loadMetalake(metalakeName); grantRootAccessToHiveMysql(); - String hiveHost = containerSuite.getHiveContainer().getContainerIpAddress(); + String hiveHost = hive.getContainerIpAddress(); createdMetalake.createCatalog( catalogName, @@ -1206,7 +1203,7 @@ void testMySQLTableCreatedByTrino() { String tableName = GravitinoITUtils.genRandomName("mysql_table").toLowerCase(); GravitinoMetalake createdMetalake = client.loadMetalake(metalakeName); grantRootAccessToHiveMysql(); - String hiveHost = containerSuite.getHiveContainer().getContainerIpAddress(); + String hiveHost = hive.getContainerIpAddress(); createdMetalake.createCatalog( catalogName, @@ -1314,7 +1311,7 @@ void testDropCatalogAndCreateAgain() { String catalogName = GravitinoITUtils.genRandomName("mysql_catalog").toLowerCase(); GravitinoMetalake createdMetalake = client.loadMetalake(metalakeName); grantRootAccessToHiveMysql(); - String hiveHost = containerSuite.getHiveContainer().getContainerIpAddress(); + String hiveHost = hive.getContainerIpAddress(); // Create the catalog and drop it for 3 times to test the create/drop catalog function work // well. @@ -1359,9 +1356,7 @@ private static void createCatalog() { Map properties = Maps.newHashMap(); String hiveMetastoreUris = String.format( - "thrift://%s:%d", - containerSuite.getHiveContainer().getContainerIpAddress(), - HiveContainer.HIVE_METASTORE_PORT); + "thrift://%s:%d", hive.getContainerIpAddress(), HiveContainer.HIVE_METASTORE_PORT); LOG.debug("hiveMetastoreUris is {}", hiveMetastoreUris); properties.put("metastore.uris", hiveMetastoreUris); @@ -1377,9 +1372,7 @@ private static void createCatalog() { // log in from localhost there, so the grant runs as `iceberg`, which the image creates for any // host with the grant option. private static void grantRootAccessToHiveMysql() { - String url = - String.format( - "jdbc:mysql://%s:3306/", containerSuite.getHiveContainer().getContainerIpAddress()); + String url = String.format("jdbc:mysql://%s:3306/", hive.getContainerIpAddress()); try (Connection connection = DriverManager.getConnection(url, "iceberg", "iceberg"); Statement statement = connection.createStatement()) { statement.execute("GRANT ALL PRIVILEGES ON *.* TO root@'%' IDENTIFIED BY 'ds123'"); From c8efd35bc4093cc3ae97e3154cbeb1e3da0c866a Mon Sep 17 00:00:00 2001 From: Nevin Zheng Date: Mon, 19 Oct 2026 16:09:29 -0700 Subject: [PATCH 12/14] [#14002] improvement(test): Add field fixtures, startup, MySQL and PostgreSQL fixtures (part 4/6) - Review follow-ups from the first fixture changes. - A fixture annotation on a field picks that field's container. - startup = EAGER | LAZY; a warning for fixtures no test injects. - MySQL and PostgreSQL fixtures; SparkQueryRunner starts Hive through its spec. - Only the run's closer closes the container suite. - Split container start time into pull, start and readiness; give the Hive metastore check a real time budget. --- .../test/GvfsMultipleClusterIT.java | 28 ++- .../test/container/BaseContainer.java | 24 ++- .../test/container/ClickHouseFixture.java | 9 +- .../test/container/ClickHouseSpec.java | 5 + .../test/container/ContainerExtension.java | 109 ++++++++++- .../test/container/ContainerFixture.java | 10 ++ .../test/container/ContainerSuite.java | 65 +++++-- .../test/container/DorisFixture.java | 9 +- .../integration/test/container/DorisSpec.java | 5 + .../test/container/FixtureContainer.java | 10 ++ .../test/container/HiveFixture.java | 16 +- .../integration/test/container/HiveSpec.java | 5 + .../test/container/KafkaFixture.java | 12 +- .../integration/test/container/KafkaSpec.java | 5 + .../test/container/LocalStackFixture.java | 12 +- .../test/container/LocalStackSpec.java | 5 + .../test/container/MotoFixture.java | 12 +- .../integration/test/container/MotoSpec.java | 5 + .../test/container/MySQLFixture.java | 71 ++++++++ .../integration/test/container/MySQLSpec.java | 67 +++++++ .../test/container/MySQLVersion.java | 27 +++ .../test/container/OceanBaseFixture.java | 12 +- .../test/container/OceanBaseSpec.java | 5 + .../test/container/PostgreSQLFixture.java | 64 +++++++ .../test/container/PostgreSQLSpec.java | 59 ++++++ .../test/container/RangerFixture.java | 12 +- .../test/container/RangerSpec.java | 5 + .../test/container/RustFSFixture.java | 12 +- .../test/container/RustFSSpec.java | 5 + .../test/container/StarRocksFixture.java | 12 +- .../test/container/StarRocksSpec.java | 5 + .../integration/test/container/Startup.java | 35 ++++ .../container/TestContainerExtension.java | 169 +++++++++++++++++- .../test/container/ZooKeeperFixture.java | 12 +- .../test/container/ZooKeeperSpec.java | 5 + .../test/util/CloseContainerExtension.java | 24 +-- .../test/sql/SparkQueryRunner.java | 18 +- 37 files changed, 873 insertions(+), 92 deletions(-) create mode 100644 integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/MySQLFixture.java create mode 100644 integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/MySQLSpec.java create mode 100644 integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/MySQLVersion.java create mode 100644 integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/PostgreSQLFixture.java create mode 100644 integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/PostgreSQLSpec.java create mode 100644 integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/Startup.java diff --git a/clients/filesystem-hadoop3/src/test/java/org/apache/gravitino/filesystem/hadoop/integration/test/GvfsMultipleClusterIT.java b/clients/filesystem-hadoop3/src/test/java/org/apache/gravitino/filesystem/hadoop/integration/test/GvfsMultipleClusterIT.java index 2833088639d..0d8042cee96 100644 --- a/clients/filesystem-hadoop3/src/test/java/org/apache/gravitino/filesystem/hadoop/integration/test/GvfsMultipleClusterIT.java +++ b/clients/filesystem-hadoop3/src/test/java/org/apache/gravitino/filesystem/hadoop/integration/test/GvfsMultipleClusterIT.java @@ -35,8 +35,10 @@ import org.apache.gravitino.NameIdentifier; import org.apache.gravitino.client.GravitinoMetalake; import org.apache.gravitino.file.Fileset; -import org.apache.gravitino.integration.test.container.ContainerSuite; +import org.apache.gravitino.integration.test.container.FixtureContainer; +import org.apache.gravitino.integration.test.container.HiveAuth; import org.apache.gravitino.integration.test.container.HiveContainer; +import org.apache.gravitino.integration.test.container.HiveFixture; import org.apache.gravitino.integration.test.util.BaseIT; import org.apache.gravitino.integration.test.util.GravitinoITUtils; import org.apache.hadoop.conf.Configuration; @@ -62,9 +64,11 @@ @DisabledIf("org.apache.gravitino.integration.test.util.ITUtils#isEmbedded") public class GvfsMultipleClusterIT extends BaseIT { private static final Logger LOG = LoggerFactory.getLogger(GvfsMultipleClusterIT.class); - protected static final ContainerSuite containerSuite = ContainerSuite.getInstance(); - protected HiveContainer hiveContainer; - protected HiveContainer kerberosHiveContainer; + @HiveFixture @FixtureContainer private HiveContainer hive; + + @HiveFixture(auth = HiveAuth.KERBEROS) + @FixtureContainer + private HiveContainer kerberosHive; protected String metalakeName = GravitinoITUtils.genRandomName("gvfs_it_metalake"); protected String catalogName = GravitinoITUtils.genRandomName("catalog"); @@ -76,12 +80,6 @@ public class GvfsMultipleClusterIT extends BaseIT { @BeforeAll public void startUp() throws Exception { - containerSuite.startHiveContainer(); - hiveContainer = containerSuite.getHiveContainer(); - - containerSuite.startKerberosHiveContainer(); - kerberosHiveContainer = containerSuite.getKerberosHiveContainer(); - setupKerberosEnv(); Assertions.assertFalse(client.metalakeExists(metalakeName)); @@ -112,14 +110,14 @@ private void setupKerberosEnv() throws Exception { // Keytab of the Gravitino SDK client String keytabPath = tmpDir + "/admin.keytab"; - kerberosHiveContainer.getContainer().copyFileFromContainer("/etc/admin.keytab", keytabPath); + kerberosHive.getContainer().copyFileFromContainer("/etc/admin.keytab", keytabPath); String tmpKrb5Path = tmpDir + "/krb5.conf_tmp"; String krb5Path = tmpDir + "/krb5.conf"; - kerberosHiveContainer.getContainer().copyFileFromContainer("/etc/krb5.conf", tmpKrb5Path); + kerberosHive.getContainer().copyFileFromContainer("/etc/krb5.conf", tmpKrb5Path); // Modify the krb5.conf and change the kdc and admin_server to the container IP - String ip = containerSuite.getKerberosHiveContainer().getContainerIpAddress(); + String ip = kerberosHive.getContainerIpAddress(); String content = FileUtils.readFileToString(new File(tmpKrb5Path), StandardCharsets.UTF_8); content = content.replace("kdc = localhost:88", "kdc = " + ip + ":88"); content = content.replace("admin_server = localhost", "admin_server = " + ip + ":749"); @@ -209,7 +207,7 @@ public void testFsOperation() throws IOException { String normalFilesetName = GravitinoITUtils.genRandomName("fileset_normal"); NameIdentifier normalFilesetIdent = NameIdentifier.of(schemaName, normalFilesetName); Catalog catalog = metalake.loadCatalog(catalogName); - String location = baseHdfsPath(hiveContainer.getContainerIpAddress(), normalFilesetName); + String location = baseHdfsPath(hive.getContainerIpAddress(), normalFilesetName); catalog .asFilesetCatalog() .createMultipleLocationFileset( @@ -223,7 +221,7 @@ public void testFsOperation() throws IOException { // create a fileset with kerberos cluster String kerberosFilesetName = GravitinoITUtils.genRandomName("fileset_kerberos"); NameIdentifier kerberosFilesetIdent = NameIdentifier.of(schemaName, kerberosFilesetName); - location = baseHdfsPath(kerberosHiveContainer.getContainerIpAddress(), kerberosFilesetName); + location = baseHdfsPath(kerberosHive.getContainerIpAddress(), kerberosFilesetName); String configResources = configResourcesPath + "/core-site.xml," + configResourcesPath + "/hdfs-site.xml"; catalog diff --git a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/BaseContainer.java b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/BaseContainer.java index ff0cb8666b0..d10d4000316 100644 --- a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/BaseContainer.java +++ b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/BaseContainer.java @@ -51,8 +51,9 @@ /** * The base for all integration test containers: it applies the common settings and manages the * lifecycle. Gravitino's own images have no official Testcontainers module, so they start from an - * image name; services that have an official module supply that module instead, so they keep its - * defaults and readiness check. + * image name; services that have an official module supply that module instead. A supplied module + * keeps its own readiness check only when the subclass also returns {@code null} from {@link + * #waitStrategy()}; otherwise the default port check replaces it. */ public abstract class BaseContainer implements Startable { public static final Logger LOG = LoggerFactory.getLogger(BaseContainer.class); @@ -71,6 +72,8 @@ public abstract class BaseContainer implements Startable { protected final GenericContainer container; + private volatile long lastContainerStartMs; + /** * Builds the container from an image name. Used by Gravitino's own images, which have no official * Testcontainers module. @@ -222,7 +225,22 @@ public String getContainerIpAddress() { @Override public void start() { - container.start(); + long startNanos = System.nanoTime(); + try { + container.start(); + } finally { + lastContainerStartMs = (System.nanoTime() - startNanos) / 1_000_000; + } + } + + /** + * Returns how long the last {@code start()} spent in Testcontainers' own start: creating and + * starting the container and its wait strategy, without this class's own readiness check. + * + * @return the time in milliseconds, or 0 before the first start + */ + public long lastContainerStartMs() { + return lastContainerStartMs; } @Override diff --git a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/ClickHouseFixture.java b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/ClickHouseFixture.java index 7ee4d194f3a..7bc188ccd14 100644 --- a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/ClickHouseFixture.java +++ b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/ClickHouseFixture.java @@ -43,7 +43,7 @@ */ @Documented @Inherited -@Target(ElementType.TYPE) +@Target({ElementType.TYPE, ElementType.FIELD}) @Retention(RetentionPolicy.RUNTIME) @ExtendWith(ContainerExtension.class) @ContainerFixture(ClickHouseSpec.Factory.class) @@ -62,4 +62,11 @@ * @return the test's database */ TestDatabaseName database(); + + /** + * Returns when the container starts. + * + * @return the startup mode; before {@code @BeforeAll} by default + */ + Startup startup() default Startup.EAGER; } diff --git a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/ClickHouseSpec.java b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/ClickHouseSpec.java index 2d49ff13093..5a80dc30891 100644 --- a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/ClickHouseSpec.java +++ b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/ClickHouseSpec.java @@ -97,6 +97,11 @@ private static Path copyRemoteServersTemplate() { } static final class Factory implements ContainerFixture.SpecFactory { + @Override + public Startup startupFor(ClickHouseFixture fixture) { + return fixture.startup(); + } + @Override public ClickHouseSpec specFor(ClickHouseFixture fixture) { return new ClickHouseSpec(fixture.nodes(), fixture.database()); diff --git a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/ContainerExtension.java b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/ContainerExtension.java index be04a34c6bd..ae423590cd0 100644 --- a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/ContainerExtension.java +++ b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/ContainerExtension.java @@ -24,10 +24,14 @@ import java.util.ArrayList; import java.util.HashSet; import java.util.List; +import java.util.Optional; import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.function.Consumer; import java.util.function.Predicate; import javax.annotation.Nullable; import org.apache.gravitino.integration.test.util.CloseContainerExtension; +import org.junit.jupiter.api.extension.AfterAllCallback; import org.junit.jupiter.api.extension.BeforeAllCallback; import org.junit.jupiter.api.extension.BeforeEachCallback; import org.junit.jupiter.api.extension.ExtensionConfigurationException; @@ -40,6 +44,8 @@ import org.junit.platform.commons.support.AnnotationSupport; import org.junit.platform.commons.support.HierarchyTraversalMode; import org.junit.platform.commons.support.ModifierSupport; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * Starts the containers a test class declares with fixture annotations, such as {@link @@ -53,6 +59,9 @@ * the same annotation type, so {@code @HiveFixture(version = HIVE3)} on a subclass of a Hive2 test * runs Hive3 only. * + *

A fixture annotation on a {@link FixtureContainer} field picks that field's container instead + * of the class's fixtures, so one class can hold two containers of one type. + * *

By default ({@link Scope#RUN}) each container starts once per run and is shared by every class * that asks for the same spec; the containers close when the run ends, through {@link * CloseContainerExtension}. A {@link Scope#CLASS} container is private to the class and closes @@ -60,38 +69,83 @@ * only reach instance fields and the parameters of test and {@code @BeforeEach} methods. */ public class ContainerExtension - implements BeforeAllCallback, BeforeEachCallback, ParameterResolver, TestInstancePostProcessor { + implements AfterAllCallback, + BeforeAllCallback, + BeforeEachCallback, + ParameterResolver, + TestInstancePostProcessor { private static final String PRIVATE_CONTAINER = "privateContainer"; + private static final String INJECTED = "injectedSpecs"; + private static final Logger LOG = LoggerFactory.getLogger(ContainerExtension.class); + + // Where unused-fixture warnings go; a test swaps it to read them. + static Consumer warnings = LOG::warn; private final CloseContainerExtension suiteCloser = new CloseContainerExtension(); /** A fixture on the test class: what it asks for and how long its container lives. */ - record Declaration(ContainerSpec spec, Scope scope) {} + record Declaration(ContainerSpec spec, Scope scope, Startup startup) {} @Override public void beforeAll(ExtensionContext context) { suiteCloser.beforeAll(context); Class testClass = context.getRequiredTestClass(); for (Declaration declaration : declarations(testClass)) { - if (declaration.scope() != Scope.METHOD) { + if (declaration.scope() != Scope.METHOD && declaration.startup() == Startup.EAGER) { container(declaration, context); } } injectFields(testClass, null, ModifierSupport::isStatic, context, false); + // JUnit registers an extension declared only on an instance field after it post-processes the + // instance, so postProcessTestInstance() never ran for it. A PER_CLASS instance exists here. + context + .getTestInstance() + .ifPresent( + instance -> + injectFields( + instance.getClass(), instance, ModifierSupport::isNotStatic, context, false)); + } + + // A fixture whose container no field or parameter received still started; say so, since it + // usually means a leftover fixture or a test that reaches the container another way. + @Override + public void afterAll(ExtensionContext context) { + Set> injected = injected(context); + for (Declaration declaration : declarations(context.getRequiredTestClass())) { + if (!injected.contains(declaration.spec())) { + warnings.accept( + String.format( + "%s declares a fixture for %s but never injects its container; inject it into a" + + " field or parameter, or remove the fixture", + context.getRequiredTestClass().getSimpleName(), declaration.spec())); + } + } + } + + @SuppressWarnings("unchecked") + private static Set> injected(ExtensionContext context) { + return classContext(context) + .getStore(Namespace.create(ContainerExtension.class)) + .computeIfAbsent(INJECTED, key -> ConcurrentHashMap.newKeySet(), Set.class); } @Override public void beforeEach(ExtensionContext context) { + // An extension declared only on an instance field of a PER_METHOD class never gets + // beforeAll(), so register the run's closer here too; registering twice is harmless. + suiteCloser.beforeAll(context); Class testClass = context.getRequiredTestClass(); for (Declaration declaration : declarations(testClass)) { if (declaration.scope() == Scope.METHOD) { container(declaration, context); } } + Object instance = context.getRequiredTestInstance(); + // An extension declared only on an instance field missed postProcessTestInstance(). + injectFields(testClass, instance, ModifierSupport::isNotStatic, context, false); // Each test gets a new METHOD container, so refill the fields that hold one. - injectFields( - testClass, context.getRequiredTestInstance(), ModifierSupport::isNotStatic, context, true); + injectFields(testClass, instance, ModifierSupport::isNotStatic, context, true); } // Runs when the instance is created, before beforeAll(), so instance fields are filled before an @@ -119,7 +173,9 @@ private static void injectFields( testClass.getSimpleName(), field.getName())); } Declaration declaration = - declarationFor(field.getType(), testClass, "field " + field.getName()); + fieldDeclaration(field) + .orElseGet( + () -> declarationFor(field.getType(), testClass, "field " + field.getName())); if (declaration.scope() == Scope.METHOD && ModifierSupport.isStatic(field)) { throw methodScopeOutsideTest(testClass, declaration.spec()); } @@ -128,13 +184,50 @@ private static void injectFields( } try { field.setAccessible(true); + // A RUN or CLASS container does not change within an instance, so a filled instance field + // stays; several callbacks below may each be the first to see the instance. + if (!methodScoped && testInstance != null && field.get(testInstance) != null) { + continue; + } field.set(testInstance, container(declaration, context)); + injected(context).add(declaration.spec()); } catch (IllegalAccessException e) { throw new ExtensionConfigurationException("Cannot set field " + field.getName(), e); } } } + // A fixture annotation on the field itself, which wins over the class's fixtures. + private static Optional fieldDeclaration(Field field) { + List found = new ArrayList<>(); + for (Annotation annotation : field.getDeclaredAnnotations()) { + ContainerFixture fixture = annotation.annotationType().getAnnotation(ContainerFixture.class); + if (fixture != null) { + found.add(declare(fixture, annotation)); + } + } + if (found.size() > 1) { + throw new ExtensionConfigurationException( + String.format( + "%s: field %s has %d fixture annotations; use one", + field.getDeclaringClass().getSimpleName(), field.getName(), found.size())); + } + if (found.isEmpty()) { + return Optional.empty(); + } + Class containerType = found.get(0).spec().containerType(); + if (!field.getType().isAssignableFrom(containerType)) { + throw new ExtensionConfigurationException( + String.format( + "%s: field %s of type %s cannot hold a %s", + field.getDeclaringClass().getSimpleName(), + field.getName(), + field.getType().getSimpleName(), + containerType.getSimpleName())); + } + return Optional.of(found.get(0)); + } + @Override public boolean supportsParameter(ParameterContext parameter, ExtensionContext context) { return !matching(parameter.getParameter().getType(), context.getRequiredTestClass()).isEmpty(); @@ -147,6 +240,7 @@ public Object resolveParameter(ParameterContext parameter, ExtensionContext cont parameter.getParameter().getType(), context.getRequiredTestClass(), "parameter " + parameter.getParameter().getName()); + injected(context).add(declaration.spec()); return container(declaration, context); } @@ -249,6 +343,7 @@ private static Declaration declare(ContainerFixture fixture, Annotation annotati throw new IllegalStateException( "Cannot create the spec factory " + fixture.value().getName(), e); } - return new Declaration(factory.specFor(annotation), factory.scopeFor(annotation)); + return new Declaration( + factory.specFor(annotation), factory.scopeFor(annotation), factory.startupFor(annotation)); } } diff --git a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/ContainerFixture.java b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/ContainerFixture.java index de4edf4443f..2bc517d5621 100644 --- a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/ContainerFixture.java +++ b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/ContainerFixture.java @@ -66,5 +66,15 @@ interface SpecFactory { default Scope scopeFor(A fixture) { return Scope.RUN; } + + /** + * Returns when the fixture's container starts. + * + * @param fixture the fixture annotation on the test class + * @return the startup mode; {@link Startup#EAGER} unless the fixture has a startup attribute + */ + default Startup startupFor(A fixture) { + return Startup.EAGER; + } } } diff --git a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/ContainerSuite.java b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/ContainerSuite.java index 6327b94f61d..3502d644e13 100644 --- a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/ContainerSuite.java +++ b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/ContainerSuite.java @@ -26,7 +26,6 @@ import com.github.dockerjava.api.model.Network.Ipam.Config; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; -import java.io.Closeable; import java.io.File; import java.io.IOException; import java.io.InputStream; @@ -46,6 +45,8 @@ import java.util.NoSuchElementException; import java.util.Objects; import java.util.Set; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Future; import java.util.function.Consumer; import java.util.function.Function; import javax.annotation.Nullable; @@ -55,12 +56,14 @@ import org.apache.gravitino.integration.test.util.ITUtils; import org.apache.gravitino.integration.test.util.ProcessData; import org.apache.gravitino.integration.test.util.TestDatabaseName; +import org.junit.jupiter.api.extension.ExtensionContext; +import org.junit.jupiter.api.extension.ExtensionContext.Namespace; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.testcontainers.DockerClientFactory; import org.testcontainers.containers.Network; -public class ContainerSuite implements Closeable { +public class ContainerSuite { public static final Logger LOG = LoggerFactory.getLogger(ContainerSuite.class); private static volatile ContainerSuite instance = null; private static volatile boolean initialized = false; @@ -869,24 +872,50 @@ private static void start(BaseContainer container) { // another one beside it. The closer still holds it; closing it again at run end is a no-op. static void startTimed(BaseContainer container) { long startNanos = System.nanoTime(); + long pullMs = 0; boolean ok = false; try { + pullMs = pullImage(container); container.start(); ok = true; } catch (Throwable e) { closeFailedStart(container, e); throw e; } finally { + long totalMs = (System.nanoTime() - startNanos) / 1_000_000; + long containerMs = container.lastContainerStartMs(); LOG.info( - "[CONTAINER-TIMING] container={} image={} hostName={} ok={} startMs={}", + "[CONTAINER-TIMING] container={} image={} hostName={} ok={} startMs={} pullMs={}" + + " containerMs={} readyMs={}", container.getClass().getSimpleName(), container.getContainer().getDockerImageName(), container.hostName, ok, - (System.nanoTime() - startNanos) / 1_000_000); + totalMs, + pullMs, + containerMs, + Math.max(0, totalMs - pullMs - containerMs)); } } + // Resolves (and if needed pulls) the image before start(), so the timing line can report the pull + // apart from the start: CI runners pull every image fresh, a warm Mac does not. + private static long pullImage(BaseContainer container) { + long pullNanos = System.nanoTime(); + Future image = container.getContainer().getImage(); + if (image != null) { + try { + image.get(); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new IllegalStateException("Interrupted while pulling " + container.hostName, e); + } catch (ExecutionException e) { + throw new IllegalStateException("Failed to pull the image for " + container.hostName, e); + } + } + return (System.nanoTime() - pullNanos) / 1_000_000; + } + private static void closeFailedStart(BaseContainer container, Throwable startFailure) { LOG.warn( "Closing {} (image={}, hostName={}) after its start failed", @@ -1025,14 +1054,30 @@ private static long[] cidrToRange(String cidr) throws Exception { } /** - * Stops every container and resets the suite. Only {@code CloseContainerExtension} calls this, - * once at the end of the run; tests never close the shared suite, because later classes in the - * same JVM would have to restart every container. + * Closes the suite when the JUnit run ends, through a resource in the root store. Calling it + * again is harmless; the store keeps one resource. {@code CloseContainerExtension} calls it + * before every class, so the run's containers stop once, after the last class. * - * @throws IOException never; close failures are logged + * @param context any extension context of the run */ - @Override - public void close() throws IOException { + public static void closeWhenRunEnds(ExtensionContext context) { + // Store.computeIfAbsent is atomic, so concurrent classes register the closer once. + context.getRoot().getStore(Namespace.GLOBAL).computeIfAbsent(RunCloser.class); + } + + private static final class RunCloser implements AutoCloseable { + @Override + public void close() { + getInstance().close(); + LOG.info("Containers were closed at the end of the run"); + } + } + + // Stops every container and resets the suite; logs instead of failing, so a teardown error does + // not fail a green run. Package-private so tests cannot close the shared suite: later classes in + // the same JVM would have to restart every container. Only the run's closer above and this + // package's unit tests call it. + void close() { try { closer.close(); } catch (Exception e) { diff --git a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/DorisFixture.java b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/DorisFixture.java index e2cc1f8602a..bf53d0a54c1 100644 --- a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/DorisFixture.java +++ b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/DorisFixture.java @@ -32,7 +32,7 @@ */ @Documented @Inherited -@Target(ElementType.TYPE) +@Target({ElementType.TYPE, ElementType.FIELD}) @Retention(RetentionPolicy.RUNTIME) @ExtendWith(ContainerExtension.class) @ContainerFixture(DorisSpec.Factory.class) @@ -44,4 +44,11 @@ * @return the Doris image; 1.2 by default */ DorisImageName version() default DorisImageName.VERSION_1_2; + + /** + * Returns when the container starts. + * + * @return the startup mode; before {@code @BeforeAll} by default + */ + Startup startup() default Startup.EAGER; } diff --git a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/DorisSpec.java b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/DorisSpec.java index b82c5c13cac..1ec829d067e 100644 --- a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/DorisSpec.java +++ b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/DorisSpec.java @@ -36,6 +36,11 @@ public DorisContainer getOrStart(ContainerSuite suite) { } static final class Factory implements ContainerFixture.SpecFactory { + @Override + public Startup startupFor(DorisFixture fixture) { + return fixture.startup(); + } + @Override public DorisSpec specFor(DorisFixture fixture) { return new DorisSpec(fixture.version()); diff --git a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/FixtureContainer.java b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/FixtureContainer.java index b69561488f0..82b2e146eda 100644 --- a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/FixtureContainer.java +++ b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/FixtureContainer.java @@ -31,6 +31,16 @@ * *

Use it where a method parameter cannot reach, such as an overridden {@code @BeforeAll} method. * + *

A fixture annotation on the field itself picks that field's container, so one class can hold + * two containers of the same type. A field fixture does not register for method parameters. + * + * 

{@code
+ * class GvfsMultipleClusterIT extends BaseIT {
+ *   @HiveFixture @FixtureContainer private HiveContainer hive;
+ *   @HiveFixture(auth = HiveAuth.KERBEROS) @FixtureContainer private HiveContainer kerberosHive;
+ * }
+ * }
+ * * 
{@code
  * @KafkaFixture
  * class OwnerIT extends BaseIT {
diff --git a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/HiveFixture.java b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/HiveFixture.java
index 86a28fc275b..cfd1af83d3c 100644
--- a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/HiveFixture.java
+++ b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/HiveFixture.java
@@ -30,13 +30,14 @@
  * Declares that a test class needs a Hive server. Test and lifecycle methods receive it as a {@link
  * HiveContainer} parameter, and fields marked {@link FixtureContainer} receive it too.
  *
- * 
A class gets one Hive: the annotation is not repeatable, and both authentication modes inject
- * as {@link HiveContainer}. A subclass may declare its own {@code HiveFixture}; it replaces the
- * superclass's.
+ * 
On the class, it declares one Hive: the annotation is not repeatable, and both authentication
+ * modes inject as {@link HiveContainer}. A subclass may declare its own {@code HiveFixture}; it
+ * replaces the superclass's. A class that needs two Hives, such as a plain and a Kerberos one, puts
+ * the annotation on each {@link FixtureContainer} field instead.
  */
 @Documented
 @Inherited
-@Target(ElementType.TYPE)
+@Target({ElementType.TYPE, ElementType.FIELD})
 @Retention(RetentionPolicy.RUNTIME)
 @ExtendWith(ContainerExtension.class)
 @ContainerFixture(HiveSpec.Factory.class)
@@ -63,4 +64,11 @@
    * @return the scope; the whole run by default, shared with other classes
    */
   Scope scope() default Scope.RUN;
+
+  /**
+   * Returns when the container starts.
+   *
+   * @return the startup mode; before {@code @BeforeAll} by default
+   */
+  Startup startup() default Startup.EAGER;
 }
diff --git a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/HiveSpec.java b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/HiveSpec.java
index 829088e0a71..192e8e1b51e 100644
--- a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/HiveSpec.java
+++ b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/HiveSpec.java
@@ -70,6 +70,11 @@ private Map env() {
   }
 
   static final class Factory implements ContainerFixture.SpecFactory {
+    @Override
+    public Startup startupFor(HiveFixture fixture) {
+      return fixture.startup();
+    }
+
     @Override
     public HiveSpec specFor(HiveFixture fixture) {
       return new HiveSpec(fixture.version(), fixture.auth());
diff --git a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/KafkaFixture.java b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/KafkaFixture.java
index d2352630133..c8a1a2a7228 100644
--- a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/KafkaFixture.java
+++ b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/KafkaFixture.java
@@ -40,8 +40,16 @@
  */
 @Documented
 @Inherited
-@Target(ElementType.TYPE)
+@Target({ElementType.TYPE, ElementType.FIELD})
 @Retention(RetentionPolicy.RUNTIME)
 @ExtendWith(ContainerExtension.class)
 @ContainerFixture(KafkaSpec.Factory.class)
-public @interface KafkaFixture {}
+public @interface KafkaFixture {
+
+  /**
+   * Returns when the container starts.
+   *
+   * @return the startup mode; before {@code @BeforeAll} by default
+   */
+  Startup startup() default Startup.EAGER;
+}
diff --git a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/KafkaSpec.java b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/KafkaSpec.java
index 7ce2f989f73..a5a13de6520 100644
--- a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/KafkaSpec.java
+++ b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/KafkaSpec.java
@@ -33,6 +33,11 @@ public KafkaContainer getOrStart(ContainerSuite suite) {
   }
 
   static final class Factory implements ContainerFixture.SpecFactory {
+    @Override
+    public Startup startupFor(KafkaFixture fixture) {
+      return fixture.startup();
+    }
+
     @Override
     public KafkaSpec specFor(KafkaFixture fixture) {
       return new KafkaSpec();
diff --git a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/LocalStackFixture.java b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/LocalStackFixture.java
index 5301a060dfd..e9e5680ef47 100644
--- a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/LocalStackFixture.java
+++ b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/LocalStackFixture.java
@@ -33,8 +33,16 @@
  */
 @Documented
 @Inherited
-@Target(ElementType.TYPE)
+@Target({ElementType.TYPE, ElementType.FIELD})
 @Retention(RetentionPolicy.RUNTIME)
 @ExtendWith(ContainerExtension.class)
 @ContainerFixture(LocalStackSpec.Factory.class)
-public @interface LocalStackFixture {}
+public @interface LocalStackFixture {
+
+  /**
+   * Returns when the container starts.
+   *
+   * @return the startup mode; before {@code @BeforeAll} by default
+   */
+  Startup startup() default Startup.EAGER;
+}
diff --git a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/LocalStackSpec.java b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/LocalStackSpec.java
index 6a66d8dd059..15a95f4803c 100644
--- a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/LocalStackSpec.java
+++ b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/LocalStackSpec.java
@@ -33,6 +33,11 @@ public GravitinoLocalStackContainer getOrStart(ContainerSuite suite) {
   }
 
   static final class Factory implements ContainerFixture.SpecFactory {
+    @Override
+    public Startup startupFor(LocalStackFixture fixture) {
+      return fixture.startup();
+    }
+
     @Override
     public LocalStackSpec specFor(LocalStackFixture fixture) {
       return new LocalStackSpec();
diff --git a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/MotoFixture.java b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/MotoFixture.java
index 28a96a4efa0..f6526f7c80d 100644
--- a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/MotoFixture.java
+++ b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/MotoFixture.java
@@ -40,8 +40,16 @@
  */
 @Documented
 @Inherited
-@Target(ElementType.TYPE)
+@Target({ElementType.TYPE, ElementType.FIELD})
 @Retention(RetentionPolicy.RUNTIME)
 @ExtendWith(ContainerExtension.class)
 @ContainerFixture(MotoSpec.Factory.class)
-public @interface MotoFixture {}
+public @interface MotoFixture {
+
+  /**
+   * Returns when the container starts.
+   *
+   * @return the startup mode; before {@code @BeforeAll} by default
+   */
+  Startup startup() default Startup.EAGER;
+}
diff --git a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/MotoSpec.java b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/MotoSpec.java
index 7cf7c3df29e..8ab3d61968c 100644
--- a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/MotoSpec.java
+++ b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/MotoSpec.java
@@ -33,6 +33,11 @@ public GravitinoMotoContainer getOrStart(ContainerSuite suite) {
   }
 
   static final class Factory implements ContainerFixture.SpecFactory {
+    @Override
+    public Startup startupFor(MotoFixture fixture) {
+      return fixture.startup();
+    }
+
     @Override
     public MotoSpec specFor(MotoFixture fixture) {
       return new MotoSpec();
diff --git a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/MySQLFixture.java b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/MySQLFixture.java
new file mode 100644
index 00000000000..6d37e4091b7
--- /dev/null
+++ b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/MySQLFixture.java
@@ -0,0 +1,71 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ *  http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.gravitino.integration.test.container;
+
+import java.lang.annotation.Documented;
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Inherited;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+import org.apache.gravitino.integration.test.util.TestDatabaseName;
+import org.junit.jupiter.api.extension.ExtendWith;
+
+/**
+ * Declares that a test class needs a MySQL server with its database. Test and lifecycle methods
+ * receive it as a {@link MySQLContainer} parameter, and fields marked {@link FixtureContainer}
+ * receive it too.
+ *
+ * 
{@code
+ * @MySQLFixture(database = TestDatabaseName.MYSQL_CATALOG_MYSQL_IT)
+ * class CatalogMysqlIT extends BaseIT {
+ *   @FixtureContainer private MySQLContainer mysql;
+ * }
+ * }

+ */
+@Documented
+@Inherited
+@Target({ElementType.TYPE, ElementType.FIELD})
+@Retention(RetentionPolicy.RUNTIME)
+@ExtendWith(ContainerExtension.class)
+@ContainerFixture(MySQLSpec.Factory.class)
+public @interface MySQLFixture {
+
+  /**
+   * Returns the MySQL version to run.
+   *
+   * @return the version; 8.0 by default
+   */
+  MySQLVersion version() default MySQLVersion.MYSQL_8_0;
+
+  /**
+   * Returns the database to create on the server before the test runs. Like {@link
+   * ClickHouseFixture#database()}, this stays until a database fixture owns test databases.
+   *
+   * @return the test's database
+   */
+  TestDatabaseName database();
+
+  /**
+   * Returns when the container starts.
+   *
+   * @return the startup mode; before {@code @BeforeAll} by default
+   */
+  Startup startup() default Startup.EAGER;
+}
diff --git a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/MySQLSpec.java b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/MySQLSpec.java
new file mode 100644
index 00000000000..0d8484c5409
--- /dev/null
+++ b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/MySQLSpec.java
@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ *  http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.gravitino.integration.test.container;
+
+import java.util.Objects;
+import org.apache.gravitino.integration.test.util.TestDatabaseName;
+
+/**
+ * A MySQL server of one version, with {@code database} created on it. Versions run side by side,
+ * each under its own host name; every class that asks for a version shares its server.
+ */
+public record MySQLSpec(MySQLVersion version, TestDatabaseName database)
+    implements ContainerSpec {
+
+  /** Checks that both parts are set. */
+  public MySQLSpec {
+    Objects.requireNonNull(version, "version");
+    Objects.requireNonNull(database, "database");
+  }
+
+  @Override
+  public Class containerType() {
+    return MySQLContainer.class;
+  }
+
+  @Override
+  public MySQLContainer getOrStart(ContainerSuite suite) {
+    return switch (version) {
+      case MYSQL_8_0 -> {
+        suite.startMySQLContainer(database);
+        yield suite.getMySQLContainer();
+      }
+      case MYSQL_5_7 -> {
+        suite.startMySQLVersion5Container(database);
+        yield suite.getMySQLVersion5Container();
+      }
+    };
+  }
+
+  static final class Factory implements ContainerFixture.SpecFactory {
+    @Override
+    public Startup startupFor(MySQLFixture fixture) {
+      return fixture.startup();
+    }
+
+    @Override
+    public MySQLSpec specFor(MySQLFixture fixture) {
+      return new MySQLSpec(fixture.version(), fixture.database());
+    }
+  }
+}
diff --git a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/MySQLVersion.java b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/MySQLVersion.java
new file mode 100644
index 00000000000..0ea5a0d1f7a
--- /dev/null
+++ b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/MySQLVersion.java
@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ *  http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.gravitino.integration.test.container;
+
+/** The MySQL versions tests can ask for with {@link MySQLFixture}. */
+public enum MySQLVersion {
+  /** MySQL 8.0, {@link TestImages#MYSQL}. */
+  MYSQL_8_0,
+  /** MySQL 5.7, {@link TestImages#MYSQL_5_7}. */
+  MYSQL_5_7
+}
diff --git a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/OceanBaseFixture.java b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/OceanBaseFixture.java
index 04d917f7d61..b1c5ad53025 100644
--- a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/OceanBaseFixture.java
+++ b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/OceanBaseFixture.java
@@ -32,8 +32,16 @@
  */
 @Documented
 @Inherited
-@Target(ElementType.TYPE)
+@Target({ElementType.TYPE, ElementType.FIELD})
 @Retention(RetentionPolicy.RUNTIME)
 @ExtendWith(ContainerExtension.class)
 @ContainerFixture(OceanBaseSpec.Factory.class)
-public @interface OceanBaseFixture {}
+public @interface OceanBaseFixture {
+
+  /**
+   * Returns when the container starts.
+   *
+   * @return the startup mode; before {@code @BeforeAll} by default
+   */
+  Startup startup() default Startup.EAGER;
+}
diff --git a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/OceanBaseSpec.java b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/OceanBaseSpec.java
index bb100b2e662..139f611a627 100644
--- a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/OceanBaseSpec.java
+++ b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/OceanBaseSpec.java
@@ -33,6 +33,11 @@ public OceanBaseContainer getOrStart(ContainerSuite suite) {
   }
 
   static final class Factory implements ContainerFixture.SpecFactory {
+    @Override
+    public Startup startupFor(OceanBaseFixture fixture) {
+      return fixture.startup();
+    }
+
     @Override
     public OceanBaseSpec specFor(OceanBaseFixture fixture) {
       return new OceanBaseSpec();
diff --git a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/PostgreSQLFixture.java b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/PostgreSQLFixture.java
new file mode 100644
index 00000000000..b993dece9e1
--- /dev/null
+++ b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/PostgreSQLFixture.java
@@ -0,0 +1,64 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ *  http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.gravitino.integration.test.container;
+
+import java.lang.annotation.Documented;
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Inherited;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+import org.apache.gravitino.integration.test.util.TestDatabaseName;
+import org.junit.jupiter.api.extension.ExtendWith;
+
+/**
+ * Declares that a test class needs a PostgreSQL server with its database. Test and lifecycle
+ * methods receive it as a {@link PostgreSQLContainer} parameter, and fields marked {@link
+ * FixtureContainer} receive it too.
+ */
+@Documented
+@Inherited
+@Target({ElementType.TYPE, ElementType.FIELD})
+@Retention(RetentionPolicy.RUNTIME)
+@ExtendWith(ContainerExtension.class)
+@ContainerFixture(PostgreSQLSpec.Factory.class)
+public @interface PostgreSQLFixture {
+
+  /**
+   * Returns the PostgreSQL version to run.
+   *
+   * @return the image; 13 by default
+   */
+  PGImageName version() default PGImageName.VERSION_13;
+
+  /**
+   * Returns the database to create on the server before the test runs. Like {@link
+   * ClickHouseFixture#database()}, this stays until a database fixture owns test databases.
+   *
+   * @return the test's database
+   */
+  TestDatabaseName database();
+
+  /**
+   * Returns when the container starts.
+   *
+   * @return the startup mode; before {@code @BeforeAll} by default
+   */
+  Startup startup() default Startup.EAGER;
+}
diff --git a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/PostgreSQLSpec.java b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/PostgreSQLSpec.java
new file mode 100644
index 00000000000..439bebd9ecd
--- /dev/null
+++ b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/PostgreSQLSpec.java
@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ *  http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.gravitino.integration.test.container;
+
+import java.util.Objects;
+import org.apache.gravitino.integration.test.util.TestDatabaseName;
+
+/**
+ * A PostgreSQL server of one version, with {@code database} created on it. Versions run side by
+ * side; every class that asks for a version shares its server.
+ */
+public record PostgreSQLSpec(PGImageName version, TestDatabaseName database)
+    implements ContainerSpec {
+
+  /** Checks that both parts are set. */
+  public PostgreSQLSpec {
+    Objects.requireNonNull(version, "version");
+    Objects.requireNonNull(database, "database");
+  }
+
+  @Override
+  public Class containerType() {
+    return PostgreSQLContainer.class;
+  }
+
+  @Override
+  public PostgreSQLContainer getOrStart(ContainerSuite suite) {
+    suite.startPostgreSQLContainer(database, version);
+    return suite.getPostgreSQLContainer(version);
+  }
+
+  static final class Factory implements ContainerFixture.SpecFactory {
+    @Override
+    public Startup startupFor(PostgreSQLFixture fixture) {
+      return fixture.startup();
+    }
+
+    @Override
+    public PostgreSQLSpec specFor(PostgreSQLFixture fixture) {
+      return new PostgreSQLSpec(fixture.version(), fixture.database());
+    }
+  }
+}
diff --git a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/RangerFixture.java b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/RangerFixture.java
index 895baa944e2..628e8c53c04 100644
--- a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/RangerFixture.java
+++ b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/RangerFixture.java
@@ -40,8 +40,16 @@
  */
 @Documented
 @Inherited
-@Target(ElementType.TYPE)
+@Target({ElementType.TYPE, ElementType.FIELD})
 @Retention(RetentionPolicy.RUNTIME)
 @ExtendWith(ContainerExtension.class)
 @ContainerFixture(RangerSpec.Factory.class)
-public @interface RangerFixture {}
+public @interface RangerFixture {
+
+  /**
+   * Returns when the container starts.
+   *
+   * @return the startup mode; before {@code @BeforeAll} by default
+   */
+  Startup startup() default Startup.EAGER;
+}
diff --git a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/RangerSpec.java b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/RangerSpec.java
index 419eafb454c..37edaa894cb 100644
--- a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/RangerSpec.java
+++ b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/RangerSpec.java
@@ -33,6 +33,11 @@ public RangerContainer getOrStart(ContainerSuite suite) {
   }
 
   static final class Factory implements ContainerFixture.SpecFactory {
+    @Override
+    public Startup startupFor(RangerFixture fixture) {
+      return fixture.startup();
+    }
+
     @Override
     public RangerSpec specFor(RangerFixture fixture) {
       return new RangerSpec();
diff --git a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/RustFSFixture.java b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/RustFSFixture.java
index 60a357e512b..e457d65b099 100644
--- a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/RustFSFixture.java
+++ b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/RustFSFixture.java
@@ -32,8 +32,16 @@
  */
 @Documented
 @Inherited
-@Target(ElementType.TYPE)
+@Target({ElementType.TYPE, ElementType.FIELD})
 @Retention(RetentionPolicy.RUNTIME)
 @ExtendWith(ContainerExtension.class)
 @ContainerFixture(RustFSSpec.Factory.class)
-public @interface RustFSFixture {}
+public @interface RustFSFixture {
+
+  /**
+   * Returns when the container starts.
+   *
+   * @return the startup mode; before {@code @BeforeAll} by default
+   */
+  Startup startup() default Startup.EAGER;
+}
diff --git a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/RustFSSpec.java b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/RustFSSpec.java
index b739bda6bbf..f7a82205cea 100644
--- a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/RustFSSpec.java
+++ b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/RustFSSpec.java
@@ -33,6 +33,11 @@ public RustFSContainer getOrStart(ContainerSuite suite) {
   }
 
   static final class Factory implements ContainerFixture.SpecFactory {
+    @Override
+    public Startup startupFor(RustFSFixture fixture) {
+      return fixture.startup();
+    }
+
     @Override
     public RustFSSpec specFor(RustFSFixture fixture) {
       return new RustFSSpec();
diff --git a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/StarRocksFixture.java b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/StarRocksFixture.java
index e0b79ed9f84..dd8f65b7e98 100644
--- a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/StarRocksFixture.java
+++ b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/StarRocksFixture.java
@@ -32,8 +32,16 @@
  */
 @Documented
 @Inherited
-@Target(ElementType.TYPE)
+@Target({ElementType.TYPE, ElementType.FIELD})
 @Retention(RetentionPolicy.RUNTIME)
 @ExtendWith(ContainerExtension.class)
 @ContainerFixture(StarRocksSpec.Factory.class)
-public @interface StarRocksFixture {}
+public @interface StarRocksFixture {
+
+  /**
+   * Returns when the container starts.
+   *
+   * @return the startup mode; before {@code @BeforeAll} by default
+   */
+  Startup startup() default Startup.EAGER;
+}
diff --git a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/StarRocksSpec.java b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/StarRocksSpec.java
index 19501562fa4..8711a1a724d 100644
--- a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/StarRocksSpec.java
+++ b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/StarRocksSpec.java
@@ -33,6 +33,11 @@ public StarRocksContainer getOrStart(ContainerSuite suite) {
   }
 
   static final class Factory implements ContainerFixture.SpecFactory {
+    @Override
+    public Startup startupFor(StarRocksFixture fixture) {
+      return fixture.startup();
+    }
+
     @Override
     public StarRocksSpec specFor(StarRocksFixture fixture) {
       return new StarRocksSpec();
diff --git a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/Startup.java b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/Startup.java
new file mode 100644
index 00000000000..d43705f4849
--- /dev/null
+++ b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/Startup.java
@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ *  http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.gravitino.integration.test.container;
+
+/**
+ * When a fixture's container starts.
+ *
+ * 
A field marked {@link FixtureContainer} is filled before {@code @BeforeAll}, so it starts its
+ * container then either way; {@link #LAZY} only defers containers that tests receive as parameters.
+ */
+public enum Startup {
+  /** Before the class's {@code @BeforeAll} methods, so start failures show up in class setup. */
+  EAGER,
+  /**
+   * When a test or lifecycle method first asks for it, for containers that need something a test
+   * sets up first, such as Trino needing the Gravitino server's port.
+   */
+  LAZY
+}
diff --git a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/TestContainerExtension.java b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/TestContainerExtension.java
index a18d68c7053..a628db9372f 100644
--- a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/TestContainerExtension.java
+++ b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/TestContainerExtension.java
@@ -32,6 +32,8 @@
 import java.util.Collections;
 import java.util.List;
 import java.util.Optional;
+import java.util.function.Consumer;
+import org.apache.gravitino.integration.test.util.TestDatabaseName;
 import org.junit.jupiter.api.Assertions;
 import org.junit.jupiter.api.BeforeAll;
 import org.junit.jupiter.api.BeforeEach;
@@ -98,6 +100,58 @@ void testStaticFieldIsFilledBeforeAStaticBeforeAll() {
         .assertStatistics(stats -> stats.succeeded(1).failed(0));
   }
 
+  @Test
+  void testFieldFixturesGiveOneClassTwoContainersOfOneType() {
+    run(TwoFieldFixtures.class)
+        .testEvents()
+        .assertStatistics(stats -> stats.succeeded(1).failed(0));
+    run(TwoStaticFieldFixtures.class)
+        .testEvents()
+        .assertStatistics(stats -> stats.succeeded(1).failed(0));
+    run(TwoFieldFixturesPerMethod.class)
+        .testEvents()
+        .assertStatistics(stats -> stats.succeeded(1).failed(0));
+  }
+
+  @Test
+  void testFieldFixtureOfWrongTypeFailsClearly() {
+    run(WrongTypeFieldFixture.class)
+        .allEvents()
+        .assertThatEvents()
+        .haveExactly(
+            1,
+            event(
+                finishedWithFailure(
+                    instanceOf(ExtensionConfigurationException.class),
+                    message(m -> m.contains("cannot hold a OtherContainer")))));
+  }
+
+  @Test
+  void testLazyFixtureStartsWhenAsked() {
+    run(LazyFixture.class).testEvents().assertStatistics(stats -> stats.succeeded(1).failed(0));
+
+    // The superclass @BeforeAll runs before the container starts, as BaseIT's server start would.
+    Assertions.assertEquals(List.of("setUp", "start first", "test"), LOG);
+  }
+
+  @Test
+  void testUnusedFixtureWarns() {
+    List warnings = new ArrayList<>();
+    Consumer original = ContainerExtension.warnings;
+    ContainerExtension.warnings = warnings::add;
+    try {
+      run(UnusedFixture.class).testEvents().assertStatistics(stats -> stats.succeeded(1).failed(0));
+      run(InjectedFixture.class).testEvents().assertStatistics(stats -> stats.succeeded(2));
+      run(FieldFixture.class).testEvents().assertStatistics(stats -> stats.succeeded(1));
+    } finally {
+      ContainerExtension.warnings = original;
+    }
+
+    Assertions.assertEquals(1, warnings.size(), warnings.toString());
+    Assertions.assertTrue(
+        warnings.get(0).startsWith("UnusedFixture declares a fixture for FakeSpec"));
+  }
+
   @Test
   void testFinalFieldFailsClearly() {
     run(FinalFieldFixture.class)
@@ -149,6 +203,11 @@ void testSpecsReadsEveryFixture() {
     Assertions.assertEquals(
         List.of(new HiveSpec(HiveVersion.HIVE3, HiveAuth.KERBEROS)),
         ContainerExtension.specs(HiveFixtureOnly.class));
+    Assertions.assertEquals(
+        List.of(
+            new MySQLSpec(MySQLVersion.MYSQL_5_7, TestDatabaseName.MYSQL_CATALOG_MYSQL_IT),
+            new PostgreSQLSpec(PGImageName.VERSION_13, TestDatabaseName.PG_CATALOG_POSTGRESQL_IT)),
+        ContainerExtension.specs(DatabaseFixturesOnly.class));
   }
 
   @Test
@@ -286,6 +345,11 @@ public ContainerSpec specFor(FakeFirstFixture fixture) {
     public Scope scopeFor(FakeFirstFixture fixture) {
       return fixture.scope();
     }
+
+    @Override
+    public Startup startupFor(FakeFirstFixture fixture) {
+      return fixture.startup();
+    }
   }
 
   static final class OtherFactory implements ContainerFixture.SpecFactory {
@@ -302,7 +366,7 @@ public ContainerSpec specFor(FakeSecondFixture fixture) {
     }
   }
 
-  @Target(ElementType.TYPE)
+  @Target({ElementType.TYPE, ElementType.FIELD})
   @Retention(RetentionPolicy.RUNTIME)
   @ExtendWith(ContainerExtension.class)
   @ContainerFixture(FirstFactory.class)
@@ -310,15 +374,17 @@ public ContainerSpec specFor(FakeSecondFixture fixture) {
     String value() default "first";
 
     Scope scope() default Scope.RUN;
+
+    Startup startup() default Startup.EAGER;
   }
 
-  @Target(ElementType.TYPE)
+  @Target({ElementType.TYPE, ElementType.FIELD})
   @Retention(RetentionPolicy.RUNTIME)
   @ExtendWith(ContainerExtension.class)
   @ContainerFixture(SecondFactory.class)
   @interface FakeSecondFixture {}
 
-  @Target(ElementType.TYPE)
+  @Target({ElementType.TYPE, ElementType.FIELD})
   @Retention(RetentionPolicy.RUNTIME)
   @ExtendWith(ContainerExtension.class)
   @ContainerFixture(OtherFactory.class)
@@ -399,6 +465,96 @@ static class FinalFieldFixture {
     void test() {}
   }
 
+  // Shaped like GvfsMultipleClusterIT: no class fixture, two fields of one type, read in an
+  // overridden @BeforeAll.
+  @Tag(FIXTURE)
+  static class TwoFieldFixtures extends SetUpBase {
+    @FakeFirstFixture @FixtureContainer private FakeContainer first;
+    @FakeSecondFixture @FixtureContainer private FakeContainer second;
+
+    @Override
+    @BeforeAll
+    void setUp() {
+      Assertions.assertSame(FIRST, first);
+      Assertions.assertSame(SECOND, second);
+    }
+
+    @Test
+    void test() {
+      Assertions.assertSame(FIRST, first);
+      Assertions.assertSame(SECOND, second);
+    }
+  }
+
+  @Tag(FIXTURE)
+  static class TwoStaticFieldFixtures {
+    @FakeFirstFixture @FixtureContainer private static FakeContainer first;
+    @FakeSecondFixture @FixtureContainer private static FakeContainer second;
+
+    @BeforeAll
+    static void setUp() {
+      Assertions.assertSame(FIRST, first);
+      Assertions.assertSame(SECOND, second);
+    }
+
+    @Test
+    void test() {
+      Assertions.assertSame(SECOND, second);
+    }
+  }
+
+  @Tag(FIXTURE)
+  static class TwoFieldFixturesPerMethod {
+    @FakeFirstFixture @FixtureContainer private FakeContainer first;
+    @FakeSecondFixture @FixtureContainer private FakeContainer second;
+
+    @Test
+    void test() {
+      Assertions.assertSame(FIRST, first);
+      Assertions.assertSame(SECOND, second);
+    }
+  }
+
+  @Tag(FIXTURE)
+  static class WrongTypeFieldFixture {
+    @SuppressWarnings("UnusedVariable")
+    @FakeOtherFixture
+    @FixtureContainer
+    private static FakeContainer wrong;
+
+    @Test
+    void test() {}
+  }
+
+  @TestInstance(TestInstance.Lifecycle.PER_CLASS)
+  static class LoggingSetUpBase {
+    @BeforeAll
+    void setUp() {
+      LOG.add("setUp");
+    }
+  }
+
+  @Tag(FIXTURE)
+  @FakeFirstFixture(startup = Startup.LAZY)
+  static class LazyFixture extends LoggingSetUpBase {
+    @BeforeAll
+    void startAfterSetUp(FakeContainer container) {
+      Assertions.assertSame(FIRST, container);
+    }
+
+    @Test
+    void test() {
+      LOG.add("test");
+    }
+  }
+
+  @Tag(FIXTURE)
+  @FakeFirstFixture
+  static class UnusedFixture {
+    @Test
+    void test() {}
+  }
+
   @FakeFirstFixture
   abstract static class FixtureBase {}
 
@@ -435,6 +591,13 @@ static class KafkaFixtureOnly {}
   @HiveFixture(version = HiveVersion.HIVE3, auth = HiveAuth.KERBEROS)
   static class HiveFixtureOnly {}
 
+  @Tag(FIXTURE)
+  @MySQLFixture(
+      version = MySQLVersion.MYSQL_5_7,
+      database = TestDatabaseName.MYSQL_CATALOG_MYSQL_IT)
+  @PostgreSQLFixture(database = TestDatabaseName.PG_CATALOG_POSTGRESQL_IT)
+  static class DatabaseFixturesOnly {}
+
   @Tag(FIXTURE)
   @FakeFirstFixture(scope = Scope.CLASS)
   static class ClassScopeFixture {
diff --git a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/ZooKeeperFixture.java b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/ZooKeeperFixture.java
index 9f1cd540b77..cbb9102f613 100644
--- a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/ZooKeeperFixture.java
+++ b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/ZooKeeperFixture.java
@@ -32,8 +32,16 @@
  */
 @Documented
 @Inherited
-@Target(ElementType.TYPE)
+@Target({ElementType.TYPE, ElementType.FIELD})
 @Retention(RetentionPolicy.RUNTIME)
 @ExtendWith(ContainerExtension.class)
 @ContainerFixture(ZooKeeperSpec.Factory.class)
-public @interface ZooKeeperFixture {}
+public @interface ZooKeeperFixture {
+
+  /**
+   * Returns when the container starts.
+   *
+   * @return the startup mode; before {@code @BeforeAll} by default
+   */
+  Startup startup() default Startup.EAGER;
+}
diff --git a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/ZooKeeperSpec.java b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/ZooKeeperSpec.java
index e567e60f542..bc623b20ed6 100644
--- a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/ZooKeeperSpec.java
+++ b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/ZooKeeperSpec.java
@@ -33,6 +33,11 @@ public ZooKeeperContainer getOrStart(ContainerSuite suite) {
   }
 
   static final class Factory implements ContainerFixture.SpecFactory {
+    @Override
+    public Startup startupFor(ZooKeeperFixture fixture) {
+      return fixture.startup();
+    }
+
     @Override
     public ZooKeeperSpec specFor(ZooKeeperFixture fixture) {
       return new ZooKeeperSpec();
diff --git a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/util/CloseContainerExtension.java b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/util/CloseContainerExtension.java
index 1e1b2aa41d4..1606d27d396 100644
--- a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/util/CloseContainerExtension.java
+++ b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/util/CloseContainerExtension.java
@@ -21,8 +21,6 @@
 import org.apache.gravitino.integration.test.container.ContainerSuite;
 import org.junit.jupiter.api.extension.BeforeAllCallback;
 import org.junit.jupiter.api.extension.ExtensionContext;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
 
 /**
  * This is an extension for JUnit 5, which aims to perform certain operations (such as resource
@@ -32,26 +30,6 @@
 public class CloseContainerExtension implements BeforeAllCallback {
   @Override
   public void beforeAll(ExtensionContext extensionContext) {
-    // Store.computeIfAbsent is atomic, so concurrent classes register the closer once.
-    extensionContext
-        .getRoot()
-        .getStore(ExtensionContext.Namespace.GLOBAL)
-        .computeIfAbsent(CloseableContainer.class);
-  }
-
-  private static class CloseableContainer implements AutoCloseable {
-    private static final Logger LOGGER = LoggerFactory.getLogger(CloseableContainer.class);
-    private static final ContainerSuite CONTAINER_SUITE = ContainerSuite.getInstance();
-
-    // Log instead of failing: a container teardown error must not fail an otherwise green run.
-    @Override
-    public void close() {
-      try {
-        CONTAINER_SUITE.close();
-        LOGGER.info("Containers were closed successfully");
-      } catch (Exception e) {
-        LOGGER.warn("Containers were not closed as expected", e);
-      }
-    }
+    ContainerSuite.closeWhenRunEnds(extensionContext);
   }
 }
diff --git a/spark-connector/spark-common/src/test/java/org/apache/gravitino/spark/connector/integration/test/sql/SparkQueryRunner.java b/spark-connector/spark-common/src/test/java/org/apache/gravitino/spark/connector/integration/test/sql/SparkQueryRunner.java
index 21a7c260e58..33278fc8d94 100644
--- a/spark-connector/spark-common/src/test/java/org/apache/gravitino/spark/connector/integration/test/sql/SparkQueryRunner.java
+++ b/spark-connector/spark-common/src/test/java/org/apache/gravitino/spark/connector/integration/test/sql/SparkQueryRunner.java
@@ -36,7 +36,10 @@
 import org.apache.gravitino.client.GravitinoAdminClient;
 import org.apache.gravitino.client.GravitinoMetalake;
 import org.apache.gravitino.integration.test.container.ContainerSuite;
+import org.apache.gravitino.integration.test.container.HiveAuth;
 import org.apache.gravitino.integration.test.container.HiveContainer;
+import org.apache.gravitino.integration.test.container.HiveSpec;
+import org.apache.gravitino.integration.test.container.HiveVersion;
 import org.apache.gravitino.integration.test.util.BaseIT;
 import org.apache.gravitino.spark.connector.GravitinoSparkConfig;
 import org.apache.gravitino.spark.connector.iceberg.IcebergPropertiesConstants;
@@ -63,8 +66,6 @@ public class SparkQueryRunner {
   private String dataDir;
   private BaseIT baseIT;
 
-  private static final ContainerSuite containerSuite = ContainerSuite.getInstance();
-
   public SparkQueryRunner(SparkTestConfig sparkTestConfig) {
     this.regenerateGoldenFiles = sparkTestConfig.generateGoldenFiles();
     this.isGravitinoEnvSetup = sparkTestConfig.isGravitinoEnvSetUp();
@@ -122,18 +123,17 @@ public void close() throws Exception {
   }
 
   private void setupGravitinoEnv() throws Exception {
-    // Start Hive and Hdfs
-    containerSuite.startHiveContainer();
+    // Start Hive and HDFS. Not a JUnit class, so it asks for the spec @HiveFixture would use and
+    // shares the run's Hive with fixture classes.
+    HiveContainer hive =
+        new HiveSpec(HiveVersion.HIVE2, HiveAuth.NONE).getOrStart(ContainerSuite.getInstance());
     String hiveMetastoreUri =
         String.format(
-            "thrift://%s:%d",
-            containerSuite.getHiveContainer().getContainerIpAddress(),
-            HiveContainer.HIVE_METASTORE_PORT);
+            "thrift://%s:%d", hive.getContainerIpAddress(), HiveContainer.HIVE_METASTORE_PORT);
     this.warehouse =
         String.format(
             "hdfs://%s:%d/user/hive/warehouse",
-            containerSuite.getHiveContainer().getContainerIpAddress(),
-            HiveContainer.HDFS_DEFAULTFS_PORT);
+            hive.getContainerIpAddress(), HiveContainer.HDFS_DEFAULTFS_PORT);
 
     // Start Gravitino server
     baseIT.stopIntegrationTest();

From f300f8b64b2fbefad2f00f8d5e77e01e41a71fe2 Mon Sep 17 00:00:00 2001
From: Nevin Zheng 
Date: Tue, 20 Oct 2026 00:40:27 -0700
Subject: [PATCH 13/14] [#14002] improvement(test): Add container dependencies
 and the remaining Hive variants (part 5/6)

- ContainerSpec.dependencies(): dependencies start first, deepest first,
  with a cycle check.
- ClickHouseCluster injected for the three-node cluster.
- Hive auth = RANGER and storage = S3 | GCS | ABS; the CatalogHive IT family
  and the Ranger ITs use them.
- HiveUserAuthenticationIT on the Kerberos Hive fixture.
---
 .../test/TestChainedAuthorizationIT.java      |   8 +-
 .../integration/test/RangerBaseE2EIT.java     |   4 +
 .../integration/test/RangerFilesetIT.java     |   5 +-
 .../integration/test/RangerHiveE2EIT.java     |   8 +-
 .../integration/test/RangerHiveHdfsE2EIT.java |  11 +-
 .../ranger/integration/test/RangerITEnv.java  |  36 +++--
 .../integration/test/RangerIcebergE2EIT.java  |   8 +-
 .../integration/test/RangerPaimonE2EIT.java   |  11 +-
 .../test/CatalogClickHouseClusterIT.java      |   9 +-
 .../hive/integration/test/CatalogHive2IT.java |  22 ++-
 .../hive/integration/test/CatalogHive3IT.java |  13 +-
 .../test/CatalogHive3ITWithCatalog.java       |   2 +-
 .../integration/test/CatalogHiveABSIT.java    |  20 +--
 .../integration/test/CatalogHiveGCSIT.java    |  26 +---
 .../integration/test/CatalogHiveS3IT.java     |  55 ++------
 .../test/HiveUserAuthenticationIT.java        |  29 ++--
 .../test/MultipleHMSUserAuthenticationIT.java |   2 +-
 .../test/container/ClickHouseCluster.java     |  43 ++++++
 .../test/container/ClickHouseFixture.java     |   4 +-
 .../test/container/ClickHouseSpec.java        |  21 +++
 .../test/container/ContainerExtension.java    |  22 ++-
 .../test/container/ContainerSpec.java         |  83 ++++++++++++
 .../test/container/ContainerSuite.java        |  51 +++----
 .../integration/test/container/HiveAuth.java  |  16 ++-
 .../test/container/HiveFixture.java           |   7 +
 .../integration/test/container/HiveSpec.java  | 115 +++++++++++++++-
 .../test/container/HiveStorage.java           |  40 ++++++
 .../test/container/RangerContainer.java       |   9 ++
 .../container/TestContainerExtension.java     |  14 ++
 .../TestContainerSpecDependencies.java        | 128 ++++++++++++++++++
 30 files changed, 607 insertions(+), 215 deletions(-)
 create mode 100644 integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/ClickHouseCluster.java
 create mode 100644 integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/HiveStorage.java
 create mode 100644 integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/TestContainerSpecDependencies.java

diff --git a/authorizations/authorization-chain/src/test/java/org/apache/gravitino/authorization/chain/integration/test/TestChainedAuthorizationIT.java b/authorizations/authorization-chain/src/test/java/org/apache/gravitino/authorization/chain/integration/test/TestChainedAuthorizationIT.java
index d9d823cff0e..6225d772547 100644
--- a/authorizations/authorization-chain/src/test/java/org/apache/gravitino/authorization/chain/integration/test/TestChainedAuthorizationIT.java
+++ b/authorizations/authorization-chain/src/test/java/org/apache/gravitino/authorization/chain/integration/test/TestChainedAuthorizationIT.java
@@ -81,21 +81,19 @@ public void startIntegrationTest() throws Exception {
 
     super.startIntegrationTest();
     RangerITEnv.init(ranger, RangerBaseE2EIT.metalakeName, false);
-    RangerITEnv.startHiveRangerContainer();
+    hiveRanger = RangerITEnv.startHiveRangerContainer();
 
     HIVE_METASTORE_URIS =
         String.format(
             "thrift://%s:%d",
-            containerSuite.getHiveRangerContainer().getContainerIpAddress(),
-            HiveContainer.HIVE_METASTORE_PORT);
+            hiveRanger.getContainerIpAddress(), HiveContainer.HIVE_METASTORE_PORT);
 
     generateRangerSparkSecurityXML("authorization-chain");
 
     DEFAULT_FS =
         String.format(
             "hdfs://%s:%d/user/hive/warehouse",
-            containerSuite.getHiveRangerContainer().getContainerIpAddress(),
-            HiveContainer.HDFS_DEFAULTFS_PORT);
+            hiveRanger.getContainerIpAddress(), HiveContainer.HDFS_DEFAULTFS_PORT);
     BaseIT.runInEnv(
         "HADOOP_USER_NAME",
         "test123",
diff --git a/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerBaseE2EIT.java b/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerBaseE2EIT.java
index 67e21db183e..1f73a96b3c3 100644
--- a/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerBaseE2EIT.java
+++ b/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerBaseE2EIT.java
@@ -43,6 +43,7 @@
 import org.apache.gravitino.authorization.SecurableObjects;
 import org.apache.gravitino.client.GravitinoMetalake;
 import org.apache.gravitino.integration.test.container.FixtureContainer;
+import org.apache.gravitino.integration.test.container.HiveContainer;
 import org.apache.gravitino.integration.test.container.RangerContainer;
 import org.apache.gravitino.integration.test.container.RangerFixture;
 import org.apache.gravitino.integration.test.util.BaseIT;
@@ -111,6 +112,9 @@ public abstract class RangerBaseE2EIT extends BaseIT {
 
   @FixtureContainer protected RangerContainer ranger;
 
+  // The Ranger-enabled Hive; subclasses start it after RangerITEnv.init().
+  protected static HiveContainer hiveRanger;
+
   protected static void generateRangerSparkSecurityXML(String modeName) throws IOException {
     String templatePath =
         String.join(
diff --git a/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerFilesetIT.java b/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerFilesetIT.java
index b0f0e3ae36f..258bbd6c64d 100644
--- a/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerFilesetIT.java
+++ b/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerFilesetIT.java
@@ -89,6 +89,7 @@ public class RangerFilesetIT extends BaseIT {
   private GravitinoMetalake metalake;
   private Catalog catalog;
   @FixtureContainer private RangerContainer ranger;
+  private static HiveContainer hiveRanger;
 
   @BeforeAll
   public void startIntegrationTest() throws Exception {
@@ -105,7 +106,7 @@ public void startIntegrationTest() throws Exception {
     super.startIntegrationTest();
 
     RangerITEnv.init(ranger, metalakeName, false);
-    RangerITEnv.startHiveRangerContainer();
+    hiveRanger = RangerITEnv.startHiveRangerContainer();
 
     RANGER_ADMIN_URL =
         String.format("http://%s:%d", ranger.getContainerIpAddress(), RANGER_SERVER_PORT);
@@ -601,7 +602,7 @@ private String defaultBaseLocation() {
       defaultBaseLocation =
           String.format(
               "hdfs://%s:%d/user/hadoop/%s",
-              containerSuite.getHiveRangerContainer().getContainerIpAddress(),
+              hiveRanger.getContainerIpAddress(),
               HiveContainer.HDFS_DEFAULTFS_PORT,
               schemaName.toLowerCase());
     }
diff --git a/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerHiveE2EIT.java b/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerHiveE2EIT.java
index 558eca5478a..1f16a9ac047 100644
--- a/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerHiveE2EIT.java
+++ b/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerHiveE2EIT.java
@@ -66,13 +66,12 @@ public void startIntegrationTest() throws Exception {
     super.startIntegrationTest();
 
     RangerITEnv.init(ranger, RangerBaseE2EIT.metalakeName, true);
-    RangerITEnv.startHiveRangerContainer();
+    hiveRanger = RangerITEnv.startHiveRangerContainer();
 
     HIVE_METASTORE_URIS =
         String.format(
             "thrift://%s:%d",
-            containerSuite.getHiveRangerContainer().getContainerIpAddress(),
-            HiveContainer.HIVE_METASTORE_PORT);
+            hiveRanger.getContainerIpAddress(), HiveContainer.HIVE_METASTORE_PORT);
 
     generateRangerSparkSecurityXML("authorization-ranger");
 
@@ -85,8 +84,7 @@ public void startIntegrationTest() throws Exception {
                 "spark.sql.warehouse.dir",
                 String.format(
                     "hdfs://%s:%d/user/hive/warehouse",
-                    containerSuite.getHiveRangerContainer().getContainerIpAddress(),
-                    HiveContainer.HDFS_DEFAULTFS_PORT))
+                    hiveRanger.getContainerIpAddress(), HiveContainer.HDFS_DEFAULTFS_PORT))
             .config("spark.sql.storeAssignmentPolicy", "LEGACY")
             .config("mapreduce.input.fileinputformat.input.dir.recursive", "true")
             .config(
diff --git a/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerHiveHdfsE2EIT.java b/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerHiveHdfsE2EIT.java
index 03e48b6b776..cd35f2bac48 100644
--- a/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerHiveHdfsE2EIT.java
+++ b/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerHiveHdfsE2EIT.java
@@ -95,19 +95,17 @@ public void startIntegrationTest() throws Exception {
 
     super.startIntegrationTest();
     RangerITEnv.init(ranger, RangerBaseE2EIT.metalakeName, false);
-    RangerITEnv.startHiveRangerContainer();
+    hiveRanger = RangerITEnv.startHiveRangerContainer();
 
     DEFAULT_FS =
         String.format(
             "hdfs://%s:%d/user/hive/warehouse",
-            containerSuite.getHiveRangerContainer().getContainerIpAddress(),
-            HiveContainer.HDFS_DEFAULTFS_PORT);
+            hiveRanger.getContainerIpAddress(), HiveContainer.HDFS_DEFAULTFS_PORT);
 
     HIVE_METASTORE_URIS =
         String.format(
             "thrift://%s:%d",
-            containerSuite.getHiveRangerContainer().getContainerIpAddress(),
-            HiveContainer.HIVE_METASTORE_PORT);
+            hiveRanger.getContainerIpAddress(), HiveContainer.HIVE_METASTORE_PORT);
 
     createMetalake();
     createCatalog();
@@ -364,8 +362,7 @@ private void initSparkSession() {
                       "spark.sql.warehouse.dir",
                       String.format(
                           "hdfs://%s:%d/user/hive/warehouse",
-                          containerSuite.getHiveRangerContainer().getContainerIpAddress(),
-                          HiveContainer.HDFS_DEFAULTFS_PORT))
+                          hiveRanger.getContainerIpAddress(), HiveContainer.HDFS_DEFAULTFS_PORT))
                   .config("spark.sql.storeAssignmentPolicy", "LEGACY")
                   .config("mapreduce.input.fileinputformat.input.dir.recursive", "true")
                   .enableHiveSupport()
diff --git a/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerITEnv.java b/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerITEnv.java
index d81b0a98803..727d738d60b 100644
--- a/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerITEnv.java
+++ b/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerITEnv.java
@@ -40,8 +40,12 @@
 import org.apache.gravitino.authorization.ranger.RangerPrivileges;
 import org.apache.gravitino.authorization.ranger.reference.RangerDefines;
 import org.apache.gravitino.connector.authorization.BaseAuthorization;
+import org.apache.gravitino.integration.test.container.ContainerSpec;
 import org.apache.gravitino.integration.test.container.ContainerSuite;
+import org.apache.gravitino.integration.test.container.HiveAuth;
 import org.apache.gravitino.integration.test.container.HiveContainer;
+import org.apache.gravitino.integration.test.container.HiveSpec;
+import org.apache.gravitino.integration.test.container.HiveVersion;
 import org.apache.gravitino.integration.test.container.RangerContainer;
 import org.apache.gravitino.integration.test.container.TrinoContainer;
 import org.apache.ranger.RangerClient;
@@ -60,14 +64,14 @@ public class RangerITEnv {
   private static final Logger LOG = LoggerFactory.getLogger(RangerITEnv.class);
   protected static final String RANGER_TRINO_REPO_NAME = "trinoDev";
   private static final String RANGER_TRINO_TYPE = "trino";
-  public static final String RANGER_HIVE_REPO_NAME = "hiveDev";
+  public static final String RANGER_HIVE_REPO_NAME = RangerContainer.HIVE_REPOSITORY;
   private static final String RANGER_HIVE_TYPE = "hive";
-  public static final String RANGER_HDFS_REPO_NAME = "hdfsDev";
+  public static final String RANGER_HDFS_REPO_NAME = RangerContainer.HDFS_REPOSITORY;
   public static final String METALAKE_ID = "1000";
   public static final String CATALOG_ID = "1001";
   private static final String RANGER_HDFS_TYPE = "hdfs";
   public static RangerClient rangerClient;
-  public static final String HADOOP_USER_NAME = "gravitino";
+  public static final String HADOOP_USER_NAME = RangerContainer.HADOOP_USER;
   private static volatile boolean initRangerService = false;
   private static final ContainerSuite containerSuite = ContainerSuite.getInstance();
 
@@ -189,22 +193,16 @@ public static void cleanup() {
     }
   }
 
-  public static void startHiveRangerContainer() {
-    containerSuite.startHiveRangerContainer(
-        new HashMap<>(
-            ImmutableMap.of(
-                HiveContainer.HIVE_RUNTIME_VERSION,
-                HiveContainer.HIVE3,
-                RangerContainer.DOCKER_ENV_RANGER_SERVER_URL,
-                String.format(
-                    "http://%s:%d",
-                    rangerContainer.getContainerIpAddress(), RangerContainer.RANGER_SERVER_PORT),
-                RangerContainer.DOCKER_ENV_RANGER_HIVE_REPOSITORY_NAME,
-                RangerITEnv.RANGER_HIVE_REPO_NAME,
-                RangerContainer.DOCKER_ENV_RANGER_HDFS_REPOSITORY_NAME,
-                RangerITEnv.RANGER_HDFS_REPO_NAME,
-                HiveContainer.HADOOP_USER_NAME,
-                HADOOP_USER_NAME)));
+  /**
+   * Starts the Ranger-enabled Hive after {@link #init} has set up the Ranger services, and returns
+   * it. Goes through the same spec as {@code @HiveFixture(version = HIVE3, auth = RANGER)}, so
+   * every class shares one Ranger Hive.
+   *
+   * @return the running Ranger-enabled Hive
+   */
+  public static HiveContainer startHiveRangerContainer() {
+    return ContainerSpec.startWithDependencies(
+        new HiveSpec(HiveVersion.HIVE3, HiveAuth.RANGER), containerSuite);
   }
 
   /** Currently we only test Ranger Hive, So wo Allow anyone to visit HDFS */
diff --git a/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerIcebergE2EIT.java b/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerIcebergE2EIT.java
index 1acacf6b193..ccfa58b4352 100644
--- a/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerIcebergE2EIT.java
+++ b/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerIcebergE2EIT.java
@@ -67,13 +67,12 @@ public void startIntegrationTest() throws Exception {
     super.startIntegrationTest();
 
     RangerITEnv.init(ranger, RangerBaseE2EIT.metalakeName, true);
-    RangerITEnv.startHiveRangerContainer();
+    hiveRanger = RangerITEnv.startHiveRangerContainer();
 
     HIVE_METASTORE_URIS =
         String.format(
             "thrift://%s:%d",
-            containerSuite.getHiveRangerContainer().getContainerIpAddress(),
-            HiveContainer.HIVE_METASTORE_PORT);
+            hiveRanger.getContainerIpAddress(), HiveContainer.HIVE_METASTORE_PORT);
 
     generateRangerSparkSecurityXML("authorization-ranger");
 
@@ -168,8 +167,7 @@ public void createCatalog() {
         IcebergConstants.WAREHOUSE,
         String.format(
             "hdfs://%s:%d/user/hive/warehouse",
-            containerSuite.getHiveRangerContainer().getContainerIpAddress(),
-            HiveContainer.HDFS_DEFAULTFS_PORT));
+            hiveRanger.getContainerIpAddress(), HiveContainer.HDFS_DEFAULTFS_PORT));
     properties.put(IMPERSONATION_ENABLE, "true");
     properties.put(AUTHORIZATION_PROVIDER, "ranger");
     properties.put(RangerAuthorizationProperties.RANGER_SERVICE_TYPE, "HadoopSQL");
diff --git a/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerPaimonE2EIT.java b/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerPaimonE2EIT.java
index 26bee1143ae..4fcea61a005 100644
--- a/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerPaimonE2EIT.java
+++ b/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerPaimonE2EIT.java
@@ -66,13 +66,12 @@ public void startIntegrationTest() throws Exception {
     super.startIntegrationTest();
 
     RangerITEnv.init(ranger, RangerBaseE2EIT.metalakeName, true);
-    RangerITEnv.startHiveRangerContainer();
+    hiveRanger = RangerITEnv.startHiveRangerContainer();
 
     HIVE_METASTORE_URIS =
         String.format(
             "thrift://%s:%d",
-            containerSuite.getHiveRangerContainer().getContainerIpAddress(),
-            HiveContainer.HIVE_METASTORE_PORT);
+            hiveRanger.getContainerIpAddress(), HiveContainer.HIVE_METASTORE_PORT);
 
     generateRangerSparkSecurityXML("authorization-ranger");
 
@@ -87,8 +86,7 @@ public void startIntegrationTest() throws Exception {
                 "spark.sql.catalog.paimon.warehouse",
                 String.format(
                     "hdfs://%s:%d/user/hive/warehouse",
-                    containerSuite.getHiveRangerContainer().getContainerIpAddress(),
-                    HiveContainer.HDFS_DEFAULTFS_PORT))
+                    hiveRanger.getContainerIpAddress(), HiveContainer.HDFS_DEFAULTFS_PORT))
             .config("spark.sql.catalog.paimon.cache-enabled", "false")
             .config(
                 "spark.sql.extensions",
@@ -189,8 +187,7 @@ public void createCatalog() {
             "warehouse",
             String.format(
                 "hdfs://%s:%d/user/hive/warehouse",
-                containerSuite.getHiveRangerContainer().getContainerIpAddress(),
-                HiveContainer.HDFS_DEFAULTFS_PORT),
+                hiveRanger.getContainerIpAddress(), HiveContainer.HDFS_DEFAULTFS_PORT),
             AUTHORIZATION_PROVIDER,
             "ranger",
             RangerAuthorizationProperties.RANGER_SERVICE_TYPE,
diff --git a/catalogs-contrib/catalog-jdbc-clickhouse/src/test/java/org/apache/gravitino/catalog/clickhouse/integration/test/CatalogClickHouseClusterIT.java b/catalogs-contrib/catalog-jdbc-clickhouse/src/test/java/org/apache/gravitino/catalog/clickhouse/integration/test/CatalogClickHouseClusterIT.java
index f6e1c9e1651..10ac0b152d8 100644
--- a/catalogs-contrib/catalog-jdbc-clickhouse/src/test/java/org/apache/gravitino/catalog/clickhouse/integration/test/CatalogClickHouseClusterIT.java
+++ b/catalogs-contrib/catalog-jdbc-clickhouse/src/test/java/org/apache/gravitino/catalog/clickhouse/integration/test/CatalogClickHouseClusterIT.java
@@ -52,9 +52,9 @@
 import org.apache.gravitino.catalog.clickhouse.operations.ClickHouseClusterUtils;
 import org.apache.gravitino.catalog.jdbc.config.JdbcConfig;
 import org.apache.gravitino.client.GravitinoMetalake;
+import org.apache.gravitino.integration.test.container.ClickHouseCluster;
 import org.apache.gravitino.integration.test.container.ClickHouseContainer;
 import org.apache.gravitino.integration.test.container.ClickHouseFixture;
-import org.apache.gravitino.integration.test.container.ContainerSuite;
 import org.apache.gravitino.integration.test.container.FixtureContainer;
 import org.apache.gravitino.integration.test.util.BaseIT;
 import org.apache.gravitino.integration.test.util.GravitinoITUtils;
@@ -85,7 +85,6 @@
 @TestInstance(Lifecycle.PER_CLASS)
 @ClickHouseFixture(nodes = 3, database = TestDatabaseName.CLICKHOUSE_CLUSTER_CLICKHOUSE_IT)
 public class CatalogClickHouseClusterIT extends BaseIT {
-  private static final ContainerSuite containerSuite = ContainerSuite.getInstance();
   private static final String provider = "jdbc-clickhouse";
 
   private final String metalakeName = GravitinoITUtils.genRandomName("ck_cluster_metalake");
@@ -107,13 +106,15 @@ public class CatalogClickHouseClusterIT extends BaseIT {
   private GravitinoMetalake metalake;
   private Catalog catalog;
   private ClickHouseService clickHouseService;
-  @FixtureContainer private ClickHouseContainer clickHouseClusterContainer;
+  @FixtureContainer private ClickHouseCluster cluster;
+  private ClickHouseContainer clickHouseClusterContainer;
   private List clickHouseClusterContainers;
   private final TestDatabaseName TEST_DB_NAME = TestDatabaseName.CLICKHOUSE_CLUSTER_CLICKHOUSE_IT;
 
   @BeforeAll
   public void startup() {
-    clickHouseClusterContainers = containerSuite.getClickHouseClusterContainers();
+    clickHouseClusterContainers = cluster.nodes();
+    clickHouseClusterContainer = cluster.first();
     Assertions.assertEquals(3, clickHouseClusterContainers.size());
 
     clickHouseService = new ClickHouseService(clickHouseClusterContainer, TEST_DB_NAME);
diff --git a/catalogs/catalog-hive/src/test/java/org/apache/gravitino/catalog/hive/integration/test/CatalogHive2IT.java b/catalogs/catalog-hive/src/test/java/org/apache/gravitino/catalog/hive/integration/test/CatalogHive2IT.java
index 3a4b9eafe94..4166fc94a92 100644
--- a/catalogs/catalog-hive/src/test/java/org/apache/gravitino/catalog/hive/integration/test/CatalogHive2IT.java
+++ b/catalogs/catalog-hive/src/test/java/org/apache/gravitino/catalog/hive/integration/test/CatalogHive2IT.java
@@ -83,8 +83,9 @@
 import org.apache.gravitino.hive.HivePartition;
 import org.apache.gravitino.hive.HiveSchema;
 import org.apache.gravitino.hive.HiveTable;
-import org.apache.gravitino.integration.test.container.ContainerSuite;
+import org.apache.gravitino.integration.test.container.FixtureContainer;
 import org.apache.gravitino.integration.test.container.HiveContainer;
+import org.apache.gravitino.integration.test.container.HiveFixture;
 import org.apache.gravitino.integration.test.util.BaseIT;
 import org.apache.gravitino.integration.test.util.GravitinoITUtils;
 import org.apache.gravitino.meta.AuditInfo;
@@ -128,6 +129,7 @@
 
 @Tag("gravitino-docker-test")
 @TestInstance(TestInstance.Lifecycle.PER_CLASS)
+@HiveFixture
 public class CatalogHive2IT extends BaseIT {
   private static final Logger LOG = LoggerFactory.getLogger(CatalogHive2IT.class);
 
@@ -146,7 +148,7 @@ public class CatalogHive2IT extends BaseIT {
   protected String hiveMetastoreUris;
   protected String hmsCatalog = "";
   protected final String provider = "hive";
-  protected final ContainerSuite containerSuite = ContainerSuite.getInstance();
+  @FixtureContainer protected HiveContainer hive;
   protected HiveClientPool hiveClientPool;
   protected GravitinoMetalake metalake;
   protected Catalog catalog;
@@ -179,14 +181,9 @@ private static String getInsertWithPartitionSql(
           "'gravitino_it_test'");
 
   protected void startNecessaryContainer() {
-    containerSuite.startHiveContainer(
-        ImmutableMap.of(HiveContainer.HIVE_RUNTIME_VERSION, HiveContainer.HIVE2));
-
     hiveMetastoreUris =
         String.format(
-            "thrift://%s:%d",
-            containerSuite.getHiveContainer().getContainerIpAddress(),
-            HiveContainer.HIVE_METASTORE_PORT);
+            "thrift://%s:%d", hive.getContainerIpAddress(), HiveContainer.HIVE_METASTORE_PORT);
   }
 
   protected void initSparkSession() {
@@ -199,8 +196,7 @@ protected void initSparkSession() {
                 "spark.sql.warehouse.dir",
                 String.format(
                     "hdfs://%s:%d/user/hive/warehouse",
-                    containerSuite.getHiveContainer().getContainerIpAddress(),
-                    HiveContainer.HDFS_DEFAULTFS_PORT))
+                    hive.getContainerIpAddress(), HiveContainer.HDFS_DEFAULTFS_PORT))
             .config("spark.sql.storeAssignmentPolicy", "LEGACY")
             .config("mapreduce.input.fileinputformat.input.dir.recursive", "true")
             .enableHiveSupport();
@@ -212,9 +208,7 @@ protected void initFileSystem() throws IOException {
     conf.set(
         "fs.defaultFS",
         String.format(
-            "hdfs://%s:%d",
-            containerSuite.getHiveContainer().getContainerIpAddress(),
-            HiveContainer.HDFS_DEFAULTFS_PORT));
+            "hdfs://%s:%d", hive.getContainerIpAddress(), HiveContainer.HDFS_DEFAULTFS_PORT));
     fileSystem = FileSystem.get(conf);
   }
 
@@ -2010,7 +2004,7 @@ protected Map createSchemaProperties() {
         "location",
         String.format(
             "hdfs://%s:%d/user/hive/warehouse/%s.db",
-            containerSuite.getHiveContainer().getContainerIpAddress(),
+            hive.getContainerIpAddress(),
             HiveContainer.HDFS_DEFAULTFS_PORT,
             schemaName.toLowerCase()));
     return properties;
diff --git a/catalogs/catalog-hive/src/test/java/org/apache/gravitino/catalog/hive/integration/test/CatalogHive3IT.java b/catalogs/catalog-hive/src/test/java/org/apache/gravitino/catalog/hive/integration/test/CatalogHive3IT.java
index 404a9fe2605..c19e721d39a 100644
--- a/catalogs/catalog-hive/src/test/java/org/apache/gravitino/catalog/hive/integration/test/CatalogHive3IT.java
+++ b/catalogs/catalog-hive/src/test/java/org/apache/gravitino/catalog/hive/integration/test/CatalogHive3IT.java
@@ -18,12 +18,13 @@
  */
 package org.apache.gravitino.catalog.hive.integration.test;
 
-import com.google.common.collect.ImmutableMap;
 import java.sql.SQLException;
 import java.util.Arrays;
 import java.util.Map;
 import org.apache.gravitino.NameIdentifier;
 import org.apache.gravitino.integration.test.container.HiveContainer;
+import org.apache.gravitino.integration.test.container.HiveFixture;
+import org.apache.gravitino.integration.test.container.HiveVersion;
 import org.apache.gravitino.rel.Column;
 import org.apache.gravitino.rel.Table;
 import org.apache.gravitino.rel.TableCatalog;
@@ -39,6 +40,7 @@
 
 @Tag("gravitino-docker-test")
 @TestInstance(TestInstance.Lifecycle.PER_CLASS)
+@HiveFixture(version = HiveVersion.HIVE3)
 public class CatalogHive3IT extends CatalogHive2IT {
 
   private HiveServer2Sql hiveSql;
@@ -53,19 +55,14 @@ public void closeHiveSql() throws SQLException {
   @Override
   protected void startNecessaryContainer() {
     hmsCatalog = "hive";
-    containerSuite.startHiveContainer(
-        ImmutableMap.of(HiveContainer.HIVE_RUNTIME_VERSION, HiveContainer.HIVE3));
-
     hiveMetastoreUris =
         String.format(
-            "thrift://%s:%d",
-            containerSuite.getHiveContainer().getContainerIpAddress(),
-            HiveContainer.HIVE_METASTORE_PORT);
+            "thrift://%s:%d", hive.getContainerIpAddress(), HiveContainer.HIVE_METASTORE_PORT);
   }
 
   /** Returns the Hive container used by this test suite. */
   protected HiveContainer hiveContainer() {
-    return containerSuite.getHiveContainer();
+    return hive;
   }
 
   /** Hive 3.x metastores support NOT NULL and DEFAULT constraints, which must round-trip to HMS. */
diff --git a/catalogs/catalog-hive/src/test/java/org/apache/gravitino/catalog/hive/integration/test/CatalogHive3ITWithCatalog.java b/catalogs/catalog-hive/src/test/java/org/apache/gravitino/catalog/hive/integration/test/CatalogHive3ITWithCatalog.java
index 79d8b079261..bc69c19e199 100644
--- a/catalogs/catalog-hive/src/test/java/org/apache/gravitino/catalog/hive/integration/test/CatalogHive3ITWithCatalog.java
+++ b/catalogs/catalog-hive/src/test/java/org/apache/gravitino/catalog/hive/integration/test/CatalogHive3ITWithCatalog.java
@@ -50,7 +50,7 @@ protected void createCatalog() {
     String location =
         String.format(
             "hdfs://%s:%d/user/hive/warehouse/%s/%s/%s",
-            containerSuite.getHiveContainer().getContainerIpAddress(),
+            hive.getContainerIpAddress(),
             HiveContainer.HDFS_DEFAULTFS_PORT,
             hmsCatalog,
             catalogName.toLowerCase(),
diff --git a/catalogs/catalog-hive/src/test/java/org/apache/gravitino/catalog/hive/integration/test/CatalogHiveABSIT.java b/catalogs/catalog-hive/src/test/java/org/apache/gravitino/catalog/hive/integration/test/CatalogHiveABSIT.java
index c09e1ef8b27..adca8133266 100644
--- a/catalogs/catalog-hive/src/test/java/org/apache/gravitino/catalog/hive/integration/test/CatalogHiveABSIT.java
+++ b/catalogs/catalog-hive/src/test/java/org/apache/gravitino/catalog/hive/integration/test/CatalogHiveABSIT.java
@@ -18,13 +18,15 @@
  */
 package org.apache.gravitino.catalog.hive.integration.test;
 
-import com.google.common.collect.ImmutableMap;
 import java.io.IOException;
 import java.net.URI;
 import java.util.HashMap;
 import java.util.Map;
 import org.apache.commons.lang3.StringUtils;
 import org.apache.gravitino.integration.test.container.HiveContainer;
+import org.apache.gravitino.integration.test.container.HiveFixture;
+import org.apache.gravitino.integration.test.container.HiveStorage;
+import org.apache.gravitino.integration.test.container.HiveVersion;
 import org.apache.gravitino.integration.test.util.GravitinoITUtils;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileSystem;
@@ -39,6 +41,7 @@
     value = "isAzureBlobStorageConfigured",
     disabledReason = "Azure Blob Storage is not prepared.")
 @TestInstance(TestInstance.Lifecycle.PER_CLASS)
+@HiveFixture(version = HiveVersion.HIVE3, storage = HiveStorage.ABS)
 public class CatalogHiveABSIT extends CatalogHive2IT {
 
   private static final String ABS_BUCKET_NAME = System.getenv("ABS_CONTAINER_NAME");
@@ -47,22 +50,9 @@ public class CatalogHiveABSIT extends CatalogHive2IT {
 
   @Override
   protected void startNecessaryContainer() {
-    Map hiveContainerEnv =
-        ImmutableMap.of(
-            "ABS_ACCOUNT_NAME",
-            ABS_USER_ACCOUNT_NAME,
-            "ABS_ACCOUNT_KEY",
-            ABS_USER_ACCOUNT_KEY,
-            HiveContainer.HIVE_RUNTIME_VERSION,
-            HiveContainer.HIVE3);
-
-    containerSuite.startHiveContainerWithS3(hiveContainerEnv);
-
     hiveMetastoreUris =
         String.format(
-            "thrift://%s:%d",
-            containerSuite.getHiveContainerWithS3().getContainerIpAddress(),
-            HiveContainer.HIVE_METASTORE_PORT);
+            "thrift://%s:%d", hive.getContainerIpAddress(), HiveContainer.HIVE_METASTORE_PORT);
   }
 
   @Override
diff --git a/catalogs/catalog-hive/src/test/java/org/apache/gravitino/catalog/hive/integration/test/CatalogHiveGCSIT.java b/catalogs/catalog-hive/src/test/java/org/apache/gravitino/catalog/hive/integration/test/CatalogHiveGCSIT.java
index 2dee713bf4f..d251cb27afe 100644
--- a/catalogs/catalog-hive/src/test/java/org/apache/gravitino/catalog/hive/integration/test/CatalogHiveGCSIT.java
+++ b/catalogs/catalog-hive/src/test/java/org/apache/gravitino/catalog/hive/integration/test/CatalogHiveGCSIT.java
@@ -19,51 +19,35 @@
 
 package org.apache.gravitino.catalog.hive.integration.test;
 
-import com.google.common.collect.ImmutableMap;
 import java.io.IOException;
 import java.net.URI;
 import java.util.HashMap;
 import java.util.Map;
 import org.apache.commons.lang3.StringUtils;
 import org.apache.gravitino.integration.test.container.HiveContainer;
+import org.apache.gravitino.integration.test.container.HiveFixture;
+import org.apache.gravitino.integration.test.container.HiveStorage;
+import org.apache.gravitino.integration.test.container.HiveVersion;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.spark.sql.SparkSession;
 import org.junit.jupiter.api.TestInstance;
 import org.junit.jupiter.api.condition.EnabledIf;
-import org.testcontainers.utility.MountableFile;
 
 @EnabledIf(value = "isGCSConfigured", disabledReason = "Google Cloud Storage(GCS) is not prepared.")
 @TestInstance(TestInstance.Lifecycle.PER_CLASS)
+@HiveFixture(version = HiveVersion.HIVE3, storage = HiveStorage.GCS)
 public class CatalogHiveGCSIT extends CatalogHive2IT {
 
   private static final String GCS_BUCKET_NAME = System.getenv("GCS_BUCKET_NAME");
   private static final String GCS_ACCOUNT_JSON_FILE =
       System.getenv("GCS_SERVICE_ACCOUNT_JSON_PATH");
-  private static final String GCS_ACCOUNT_JSON_FILE_IN_CONTAINER = "/tmp/gcs-service-account.json";
 
   @Override
   protected void startNecessaryContainer() {
-    Map hiveContainerEnv =
-        ImmutableMap.of(
-            "SERVICE_ACCOUNT_FILE",
-            GCS_ACCOUNT_JSON_FILE_IN_CONTAINER,
-            HiveContainer.HIVE_RUNTIME_VERSION,
-            HiveContainer.HIVE3);
-
-    containerSuite.startHiveContainerWithS3(hiveContainerEnv);
-
     hiveMetastoreUris =
         String.format(
-            "thrift://%s:%d",
-            containerSuite.getHiveContainerWithS3().getContainerIpAddress(),
-            HiveContainer.HIVE_METASTORE_PORT);
-
-    containerSuite
-        .getHiveContainerWithS3()
-        .getContainer()
-        .copyFileToContainer(
-            MountableFile.forHostPath(GCS_ACCOUNT_JSON_FILE), "/tmp/gcs-service-account.json");
+            "thrift://%s:%d", hive.getContainerIpAddress(), HiveContainer.HIVE_METASTORE_PORT);
   }
 
   @Override
diff --git a/catalogs/catalog-hive/src/test/java/org/apache/gravitino/catalog/hive/integration/test/CatalogHiveS3IT.java b/catalogs/catalog-hive/src/test/java/org/apache/gravitino/catalog/hive/integration/test/CatalogHiveS3IT.java
index 8364bf09630..0075d11b649 100644
--- a/catalogs/catalog-hive/src/test/java/org/apache/gravitino/catalog/hive/integration/test/CatalogHiveS3IT.java
+++ b/catalogs/catalog-hive/src/test/java/org/apache/gravitino/catalog/hive/integration/test/CatalogHiveS3IT.java
@@ -18,7 +18,6 @@
  */
 package org.apache.gravitino.catalog.hive.integration.test;
 
-import com.google.common.collect.ImmutableMap;
 import java.io.IOException;
 import java.net.URI;
 import java.util.HashMap;
@@ -26,6 +25,9 @@
 import org.apache.gravitino.integration.test.container.FixtureContainer;
 import org.apache.gravitino.integration.test.container.GravitinoLocalStackContainer;
 import org.apache.gravitino.integration.test.container.HiveContainer;
+import org.apache.gravitino.integration.test.container.HiveFixture;
+import org.apache.gravitino.integration.test.container.HiveStorage;
+import org.apache.gravitino.integration.test.container.HiveVersion;
 import org.apache.gravitino.integration.test.container.LocalStackFixture;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileSystem;
@@ -34,65 +36,39 @@
 
 @TestInstance(TestInstance.Lifecycle.PER_CLASS)
 @LocalStackFixture
+@HiveFixture(version = HiveVersion.HIVE3, storage = HiveStorage.S3)
 public class CatalogHiveS3IT extends CatalogHive3IT {
 
   private static final String S3_BUCKET_NAME = "my-test-bucket";
   @FixtureContainer private GravitinoLocalStackContainer gravitinoLocalStackContainer;
 
-  private static final String S3_ACCESS_KEY = "S3_ACCESS_KEY";
-  private static final String S3_SECRET_KEY = "S3_SECRET_KEY";
-  private static final String S3_ENDPOINT = "S3_ENDPOINT";
-
   private String getS3Endpoint;
-  private String accessKey;
-  private String secretKey;
 
   @Override
   protected void startNecessaryContainer() {
     hmsCatalog = "hive";
 
-    // The container is ready once its health check passes. LocalStack does not enforce IAM, so
-    // the shared test keys work without creating a user.
+    // The container is ready once its health check passes.
     gravitinoLocalStackContainer.createBucket(S3_BUCKET_NAME);
     gravitinoLocalStackContainer.makeBucketPublic(S3_BUCKET_NAME);
-    accessKey = GravitinoLocalStackContainer.ACCESS_KEY;
-    secretKey = GravitinoLocalStackContainer.SECRET_KEY;
 
     getS3Endpoint =
-        String.format("http://%s:%d", gravitinoLocalStackContainer.getContainerIpAddress(), 4566);
-
-    Map hiveContainerEnv =
-        ImmutableMap.of(
-            S3_ACCESS_KEY,
-            accessKey,
-            S3_SECRET_KEY,
-            secretKey,
-            S3_ENDPOINT,
-            getS3Endpoint,
-            HiveContainer.HIVE_RUNTIME_VERSION,
-            HiveContainer.HIVE3);
-
-    containerSuite.startHiveContainerWithS3(hiveContainerEnv);
+        String.format(
+            "http://%s:%d",
+            gravitinoLocalStackContainer.getContainerIpAddress(),
+            GravitinoLocalStackContainer.PORT);
 
     hiveMetastoreUris =
         String.format(
-            "thrift://%s:%d",
-            containerSuite.getHiveContainerWithS3().getContainerIpAddress(),
-            HiveContainer.HIVE_METASTORE_PORT);
-  }
-
-  /** {@inheritDoc} */
-  @Override
-  protected HiveContainer hiveContainer() {
-    return containerSuite.getHiveContainerWithS3();
+            "thrift://%s:%d", hive.getContainerIpAddress(), HiveContainer.HIVE_METASTORE_PORT);
   }
 
   @Override
   protected void initFileSystem() throws IOException {
     // Use S3a file system
     Configuration conf = new Configuration();
-    conf.set("fs.s3a.access.key", accessKey);
-    conf.set("fs.s3a.secret.key", secretKey);
+    conf.set("fs.s3a.access.key", GravitinoLocalStackContainer.ACCESS_KEY);
+    conf.set("fs.s3a.secret.key", GravitinoLocalStackContainer.SECRET_KEY);
     conf.set("fs.s3a.endpoint", getS3Endpoint);
     conf.set("fs.s3a.impl", "org.apache.hadoop.fs.s3a.S3AFileSystem");
     conf.set(
@@ -113,10 +89,9 @@ protected void initSparkSession() {
                 "spark.sql.warehouse.dir",
                 String.format(
                     "hdfs://%s:%d/user/hive/warehouse",
-                    containerSuite.getHiveContainerWithS3().getContainerIpAddress(),
-                    HiveContainer.HDFS_DEFAULTFS_PORT))
-            .config("spark.hadoop.fs.s3a.access.key", accessKey)
-            .config("spark.hadoop.fs.s3a.secret.key", secretKey)
+                    hive.getContainerIpAddress(), HiveContainer.HDFS_DEFAULTFS_PORT))
+            .config("spark.hadoop.fs.s3a.access.key", GravitinoLocalStackContainer.ACCESS_KEY)
+            .config("spark.hadoop.fs.s3a.secret.key", GravitinoLocalStackContainer.SECRET_KEY)
             .config("spark.hadoop.fs.s3a.endpoint", getS3Endpoint)
             .config("spark.hadoop.fs.s3a.impl", "org.apache.hadoop.fs.s3a.S3AFileSystem")
             .config("spark.hadoop.fs.s3a.path.style.access", "true")
diff --git a/catalogs/catalog-hive/src/test/java/org/apache/gravitino/catalog/hive/integration/test/HiveUserAuthenticationIT.java b/catalogs/catalog-hive/src/test/java/org/apache/gravitino/catalog/hive/integration/test/HiveUserAuthenticationIT.java
index 0bbf3e69ac3..d492bf77e05 100644
--- a/catalogs/catalog-hive/src/test/java/org/apache/gravitino/catalog/hive/integration/test/HiveUserAuthenticationIT.java
+++ b/catalogs/catalog-hive/src/test/java/org/apache/gravitino/catalog/hive/integration/test/HiveUserAuthenticationIT.java
@@ -41,8 +41,10 @@
 import org.apache.gravitino.client.GravitinoAdminClient;
 import org.apache.gravitino.client.GravitinoMetalake;
 import org.apache.gravitino.client.KerberosTokenProvider;
-import org.apache.gravitino.integration.test.container.ContainerSuite;
+import org.apache.gravitino.integration.test.container.FixtureContainer;
+import org.apache.gravitino.integration.test.container.HiveAuth;
 import org.apache.gravitino.integration.test.container.HiveContainer;
+import org.apache.gravitino.integration.test.container.HiveFixture;
 import org.apache.gravitino.integration.test.util.BaseIT;
 import org.apache.gravitino.integration.test.util.GravitinoITUtils;
 import org.apache.gravitino.integration.test.util.KerberosHdfs;
@@ -66,11 +68,10 @@
 
 @Tag("gravitino-docker-test")
 @TestInstance(TestInstance.Lifecycle.PER_CLASS)
+@HiveFixture(auth = HiveAuth.KERBEROS)
 public class HiveUserAuthenticationIT extends BaseIT {
   private static final Logger LOG = LoggerFactory.getLogger(HiveUserAuthenticationIT.class);
 
-  private static final ContainerSuite containerSuite = ContainerSuite.getInstance();
-
   private static final String SDK_KERBEROS_PRINCIPAL_KEY = "client.kerberos.principal";
   private static final String SDK_KERBEROS_KEYTAB_KEY = "client.kerberos.keytab";
 
@@ -91,7 +92,7 @@ public class HiveUserAuthenticationIT extends BaseIT {
 
   private static KerberosHdfs hdfs;
 
-  protected static HiveContainer kerberosHiveContainer;
+  @FixtureContainer protected static HiveContainer kerberosHive;
 
   static String METALAKE_NAME = GravitinoITUtils.genRandomName("test_metalake");
   static String CATALOG_NAME = GravitinoITUtils.genRandomName("test_catalog");
@@ -104,9 +105,6 @@ public class HiveUserAuthenticationIT extends BaseIT {
 
   @BeforeAll
   public void startIntegrationTest() throws Exception {
-    containerSuite.startKerberosHiveContainer();
-    kerberosHiveContainer = containerSuite.getKerberosHiveContainer();
-
     File baseDir = new File(System.getProperty("java.io.tmpdir"));
     File file = Files.createTempDirectory(baseDir.toPath(), "test").toFile();
     file.deleteOnExit();
@@ -115,11 +113,11 @@ public void startIntegrationTest() throws Exception {
     HIVE_METASTORE_URI =
         String.format(
             "thrift://%s:%d",
-            kerberosHiveContainer.getContainerIpAddress(), HiveContainer.HIVE_METASTORE_PORT);
+            kerberosHive.getContainerIpAddress(), HiveContainer.HIVE_METASTORE_PORT);
 
     // Prepare kerberos related-config;
     prepareKerberosConfig();
-    hdfs = KerberosHdfs.of(kerberosHiveContainer, TMP_DIR + HIVE_METASTORE_CLIENT_KEYTAB);
+    hdfs = KerberosHdfs.of(kerberosHive, TMP_DIR + HIVE_METASTORE_CLIENT_KEYTAB);
 
     // Config kerberos configuration for Gravitino server
     addKerberosConfig();
@@ -148,26 +146,26 @@ public void stop() {
 
   private static void prepareKerberosConfig() throws Exception {
     // Keytab of the Gravitino SDK client
-    kerberosHiveContainer
+    kerberosHive
         .getContainer()
         .copyFileFromContainer("/gravitino_client.keytab", TMP_DIR + GRAVITINO_CLIENT_KEYTAB);
 
     // Keytab of the Gravitino server
-    kerberosHiveContainer
+    kerberosHive
         .getContainer()
         .copyFileFromContainer("/gravitino_server.keytab", TMP_DIR + GRAVITINO_SERVER_KEYTAB);
 
     // Keytab of Gravitino server to connector to Hive
-    kerberosHiveContainer
+    kerberosHive
         .getContainer()
         .copyFileFromContainer("/etc/admin.keytab", TMP_DIR + HIVE_METASTORE_CLIENT_KEYTAB);
 
     String tmpKrb5Path = TMP_DIR + "/krb5.conf_tmp";
     String krb5Path = TMP_DIR + "/krb5.conf";
-    kerberosHiveContainer.getContainer().copyFileFromContainer("/etc/krb5.conf", tmpKrb5Path);
+    kerberosHive.getContainer().copyFileFromContainer("/etc/krb5.conf", tmpKrb5Path);
 
     // Modify the krb5.conf and change the kdc and admin_server to the container IP
-    String ip = containerSuite.getKerberosHiveContainer().getContainerIpAddress();
+    String ip = kerberosHive.getContainerIpAddress();
     String content = FileUtils.readFileToString(new File(tmpKrb5Path), StandardCharsets.UTF_8);
     content = content.replace("kdc = localhost:88", "kdc = " + ip + ":88");
     content = content.replace("admin_server = localhost", "admin_server = " + ip + ":749");
@@ -227,8 +225,7 @@ public void testUserAuthentication() {
     properties.put(CATALOG_BYPASS_PREFIX + HADOOP_SECURITY_AUTHENTICATION, "kerberos");
     properties.put(
         CATALOG_BYPASS_PREFIX + "hive.metastore.kerberos.principal",
-        "hive/_HOST@HADOOPKRB"
-            .replace("_HOST", containerSuite.getKerberosHiveContainer().getHostName()));
+        "hive/_HOST@HADOOPKRB".replace("_HOST", kerberosHive.getHostName()));
     properties.put(CATALOG_BYPASS_PREFIX + "hive.metastore.sasl.enabled", "true");
     Catalog catalog =
         gravitinoMetalake.createCatalog(
diff --git a/catalogs/catalog-hive/src/test/java/org/apache/gravitino/catalog/hive/integration/test/MultipleHMSUserAuthenticationIT.java b/catalogs/catalog-hive/src/test/java/org/apache/gravitino/catalog/hive/integration/test/MultipleHMSUserAuthenticationIT.java
index f6265695100..a6817239629 100644
--- a/catalogs/catalog-hive/src/test/java/org/apache/gravitino/catalog/hive/integration/test/MultipleHMSUserAuthenticationIT.java
+++ b/catalogs/catalog-hive/src/test/java/org/apache/gravitino/catalog/hive/integration/test/MultipleHMSUserAuthenticationIT.java
@@ -31,7 +31,7 @@
 public class MultipleHMSUserAuthenticationIT extends HiveUserAuthenticationIT {
   @BeforeAll
   static void setHiveURI() {
-    String ip = kerberosHiveContainer.getContainerIpAddress();
+    String ip = kerberosHive.getContainerIpAddress();
     int oldHMSPort = HiveContainer.HIVE_METASTORE_PORT;
 
     // About the value of `newAddedHMSPort`, please see the value `hive.metastore.port` in the
diff --git a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/ClickHouseCluster.java b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/ClickHouseCluster.java
new file mode 100644
index 00000000000..8000ebc30a1
--- /dev/null
+++ b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/ClickHouseCluster.java
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ *  http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.gravitino.integration.test.container;
+
+import java.util.List;
+
+/**
+ * The nodes of a ClickHouse cluster, injected for {@code @ClickHouseFixture(nodes = 3)}. A test
+ * that asks for a {@link ClickHouseContainer} instead receives the first node. {@code nodes} holds
+ * every node, first node first.
+ */
+public record ClickHouseCluster(List nodes) {
+
+  /** Copies the node list. */
+  public ClickHouseCluster {
+    nodes = List.copyOf(nodes);
+  }
+
+  /**
+   * Returns the first node, which tests use to run cluster-wide statements.
+   *
+   * @return the first node
+   */
+  public ClickHouseContainer first() {
+    return nodes.get(0);
+  }
+}
diff --git a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/ClickHouseFixture.java b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/ClickHouseFixture.java
index 7bc188ccd14..0f74c7759a0 100644
--- a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/ClickHouseFixture.java
+++ b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/ClickHouseFixture.java
@@ -31,8 +31,8 @@
  * Declares that a test class needs a ClickHouse server, or the three-node ClickHouse cluster with
  * {@code nodes = 3}, with the test's database created on it. Test and lifecycle methods receive it
  * as a {@link ClickHouseContainer} parameter, and fields marked {@link FixtureContainer} receive it
- * too. For the cluster, that is the first node; {@link
- * ContainerSuite#getClickHouseClusterContainers()} returns every node.
+ * too. For the cluster, that is the first node; a {@link ClickHouseCluster} field or parameter
+ * receives every node.
  *
  * 
{@code
  * @ClickHouseFixture(database = TestDatabaseName.CLICKHOUSE_CATALOG_CLICKHOUSE_IT)
diff --git a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/ClickHouseSpec.java b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/ClickHouseSpec.java
index 5a80dc30891..3d9a326e788 100644
--- a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/ClickHouseSpec.java
+++ b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/ClickHouseSpec.java
@@ -26,6 +26,7 @@
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.StandardCopyOption;
+import java.util.List;
 import java.util.Objects;
 import org.apache.gravitino.integration.test.util.TestDatabaseName;
 
@@ -82,6 +83,26 @@ public ClickHouseContainer getOrStart(ContainerSuite suite) {
   // ContainerSuite reads the template from a file path, and the resource may sit inside a jar.
   // Copied on first use, not in a static initializer, so loading this class never touches the
   // disk; a failed copy is not cached and the next start tries again.
+  // The cluster keeps its replication metadata in ZooKeeper.
+  @Override
+  public List> dependencies() {
+    return nodes == CLUSTER_NODES ? List.of(new ZooKeeperSpec()) : List.of();
+  }
+
+  @Override
+  public boolean supplies(Class wanted) {
+    return ContainerSpec.super.supplies(wanted)
+        || (nodes == CLUSTER_NODES && wanted == ClickHouseCluster.class);
+  }
+
+  @Override
+  public Object injectAs(Class wanted, ClickHouseContainer container) {
+    if (wanted == ClickHouseCluster.class) {
+      return new ClickHouseCluster(ContainerSuite.getInstance().getClickHouseClusterContainers());
+    }
+    return container;
+  }
+
   private static Path copyRemoteServersTemplate() {
     try (InputStream template =
         Objects.requireNonNull(
diff --git a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/ContainerExtension.java b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/ContainerExtension.java
index ae423590cd0..0dbfe6140f3 100644
--- a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/ContainerExtension.java
+++ b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/ContainerExtension.java
@@ -189,7 +189,7 @@ private static void injectFields(
         if (!methodScoped && testInstance != null && field.get(testInstance) != null) {
           continue;
         }
-        field.set(testInstance, container(declaration, context));
+        field.set(testInstance, inject(declaration, field.getType(), context));
         injected(context).add(declaration.spec());
       } catch (IllegalAccessException e) {
         throw new ExtensionConfigurationException("Cannot set field " + field.getName(), e);
@@ -216,7 +216,7 @@ private static Optional fieldDeclaration(Field field) {
       return Optional.empty();
     }
     Class containerType = found.get(0).spec().containerType();
-    if (!field.getType().isAssignableFrom(containerType)) {
+    if (!found.get(0).spec().supplies(field.getType())) {
       throw new ExtensionConfigurationException(
           String.format(
               "%s: field %s of type %s cannot hold a %s",
@@ -241,7 +241,14 @@ public Object resolveParameter(ParameterContext parameter, ExtensionContext cont
             context.getRequiredTestClass(),
             "parameter " + parameter.getParameter().getName());
     injected(context).add(declaration.spec());
-    return container(declaration, context);
+    return inject(declaration, parameter.getParameter().getType(), context);
+  }
+
+  // The running container, as the type the field or parameter asks for.
+  @SuppressWarnings({"unchecked", "rawtypes"})
+  private static Object inject(Declaration declaration, Class wanted, ExtensionContext context) {
+    ContainerSpec spec = declaration.spec();
+    return spec.injectAs(wanted, container(declaration, context));
   }
 
   private static Declaration declarationFor(Class wanted, Class testClass, String target) {
@@ -261,7 +268,7 @@ private static Declaration declarationFor(Class wanted, Class testClass, S
   private static BaseContainer container(Declaration declaration, ExtensionContext context) {
     ContainerSpec spec = declaration.spec();
     if (declaration.scope() == Scope.RUN) {
-      return spec.getOrStart(ContainerSuite.getInstance());
+      return ContainerSpec.startWithDependencies(spec, ContainerSuite.getInstance());
     }
     ExtensionContext owner =
         declaration.scope() == Scope.CLASS ? classContext(context) : methodContext(context, spec);
@@ -269,7 +276,10 @@ private static BaseContainer container(Declaration declaration, ExtensionContext
         .getStore(Namespace.create(ContainerExtension.class, spec))
         .computeIfAbsent(
             PRIVATE_CONTAINER,
-            key -> spec.startPrivate(ContainerSuite.getInstance()),
+            key -> {
+              ContainerSpec.startDependencies(spec, ContainerSuite.getInstance());
+              return spec.startPrivate(ContainerSuite.getInstance());
+            },
             BaseContainer.class);
   }
 
@@ -302,7 +312,7 @@ private static ExtensionConfigurationException methodScopeOutsideTest(
   private static List matching(Class wanted, Class testClass) {
     List matches = new ArrayList<>();
     for (Declaration declaration : declarations(testClass)) {
-      if (wanted.isAssignableFrom(declaration.spec().containerType())) {
+      if (declaration.spec().supplies(wanted)) {
         matches.add(declaration);
       }
     }
diff --git a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/ContainerSpec.java b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/ContainerSpec.java
index 34a58ccf078..c01f28f25c8 100644
--- a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/ContainerSpec.java
+++ b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/ContainerSpec.java
@@ -18,6 +18,10 @@
  */
 package org.apache.gravitino.integration.test.container;
 
+import java.util.ArrayDeque;
+import java.util.Deque;
+import java.util.List;
+
 /**
  * What a test asks for: one kind of container in one configuration.
  *
@@ -35,6 +39,85 @@ public interface ContainerSpec {
    */
   Class containerType();
 
+  /**
+   * Returns whether a field or parameter of the given type can receive this spec's container. By
+   * default it can when the type is the container type or a supertype of it; a spec may offer more
+   * types, such as a view of several containers.
+   *
+   * @param wanted the type of the field or parameter
+   * @return whether this spec supplies that type
+   */
+  default boolean supplies(Class wanted) {
+    return wanted.isAssignableFrom(containerType());
+  }
+
+  /**
+   * Returns what a field or parameter of a type this spec {@linkplain #supplies(Class) supplies}
+   * receives for the running container.
+   *
+   * @param wanted the type of the field or parameter
+   * @param container the running container
+   * @return the container itself by default
+   */
+  default Object injectAs(Class wanted, C container) {
+    return container;
+  }
+
+  /**
+   * Returns the specs whose containers must run before this one starts, such as the Ranger server a
+   * Ranger-enabled Hive reports to. {@link #startWithDependencies} starts them first, as shared
+   * run-wide containers, so they outlive this container whatever its scope. Inside {@link
+   * #getOrStart} the spec reads a dependency's running container with {@code
+   * dependency.getOrStart(suite)}, which returns the one already started.
+   *
+   * @return the direct dependencies; none by default
+   */
+  default List> dependencies() {
+    return List.of();
+  }
+
+  /**
+   * Starts every dependency of {@code spec}, deepest first, and then returns its running shared
+   * container.
+   *
+   * @param spec the spec to start
+   * @param suite the shared container suite
+   * @param  the container type
+   * @return the running container
+   * @throws IllegalStateException if the dependencies form a cycle
+   */
+  static  C startWithDependencies(
+      ContainerSpec spec, ContainerSuite suite) {
+    startDependencies(spec, suite);
+    return spec.getOrStart(suite);
+  }
+
+  /**
+   * Starts every dependency of {@code spec}, deepest first, but not {@code spec} itself; for
+   * private containers, whose own start happens elsewhere.
+   *
+   * @param spec the spec whose dependencies to start
+   * @param suite the shared container suite
+   * @throws IllegalStateException if the dependencies form a cycle
+   */
+  static void startDependencies(ContainerSpec spec, ContainerSuite suite) {
+    startDependencies(spec, suite, new ArrayDeque<>());
+  }
+
+  private static void startDependencies(
+      ContainerSpec spec, ContainerSuite suite, Deque> path) {
+    if (path.contains(spec)) {
+      path.addLast(spec);
+      throw new IllegalStateException("Container dependencies form a cycle: " + path);
+    }
+    path.addLast(spec);
+    for (ContainerSpec dependency : spec.dependencies()) {
+      startDependencies(dependency, suite, path);
+      dependency.getOrStart(suite);
+    }
+    path.removeLast();
+  }
+
   /**
    * Returns the running container for this spec, starting it on first use.
    *
diff --git a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/ContainerSuite.java b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/ContainerSuite.java
index 3502d644e13..3e68818f949 100644
--- a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/ContainerSuite.java
+++ b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/ContainerSuite.java
@@ -116,7 +116,9 @@ public class ContainerSuite {
    * container with S3 is Hive3 container and the Hive container is Hive2 container. There is
    * something wrong with the hive2 container to access the S3.
    */
-  private static volatile HiveContainer hiveContainerWithS3;
+  // A Hive whose tables live in cloud storage, keyed by storage and version; one at a time.
+  private static final ContainerSlot storageHiveContainer =
+      new ContainerSlot<>();
 
   protected static final CloseableGroup closer = CloseableGroup.create();
 
@@ -171,25 +173,28 @@ public void startHiveContainer(Map env) {
     }
   }
 
-  public void startHiveContainerWithS3(Map env) {
-    ImmutableMap.Builder builder = ImmutableMap.builder();
-    builder.putAll(env);
-    builder.put("HADOOP_USER_NAME", "anonymous");
-    if (hiveContainerWithS3 == null) {
-      synchronized (ContainerSuite.class) {
-        if (hiveContainerWithS3 == null) {
-          initIfNecessary();
-          // Start Hive container
-          HiveContainer.Builder hiveBuilder =
-              HiveContainer.builder()
-                  .withHostName("gravitino-ci-hive")
-                  .withEnvVars(builder.build())
-                  .withNetwork(network);
-          HiveContainer container = closer.register(hiveBuilder.build());
-          start(container);
-          hiveContainerWithS3 = container;
-        }
-      }
+  /**
+   * Starts, or returns the running, Hive whose tables live in the given cloud storage. Only one
+   * runs at a time: asking for another storage or version replaces it.
+   *
+   * @param storage the storage, other than {@link HiveStorage#HDFS}
+   * @param env the container environment, including {@link HiveContainer#HIVE_RUNTIME_VERSION}
+   * @param filesToMount host files to mount, keyed by container path
+   * @return the running Hive
+   */
+  public HiveContainer startStorageHiveContainer(
+      HiveStorage storage, Map env, Map filesToMount) {
+    synchronized (ContainerSuite.class) {
+      return storageHiveContainer.getOrStart(
+          storage + "/" + hiveVersion(env),
+          () -> {
+            initIfNecessary();
+            HiveContainer container =
+                closer.register(
+                    hiveBuilder(env).withFilesToMount(filesToMount).withNetwork(network).build());
+            start(container);
+            return container;
+          });
     }
   }
 
@@ -760,10 +765,6 @@ public RustFSContainer getRustFSContainer() {
     return rustFSContainer;
   }
 
-  public HiveContainer getHiveContainerWithS3() {
-    return hiveContainerWithS3;
-  }
-
   public KafkaContainer getKafkaContainer() {
     return kafkaContainer;
   }
@@ -1113,7 +1114,7 @@ private static void resetState() {
     gravitinoLocalStackContainer = null;
     rustFSContainer = null;
     motoContainer = null;
-    hiveContainerWithS3 = null;
+    storageHiveContainer.clear();
   }
 
   private static Map resolveClickHouseTestFiles() {
diff --git a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/HiveAuth.java b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/HiveAuth.java
index ba147d18a4a..db802b01860 100644
--- a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/HiveAuth.java
+++ b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/HiveAuth.java
@@ -18,15 +18,17 @@
  */
 package org.apache.gravitino.integration.test.container;
 
-/**
- * How clients authenticate to the Hive a {@link HiveFixture} runs.
- *
- * 
Ranger is not here yet: it needs a Ranger container started first, which arrives with a Ranger
- * dependency spec.
- */
+/** How clients authenticate to the Hive a {@link HiveFixture} runs. */
 public enum HiveAuth {
   /** No authentication. */
   NONE,
   /** Kerberos, from the Kerberos Hive image with its own KDC. */
-  KERBEROS
+  KERBEROS,
+  /**
+   * Ranger plugins for Hive and HDFS that report to the run's Ranger server, under the repositories
+   * {@link RangerContainer#HIVE_REPOSITORY} and {@link RangerContainer#HDFS_REPOSITORY}. Hive3
+   * only. Starts the Ranger server first, and starts lazily: a test asks for it after creating the
+   * Ranger services.
+   */
+  RANGER
 }
diff --git a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/HiveFixture.java b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/HiveFixture.java
index cfd1af83d3c..e45fe601daa 100644
--- a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/HiveFixture.java
+++ b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/HiveFixture.java
@@ -57,6 +57,13 @@
    */
   HiveAuth auth() default HiveAuth.NONE;
 
+  /**
+   * Returns where Hive keeps table data besides its own HDFS.
+   *
+   * @return the storage; HDFS only by default
+   */
+  HiveStorage storage() default HiveStorage.HDFS;
+
   /**
    * Returns how long the Hive container lives. Use {@link Scope#CLASS} or {@link Scope#METHOD} for
    * tests that change Hive in ways later tests would see, such as adding Kerberos principals.
diff --git a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/HiveSpec.java b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/HiveSpec.java
index 192e8e1b51e..df588da8e5a 100644
--- a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/HiveSpec.java
+++ b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/HiveSpec.java
@@ -18,18 +18,58 @@
  */
 package org.apache.gravitino.integration.test.container;
 
+import com.google.common.collect.ImmutableMap;
+import java.util.List;
 import java.util.Map;
 
 /**
- * A Hive server of one version and authentication mode.
+ * A Hive server of one version, authentication mode and storage.
  *
  * 
Plain and Kerberos Hive run side by side, but each holds one version at a time: asking for a
  * different version stops the running Hive of that mode and starts the requested one (see {@code
  * ContainerSlot}). Tests of one version therefore share a container, and a version switch costs a
  * restart, as with direct {@link ContainerSuite} calls. A private Hive ({@link Scope#CLASS} or
- * {@link Scope#METHOD}) leaves the shared one alone.
+ * {@link Scope#METHOD}) leaves the shared one alone. A Hive with cloud {@link HiveStorage} is a
+ * third container, one storage and version at a time.
  */
-public record HiveSpec(HiveVersion version, HiveAuth auth) implements ContainerSpec {
+public record HiveSpec(HiveVersion version, HiveAuth auth, HiveStorage storage)
+    implements ContainerSpec {
+
+  private static final String GCS_ACCOUNT_FILE_IN_CONTAINER = "/tmp/gcs-service-account.json";
+
+  /**
+   * Checks the combination.
+   *
+   * @throws IllegalArgumentException if Ranger is asked for with a Hive other than Hive3, or cloud
+   *     storage with authentication
+   */
+  public HiveSpec {
+    if (auth == HiveAuth.RANGER && version != HiveVersion.HIVE3) {
+      throw new IllegalArgumentException("Ranger Hive runs Hive3 only, not " + version);
+    }
+    if (storage != HiveStorage.HDFS && auth != HiveAuth.NONE) {
+      throw new IllegalArgumentException(
+          "Hive with " + storage + " storage supports no authentication, not " + auth);
+    }
+  }
+
+  /**
+   * A Hive that keeps its tables in its own HDFS.
+   *
+   * @param version the Hive version
+   * @param auth the authentication mode
+   */
+  public HiveSpec(HiveVersion version, HiveAuth auth) {
+    this(version, auth, HiveStorage.HDFS);
+  }
+
+  @Override
+  public List> dependencies() {
+    if (auth == HiveAuth.RANGER) {
+      return List.of(new RangerSpec());
+    }
+    return storage == HiveStorage.S3 ? List.of(new LocalStackSpec()) : List.of();
+  }
 
   @Override
   public Class containerType() {
@@ -38,6 +78,9 @@ public Class containerType() {
 
   @Override
   public HiveContainer getOrStart(ContainerSuite suite) {
+    if (storage != HiveStorage.HDFS) {
+      return suite.startStorageHiveContainer(storage, storageEnv(suite), storageFiles());
+    }
     return switch (auth) {
       case NONE -> {
         suite.startHiveContainer(env());
@@ -47,14 +90,72 @@ public HiveContainer getOrStart(ContainerSuite suite) {
         suite.startKerberosHiveContainer(env());
         yield suite.getKerberosHiveContainer();
       }
+      case RANGER -> {
+        suite.startHiveRangerContainer(rangerEnv(new RangerSpec().getOrStart(suite)));
+        yield suite.getHiveRangerContainer();
+      }
     };
   }
 
+  private Map storageEnv(ContainerSuite suite) {
+    ImmutableMap.Builder env = ImmutableMap.builder().putAll(env());
+    switch (storage) {
+      case S3 -> {
+        GravitinoLocalStackContainer localStack = new LocalStackSpec().getOrStart(suite);
+        env.put("S3_ACCESS_KEY", GravitinoLocalStackContainer.ACCESS_KEY)
+            .put("S3_SECRET_KEY", GravitinoLocalStackContainer.SECRET_KEY)
+            .put(
+                "S3_ENDPOINT",
+                String.format(
+                    "http://%s:%d",
+                    localStack.getContainerIpAddress(), GravitinoLocalStackContainer.PORT));
+      }
+      case GCS -> env.put("SERVICE_ACCOUNT_FILE", GCS_ACCOUNT_FILE_IN_CONTAINER);
+      case ABS -> env.put("ABS_ACCOUNT_NAME", requiredEnv("ABS_ACCOUNT_NAME"))
+          .put("ABS_ACCOUNT_KEY", requiredEnv("ABS_ACCOUNT_KEY"));
+      case HDFS -> {}
+    }
+    return env.build();
+  }
+
+  private Map storageFiles() {
+    return storage == HiveStorage.GCS
+        ? Map.of(GCS_ACCOUNT_FILE_IN_CONTAINER, requiredEnv("GCS_SERVICE_ACCOUNT_JSON_PATH"))
+        : Map.of();
+  }
+
+  private static String requiredEnv(String name) {
+    String value = System.getenv(name);
+    if (value == null || value.isEmpty()) {
+      throw new IllegalStateException(name + " must be set for this Hive storage");
+    }
+    return value;
+  }
+
+  private Map rangerEnv(RangerContainer ranger) {
+    return ImmutableMap.builder()
+        .putAll(env())
+        .put(
+            RangerContainer.DOCKER_ENV_RANGER_SERVER_URL,
+            String.format(
+                "http://%s:%d", ranger.getContainerIpAddress(), RangerContainer.RANGER_SERVER_PORT))
+        .put(
+            RangerContainer.DOCKER_ENV_RANGER_HIVE_REPOSITORY_NAME, RangerContainer.HIVE_REPOSITORY)
+        .put(
+            RangerContainer.DOCKER_ENV_RANGER_HDFS_REPOSITORY_NAME, RangerContainer.HDFS_REPOSITORY)
+        .put(HiveContainer.HADOOP_USER_NAME, RangerContainer.HADOOP_USER)
+        .build();
+  }
+
   // Keeps the shared Hive's host name: the Kerberos image derives its principals from it. On macOS
   // the shared network then has two containers with one alias, which is fine because tests reach
   // containers by IP; on Linux CI there is no shared network.
   @Override
   public HiveContainer startPrivate(ContainerSuite suite) {
+    if (storage != HiveStorage.HDFS) {
+      throw new UnsupportedOperationException(
+          "scope CLASS/METHOD is not supported for Hive with " + storage + " storage yet");
+    }
     return suite.startPrivate(network -> builder().withNetwork(network).build());
   }
 
@@ -62,6 +163,8 @@ private HiveContainer.Builder builder() {
     return switch (auth) {
       case NONE -> ContainerSuite.hiveBuilder(env());
       case KERBEROS -> ContainerSuite.kerberosHiveBuilder(env());
+      case RANGER -> throw new UnsupportedOperationException(
+          "scope CLASS/METHOD is not supported for Ranger Hive yet");
     };
   }
 
@@ -72,12 +175,14 @@ private Map env() {
   static final class Factory implements ContainerFixture.SpecFactory {
     @Override
     public Startup startupFor(HiveFixture fixture) {
-      return fixture.startup();
+      // Ranger's services must exist before a Ranger-enabled Hive reports to them, and tests
+      // create them in @BeforeAll, so that Hive starts when a test first asks for it.
+      return fixture.auth() == HiveAuth.RANGER ? Startup.LAZY : fixture.startup();
     }
 
     @Override
     public HiveSpec specFor(HiveFixture fixture) {
-      return new HiveSpec(fixture.version(), fixture.auth());
+      return new HiveSpec(fixture.version(), fixture.auth(), fixture.storage());
     }
 
     @Override
diff --git a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/HiveStorage.java b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/HiveStorage.java
new file mode 100644
index 00000000000..bdafc5c1e32
--- /dev/null
+++ b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/HiveStorage.java
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ *  http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.gravitino.integration.test.container;
+
+/**
+ * Where the Hive a {@link HiveFixture} runs keeps table data besides its own HDFS. Every mode but
+ * {@link #HDFS} gets its own Hive container.
+ */
+public enum HiveStorage {
+  /** HDFS inside the Hive container only. */
+  HDFS,
+  /** S3 on the run's LocalStack, which starts first. */
+  S3,
+  /**
+   * Google Cloud Storage, with the service account file named by the {@code
+   * GCS_SERVICE_ACCOUNT_JSON_PATH} environment variable.
+   */
+  GCS,
+  /**
+   * Azure Blob Storage, with the account in the {@code ABS_ACCOUNT_NAME} and {@code
+   * ABS_ACCOUNT_KEY} environment variables.
+   */
+  ABS
+}
diff --git a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/RangerContainer.java b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/RangerContainer.java
index fe059f5129c..9155c0f81b4 100644
--- a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/RangerContainer.java
+++ b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/RangerContainer.java
@@ -57,6 +57,15 @@ public class RangerContainer extends BaseContainer {
   public static final String DOCKER_ENV_RANGER_HDFS_REPOSITORY_NAME = "RANGER_HDFS_REPOSITORY_NAME";
   public static final String DOCKER_ENV_RANGER_HIVE_REPOSITORY_NAME = "RANGER_HIVE_REPOSITORY_NAME";
 
+  /** The Ranger repository a Ranger-enabled Hive reports Hive access to. */
+  public static final String HIVE_REPOSITORY = "hiveDev";
+
+  /** The Ranger repository a Ranger-enabled Hive reports HDFS access to. */
+  public static final String HDFS_REPOSITORY = "hdfsDev";
+
+  /** The Hadoop user a Ranger-enabled Hive runs as. */
+  public static final String HADOOP_USER = "gravitino";
+
   public static Builder builder() {
     return new Builder();
   }
diff --git a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/TestContainerExtension.java b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/TestContainerExtension.java
index a628db9372f..a256662174c 100644
--- a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/TestContainerExtension.java
+++ b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/TestContainerExtension.java
@@ -152,6 +152,20 @@ void testUnusedFixtureWarns() {
         warnings.get(0).startsWith("UnusedFixture declares a fixture for FakeSpec"));
   }
 
+  @Test
+  void testClickHouseClusterOnlySuppliedForThreeNodes() {
+    ClickHouseSpec single =
+        new ClickHouseSpec(1, TestDatabaseName.CLICKHOUSE_CATALOG_CLICKHOUSE_IT);
+    ClickHouseSpec cluster =
+        new ClickHouseSpec(3, TestDatabaseName.CLICKHOUSE_CLUSTER_CLICKHOUSE_IT);
+
+    Assertions.assertTrue(single.supplies(ClickHouseContainer.class));
+    Assertions.assertFalse(single.supplies(ClickHouseCluster.class));
+    Assertions.assertTrue(cluster.supplies(ClickHouseContainer.class));
+    Assertions.assertTrue(cluster.supplies(ClickHouseCluster.class));
+    Assertions.assertTrue(cluster.supplies(BaseContainer.class));
+  }
+
   @Test
   void testFinalFieldFailsClearly() {
     run(FinalFieldFixture.class)
diff --git a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/TestContainerSpecDependencies.java b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/TestContainerSpecDependencies.java
new file mode 100644
index 00000000000..27c4619762a
--- /dev/null
+++ b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/TestContainerSpecDependencies.java
@@ -0,0 +1,128 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ *  http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */ package org.apache.gravitino.integration.test.container;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.mockito.Mockito;
+
+public class TestContainerSpecDependencies {
+
+  private static final List STARTED = new ArrayList<>();
+  // Each node's direct dependencies; the graph a test sets before it starts a node.
+  private static Map> graph = Map.of();
+
+  @BeforeEach
+  void clear() {
+    STARTED.clear();
+  }
+
+  @Test
+  void testDependenciesStartDeepestFirst() {
+    graph = Map.of("hive", List.of("ranger", "localstack"), "ranger", List.of("solr"));
+
+    ContainerSpec.startWithDependencies(new Node("hive"), null);
+
+    Assertions.assertEquals(List.of("solr", "ranger", "localstack", "hive"), STARTED);
+  }
+
+  @Test
+  void testPrivateStartOnlyStartsDependencies() {
+    graph = Map.of("hive", List.of("ranger"));
+
+    ContainerSpec.startDependencies(new Node("hive"), null);
+
+    Assertions.assertEquals(List.of("ranger"), STARTED);
+  }
+
+  @Test
+  void testCycleFailsClearly() {
+    graph = Map.of("a", List.of("b"), "b", List.of("a"));
+
+    IllegalStateException e =
+        Assertions.assertThrows(
+            IllegalStateException.class,
+            () -> ContainerSpec.startWithDependencies(new Node("a"), null));
+
+    Assertions.assertTrue(e.getMessage().contains("cycle"), e.getMessage());
+    Assertions.assertTrue(STARTED.isEmpty());
+  }
+
+  @Test
+  void testHiveSpecDeclaresWhatItNeeds() {
+    Assertions.assertEquals(
+        List.of(new RangerSpec()), new HiveSpec(HiveVersion.HIVE3, HiveAuth.RANGER).dependencies());
+    Assertions.assertEquals(
+        List.of(new LocalStackSpec()),
+        new HiveSpec(HiveVersion.HIVE3, HiveAuth.NONE, HiveStorage.S3).dependencies());
+    Assertions.assertEquals(
+        List.of(), new HiveSpec(HiveVersion.HIVE2, HiveAuth.NONE).dependencies());
+
+    Assertions.assertThrows(
+        IllegalArgumentException.class, () -> new HiveSpec(HiveVersion.HIVE2, HiveAuth.RANGER));
+    Assertions.assertThrows(
+        IllegalArgumentException.class,
+        () -> new HiveSpec(HiveVersion.HIVE3, HiveAuth.KERBEROS, HiveStorage.S3));
+  }
+
+  @Test
+  void testRangerHiveStartsLazily() {
+    HiveSpec.Factory factory = new HiveSpec.Factory();
+
+    Assertions.assertEquals(
+        Startup.LAZY, factory.startupFor(RangerHiveOnly.class.getAnnotation(HiveFixture.class)));
+    Assertions.assertEquals(
+        Startup.EAGER, factory.startupFor(PlainHiveOnly.class.getAnnotation(HiveFixture.class)));
+  }
+
+  @HiveFixture(version = HiveVersion.HIVE3, auth = HiveAuth.RANGER)
+  static class RangerHiveOnly {}
+
+  @HiveFixture
+  static class PlainHiveOnly {}
+
+  // Only ever mocked, so the constructor never runs.
+  abstract static class FakeContainer extends BaseContainer {
+    FakeContainer() {
+      super((String) null, null, null, null, null, null, Optional.empty());
+    }
+  }
+
+  record Node(String name) implements ContainerSpec {
+    @Override
+    public Class containerType() {
+      return FakeContainer.class;
+    }
+
+    @Override
+    public List> dependencies() {
+      return graph.getOrDefault(name, List.of()).stream().>map(Node::new).toList();
+    }
+
+    @Override
+    public FakeContainer getOrStart(ContainerSuite suite) {
+      STARTED.add(name);
+      return Mockito.mock(FakeContainer.class);
+    }
+  }
+}

From adfc319d442090e68ab669224ec0b53bc7707b61 Mon Sep 17 00:00:00 2001
From: Nevin Zheng 
Date: Tue, 20 Oct 2026 00:58:42 -0700
Subject: [PATCH 14/14] [#14002] improvement(test): Replace the scripted Trino
 stack with TrinoCluster (part 6/6)

TrinoCluster starts the Trino stack through Testcontainers instead of
launch.sh and docker compose, with real readiness checks and Ryuk cleanup.
@TrinoFixture starts lazily and reads the Gravitino server port from the
test.
---
 agent-skills/trino-test/trino-test-guide.md   |  11 +-
 .../docker-script/compose-env.sh              |  31 -
 .../docker-compose-trino-mysql.yaml           |  74 ---
 .../docker-script/docker-compose.yaml         | 148 -----
 .../docker-script/inspect_ip.sh               |  41 --
 .../docker-script/launch.sh                   | 108 ----
 .../docker-script/shutdown.sh                 |  37 --
 .../test/container/ContainerExtension.java    |   6 +-
 .../test/container/ContainerSpec.java         |  11 +
 .../test/container/ContainerSuite.java        |  34 +-
 .../test/container/GravitinoServerAware.java  |  33 ++
 .../test/container/TestTrinoSpec.java         |  56 ++
 .../test/container/TrinoCluster.java          | 542 ++++++++++++++++++
 .../test/container/TrinoFixture.java          |  80 +++
 .../test/container/TrinoITContainers.java     | 170 ------
 .../integration/test/container/TrinoSpec.java |  68 +++
 .../integration/test/util/BaseIT.java         |   4 +-
 .../integration-test/build.gradle.kts         |   7 +-
 .../integration/test/TrinoQueryITBase.java    |  69 ++-
 .../integration/test/TrinoQueryTestTool.java  |   6 +-
 .../trino_integration_test.sh                 |  19 +-
 .../test/web/ui/CatalogsPageTest.java         |  24 +-
 .../test/web/ui/CatalogsPageTest.java         |  24 +-
 23 files changed, 902 insertions(+), 701 deletions(-)
 delete mode 100644 integration-test-common/docker-script/compose-env.sh
 delete mode 100644 integration-test-common/docker-script/docker-compose-trino-mysql.yaml
 delete mode 100644 integration-test-common/docker-script/docker-compose.yaml
 delete mode 100755 integration-test-common/docker-script/inspect_ip.sh
 delete mode 100755 integration-test-common/docker-script/launch.sh
 delete mode 100755 integration-test-common/docker-script/shutdown.sh
 create mode 100644 integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/GravitinoServerAware.java
 create mode 100644 integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/TestTrinoSpec.java
 create mode 100644 integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/TrinoCluster.java
 create mode 100644 integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/TrinoFixture.java
 delete mode 100644 integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/TrinoITContainers.java
 create mode 100644 integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/TrinoSpec.java

diff --git a/agent-skills/trino-test/trino-test-guide.md b/agent-skills/trino-test/trino-test-guide.md
index 2cf74b156d6..26e12cc266d 100644
--- a/agent-skills/trino-test/trino-test-guide.md
+++ b/agent-skills/trino-test/trino-test-guide.md
@@ -544,16 +544,15 @@ Use 5-digit number prefix for easy sorting and reference:
 
 #### Issue 3: Docker container startup failure
 
-**Cause**: Port conflicts or insufficient resources
+**Cause**: Insufficient resources, or containers left from an earlier run
 
 **Solution**:
 ```bash
-# Cleanup old containers
-./integration-test-common/docker-script/shutdown.sh
+# Stop an environment started with --env_only
+./trino-connector/integration-test/trino-test-tools/trino_integration_test.sh --stop
 
-# Check port usage
-lsof -i :8080
-lsof -i :9083
+# List test containers; Testcontainers removes them once their test JVM is gone
+docker ps -a --filter label=org.testcontainers=true
 ```
 
 ## Summary
diff --git a/integration-test-common/docker-script/compose-env.sh b/integration-test-common/docker-script/compose-env.sh
deleted file mode 100644
index 76c6e1c2594..00000000000
--- a/integration-test-common/docker-script/compose-env.sh
+++ /dev/null
@@ -1,31 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#  http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-
-TEST_CONTAINERS=${TEST_CONTAINERS:-default}
-
-if [ "$TEST_CONTAINERS" = "default" ]; then
-  COMPOSE_FILE=docker-compose.yaml
-else
-  COMPOSE_FILE=docker-compose-trino-${TEST_CONTAINERS}.yaml
-fi
-
-if [ ! -f "$COMPOSE_FILE" ]; then
-  echo "ERROR: Unsupported TEST_CONTAINERS value: $TEST_CONTAINERS (expected compose file $COMPOSE_FILE not found)."
-  exit 1
-fi
diff --git a/integration-test-common/docker-script/docker-compose-trino-mysql.yaml b/integration-test-common/docker-script/docker-compose-trino-mysql.yaml
deleted file mode 100644
index 0fe82663b28..00000000000
--- a/integration-test-common/docker-script/docker-compose-trino-mysql.yaml
+++ /dev/null
@@ -1,74 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#  http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-
-services:
-  mysql:
-    image: mysql:8.0
-    container_name: trino-ci-mysql
-    volumes:
-      - ./init/mysql:/docker-entrypoint-initdb.d/
-    environment:
-      MYSQL_ROOT_PASSWORD: ds123
-      MYSQL_USER: trino
-      MYSQL_PASSWORD: ds123
-      MYSQL_DATABASE: gt_db
-    command:
-      --default-authentication-plugin=mysql_native_password
-      --character-set-server=utf8mb4
-      --collation-server=utf8mb4_general_ci
-      --explicit_defaults_for_timestamp=true
-      --lower_case_table_names=1
-    healthcheck:
-      test: ["CMD-SHELL", "mysqladmin ping"]
-      interval: 10s
-      timeout: 60s
-      retries: 5
-
-  trino:
-    image: trinodb/trino:${TRINO_VERSION:-478}
-    container_name: trino-ci-trino
-    environment:
-      - HADOOP_USER_NAME=anonymous
-      - GRAVITINO_HOST_IP=host.docker.internal
-      - GRAVITINO_HOST_PORT=${GRAVITINO_SERVER_PORT:-8090}
-      - GRAVITINO_METALAKE_NAME=test
-      - TRINO_WORKER_NUM=0
-      - TRINO_ROLE=coordinator
-    entrypoint: /bin/bash /tmp/trino/init.sh
-    volumes:
-      - ./init/trino:/tmp/trino
-      - ${GRAVITINO_TRINO_CONNECTOR_DIR:-../../trino-connector/trino-connector-473-478/build/libs}:/usr/lib/trino/plugin/gravitino
-    extra_hosts:
-      - "host.docker.internal:host-gateway"
-    healthcheck:
-      test: ["CMD", "trino", "--execute", "SELECT 1"]
-      interval: 10s
-      timeout: 60s
-      retries: 5
-    depends_on:
-      mysql:
-        condition: service_healthy
-
-networks:
-  default:
-    name: trino-net
-    driver: bridge
-    ipam:
-      config:
-        - subnet: 10.20.31.16/28
diff --git a/integration-test-common/docker-script/docker-compose.yaml b/integration-test-common/docker-script/docker-compose.yaml
deleted file mode 100644
index 264098213c5..00000000000
--- a/integration-test-common/docker-script/docker-compose.yaml
+++ /dev/null
@@ -1,148 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#  http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-services:
-
-  hive:
-    image: apache/gravitino-ci:hive-0.1.20
-    networks:
-      - default
-    container_name: trino-ci-hive
-    environment:
-      - HADOOP_USER_NAME=anonymous
-      - HIVE_RUNTIME_VERSION=${HIVE_RUNTIME_VERSION:-hive2}
-      - TRINO_CONNECTOR_TEST=${TRINO_CONNECTOR_TEST:-false}
-    entrypoint:  /bin/bash /tmp/hive/init.sh
-    volumes:
-      - ./init/hive:/tmp/hive
-    healthcheck:
-      test: ["CMD", "/tmp/check-status.sh"]
-      interval: 10s
-      timeout: 60s
-      retries: 10
-
-  postgresql:
-    image: postgres:13
-    container_name: trino-ci-postgresql
-    restart: always
-    environment:
-      POSTGRES_USER: trino
-      POSTGRES_PASSWORD: ds123
-      ALLOW_IP_RANGE: 0.0.0.0/0
-    networks:
-      - default
-    volumes:
-      - ./init/postgres:/docker-entrypoint-initdb.d/
-    healthcheck:
-      test: ["CMD-SHELL", "pg_isready"]
-      interval: 10s
-      timeout: 60s
-      retries: 5
-
-  mysql:
-    image: mysql:8.0
-    container_name: trino-ci-mysql
-    networks:
-      - default
-    volumes:
-      - ./init/mysql:/docker-entrypoint-initdb.d/
-    environment:
-      MYSQL_ROOT_PASSWORD: ds123
-      MYSQL_USER: trino
-      MYSQL_PASSWORD: ds123
-      MYSQL_DATABASE: gt_db
-    command:
-      --default-authentication-plugin=mysql_native_password
-      --character-set-server=utf8mb4
-      --collation-server=utf8mb4_general_ci
-      --explicit_defaults_for_timestamp=true
-      --lower_case_table_names=1
-    healthcheck:
-      test: ["CMD-SHELL", "mysqladmin ping"]
-      interval: 10s
-      timeout: 60s
-      retries: 5
-
-  trino:
-    image: trinodb/trino:${TRINO_VERSION:-478}
-    networks:
-      - default
-    container_name: trino-ci-trino
-    environment:
-      - HADOOP_USER_NAME=anonymous
-      - GRAVITINO_HOST_IP=host.docker.internal
-      - GRAVITINO_HOST_PORT=${GRAVITINO_SERVER_PORT:-8090}
-      - GRAVITINO_METALAKE_NAME=test
-      - HIVE_HOST_IP=hive
-      - TRINO_WORKER_NUM=${TRINO_WORKER_NUM:-0}
-      - TRINO_ROLE=coordinator
-    entrypoint:  /bin/bash /tmp/trino/init.sh
-    volumes:
-      - ./init/trino:/tmp/trino
-      - ${GRAVITINO_TRINO_CONNECTOR_DIR:-../../trino-connector/trino-connector-473-478/build/libs}:/usr/lib/trino/plugin/gravitino
-    extra_hosts:
-      - "host.docker.internal:host-gateway"
-    healthcheck:
-      test: ["CMD", "trino", "--execute", "SELECT 1"]
-      interval: 10s
-      timeout: 60s
-      retries: 5
-    depends_on:
-      hive:
-        condition: service_healthy
-      postgresql:
-        condition: service_healthy
-      mysql:
-        condition: service_healthy
-      trino-worker:
-        condition: service_healthy
-
-  trino-worker:
-    image: trinodb/trino:${TRINO_VERSION:-478}
-    networks:
-      - default
-    deploy:
-      replicas: ${TRINO_WORKER_NUM:-0}
-    environment:
-      - HADOOP_USER_NAME=anonymous
-      - GRAVITINO_HOST_IP=host.docker.internal
-      - GRAVITINO_HOST_PORT=${GRAVITINO_SERVER_PORT:-8090}
-      - GRAVITINO_METALAKE_NAME=test
-      - HIVE_HOST_IP=hive
-      - TRINO_ROLE=worker
-    entrypoint:  /bin/bash /tmp/trino/init.sh
-    volumes:
-      - ./init/trino:/tmp/trino
-      - ${GRAVITINO_TRINO_CONNECTOR_DIR:-../../trino-connector/trino-connector-473-478/build/libs}:/usr/lib/trino/plugin/gravitino
-    extra_hosts:
-      - "host.docker.internal:host-gateway"
-    depends_on:
-      hive:
-        condition: service_healthy
-      postgresql:
-        condition: service_healthy
-      mysql:
-        condition: service_healthy
-
-networks:
-  default:
-    name: trino-net
-    driver: bridge
-    ipam:
-      config:
-        - subnet: 10.20.31.16/28
diff --git a/integration-test-common/docker-script/inspect_ip.sh b/integration-test-common/docker-script/inspect_ip.sh
deleted file mode 100755
index c43bb31f5f1..00000000000
--- a/integration-test-common/docker-script/inspect_ip.sh
+++ /dev/null
@@ -1,41 +0,0 @@
-#!/bin/bash
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#  http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-#
-cd "$(dirname "$0")"
-
-container_urls=$(docker inspect --format='{{.Name}}:{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' $(docker ps -q) | grep "/trino-ci-" | sed 's/\/trino-ci-//g')
-
-while IFS=: read -r container_name address; do
-  case "$container_name" in
-    trino)
-      echo "trino_uri=http://$address:8080"
-      ;;
-    hive)
-      echo "hive_uri=thrift://$address:9083"
-      echo "hdfs_uri=hdfs://$address:9000"
-      ;;
-    mysql)
-      echo "mysql_uri=jdbc:mysql://$address:3306"
-      ;;
-    postgresql)
-      echo "postgresql_uri=jdbc:postgresql://$address"
-      ;;
-  esac
-done <<< "$container_urls"
diff --git a/integration-test-common/docker-script/launch.sh b/integration-test-common/docker-script/launch.sh
deleted file mode 100755
index 8537e82d933..00000000000
--- a/integration-test-common/docker-script/launch.sh
+++ /dev/null
@@ -1,108 +0,0 @@
-#!/bin/bash
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#  http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-#
-#set -ex
-cd "$(dirname "$0")"
-
-playground_dir="$(dirname "${BASH_SOURCE-$0}")"
-playground_dir="$(cd "${playground_dir}">/dev/null; pwd)"
-isExist=`which docker`
-if [ ! $isExist ]
-then
-  echo "ERROR: No docker service environment found, please install docker first."
-  exit
-fi
-
-cd ${playground_dir}
-source ./compose-env.sh
-
-# create log dir
-LOG_DIR=../build/trino-ci-container-log
-rm -fr $LOG_DIR
-mkdir -p $LOG_DIR
-LOG_PATH=$LOG_DIR/trino-ci-docker-compose.log
-echo "The docker compose log is: $LOG_PATH"
-
-docker compose -f "$COMPOSE_FILE" up -d
-
-# Count actual created services (excludes services scaled to 0 replicas,
-# e.g. trino-worker when TRINO_WORKER_NUM is unset).
-EXPECTED_SERVICE_COUNT=$(docker compose -f "$COMPOSE_FILE" ps --services | wc -l)
-
-# Stream logs directly to the log file (no console output).
-nohup docker compose -f "$COMPOSE_FILE" logs -f -t | tee -a "$LOG_PATH" &
-LOG_FOLLOW_PID=$!
-cleanup_log_follow() {
-  if [ -n "$LOG_FOLLOW_PID" ] && kill -0 "$LOG_FOLLOW_PID" 2>/dev/null; then
-    kill "$LOG_FOLLOW_PID" >/dev/null 2>&1
-  fi
-}
-trap cleanup_log_follow EXIT
-
-max_attempts=300
-attempts=0
-
-while true; do
-    docker compose -f "$COMPOSE_FILE" exec -T trino trino --execute "SELECT 1" >/dev/null 2>&1 && {
-        break;
-    }
-
-    num_container=$(docker compose -f "$COMPOSE_FILE" ps -q | wc -l)
-    if [ "$num_container" -lt "$EXPECTED_SERVICE_COUNT" ]; then
-        echo "ERROR: Trino-ci containers start failed."
-        exit 1
-    fi
-
-    if [ "$attempts" -ge "$max_attempts" ]; then
-        echo "ERROR: Trino service did not start within the $max_attempts time."
-        exit 1
-    fi
-
-   ((attempts++))
-    sleep 1
-done
-
-if docker compose -f "$COMPOSE_FILE" ps --services | grep -q "^hive$"; then
-  # Wait for HDFS to be fully ready
-  echo "Waiting for HDFS to be fully ready..."
-  hdfs_attempts=0
-  hdfs_max_attempts=60
-
-  while true; do
-      # Check if HDFS DataNode is ready by testing file creation
-      docker compose -f "$COMPOSE_FILE" exec -T hive hdfs dfs -test -d / >/dev/null 2>&1 && \
-      docker compose -f "$COMPOSE_FILE" exec -T hive hdfs dfsadmin -report 2>/dev/null | grep -q "Live datanodes" && {
-          echo "HDFS is ready."
-          # Give HDFS a bit more time to stabilize
-          sleep 5
-          break;
-      }
-
-      if [ "$hdfs_attempts" -ge "$hdfs_max_attempts" ]; then
-          echo "WARNING: HDFS did not become fully ready within $hdfs_max_attempts seconds, but continuing..."
-          break
-      fi
-
-      ((hdfs_attempts++))
-      sleep 1
-  done
-fi
-
-echo "All docker compose service is now available."
diff --git a/integration-test-common/docker-script/shutdown.sh b/integration-test-common/docker-script/shutdown.sh
deleted file mode 100755
index c261938055c..00000000000
--- a/integration-test-common/docker-script/shutdown.sh
+++ /dev/null
@@ -1,37 +0,0 @@
-#!/bin/bash
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#  http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-#
-cd "$(dirname "$0")"
-
-LOG_DIR=../build/trino-ci-container-log
-if [ -d $LOG_DIR ]; then
-  if [ -n "$(docker ps -aq -f name=trino-ci-hive)" ]; then
-    docker cp trino-ci-hive:/usr/local/hadoop/logs $LOG_DIR/hdfs
-    docker cp trino-ci-hive:/tmp/root $LOG_DIR/hive
-  fi
-  if [ -n "$(docker ps -aq -f name=trino-ci-trino)" ]; then
-    docker cp trino-ci-trino:/tmp/trino.log $LOG_DIR/trino.log
-  fi
-fi
-
-export GRAVITINO_TRINO_CONNECTOR_DIR=/dev/null
-for compose_file in docker-compose*.yaml; do
-  docker compose -f "$compose_file" down --remove-orphans
-done
diff --git a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/ContainerExtension.java b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/ContainerExtension.java
index 0dbfe6140f3..e1a9098d8f9 100644
--- a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/ContainerExtension.java
+++ b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/ContainerExtension.java
@@ -266,7 +266,11 @@ private static Declaration declarationFor(Class wanted, Class testClass, S
   // so JUnit closes it with that context. Stores inherit from parent contexts, so a CLASS container
   // also resolves inside the class's tests.
   private static BaseContainer container(Declaration declaration, ExtensionContext context) {
-    ContainerSpec spec = declaration.spec();
+    ContainerSpec spec =
+        context
+            .getTestInstance()
+            .>map(instance -> declaration.spec().forTestInstance(instance))
+            .orElse(declaration.spec());
     if (declaration.scope() == Scope.RUN) {
       return ContainerSpec.startWithDependencies(spec, ContainerSuite.getInstance());
     }
diff --git a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/ContainerSpec.java b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/ContainerSpec.java
index c01f28f25c8..124d27ef2d3 100644
--- a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/ContainerSpec.java
+++ b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/ContainerSpec.java
@@ -118,6 +118,17 @@ private static void startDependencies(
     path.removeLast();
   }
 
+  /**
+   * Returns this spec completed with what the test instance provides, such as the Gravitino
+   * server's port. The extension calls it just before it starts the container.
+   *
+   * @param testInstance the test instance
+   * @return this spec by default
+   */
+  default ContainerSpec forTestInstance(Object testInstance) {
+    return this;
+  }
+
   /**
    * Returns the running container for this spec, starting it on first use.
    *
diff --git a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/ContainerSuite.java b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/ContainerSuite.java
index 3e68818f949..415e9edb86a 100644
--- a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/ContainerSuite.java
+++ b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/ContainerSuite.java
@@ -85,7 +85,8 @@ public class ContainerSuite {
   // Enable the Ranger plugin in the Hive container
   private static volatile HiveContainer hiveRangerContainer;
   private static volatile TrinoContainer trinoContainer;
-  private static volatile TrinoITContainers trinoITContainers;
+  // One cluster per Gravitino server: a test class with another server replaces it.
+  private static final ContainerSlot trinoCluster = new ContainerSlot<>();
   private static volatile RangerContainer rangerContainer;
   private static volatile KafkaContainer kafkaContainer;
   private static volatile DorisContainer dorisContainer;
@@ -773,12 +774,31 @@ public TrinoContainer getTrinoContainer() {
     return trinoContainer;
   }
 
-  public static TrinoITContainers getTrinoITContainers() {
-    if (trinoITContainers == null) {
-      trinoITContainers = new TrinoITContainers();
-      closer.register(trinoITContainers);
+  /**
+   * Starts, or returns the running, Trino cluster for the spec. A spec for another Gravitino server
+   * or shape replaces the running cluster.
+   *
+   * @param spec the cluster to run, with the Gravitino server port set
+   * @return the running cluster
+   */
+  public TrinoCluster startTrinoCluster(TrinoSpec spec) {
+    synchronized (ContainerSuite.class) {
+      return trinoCluster.getOrStart(
+          spec,
+          () -> {
+            initIfNecessary();
+            TrinoCluster cluster =
+                closer.register(
+                    TrinoCluster.builder()
+                        .withTrinoVersion(spec.version())
+                        .withWorkers(spec.workers())
+                        .withHiveRuntimeVersion(spec.hive().runtimeVersion())
+                        .withGravitinoServerPort(spec.gravitinoServerPort())
+                        .build());
+            start(cluster);
+            return cluster;
+          });
     }
-    return trinoITContainers;
   }
 
   public HiveContainer getHiveContainer() {
@@ -1096,7 +1116,7 @@ private static void resetState() {
     hiveContainer.clear();
     hiveRangerContainer = null;
     trinoContainer = null;
-    trinoITContainers = null;
+    trinoCluster.clear();
     rangerContainer = null;
     kafkaContainer = null;
     dorisContainer = null;
diff --git a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/GravitinoServerAware.java b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/GravitinoServerAware.java
new file mode 100644
index 00000000000..2c76a4c321a
--- /dev/null
+++ b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/GravitinoServerAware.java
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ *  http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.gravitino.integration.test.container;
+
+/**
+ * A test that runs a Gravitino server, so containers that call the server, such as Trino with the
+ * Gravitino connector, can find it. {@code BaseIT} implements it.
+ */
+public interface GravitinoServerAware {
+
+  /**
+   * Returns the port of the running Gravitino server.
+   *
+   * @return the server port
+   */
+  int getGravitinoServerPort();
+}
diff --git a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/TestTrinoSpec.java b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/TestTrinoSpec.java
new file mode 100644
index 00000000000..ce32c0f99dd
--- /dev/null
+++ b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/TestTrinoSpec.java
@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ *  http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */ package org.apache.gravitino.integration.test.container;
+
+import java.util.List;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+
+public class TestTrinoSpec {
+
+  @Test
+  void testFixtureSpecHasNoPortUntilBoundToAServerTest() {
+    TrinoSpec unbound = (TrinoSpec) ContainerExtension.specs(TrinoOnly.class).get(0);
+
+    Assertions.assertEquals(new TrinoSpec(478, 2, HiveVersion.HIVE3, -1), unbound);
+    Assertions.assertEquals(
+        new TrinoSpec(478, 2, HiveVersion.HIVE3, 8090),
+        unbound.forTestInstance((GravitinoServerAware) () -> 8090));
+    Assertions.assertSame(unbound, unbound.forTestInstance(new Object()));
+  }
+
+  @Test
+  void testUnboundSpecRefusesToStart() {
+    IllegalStateException e =
+        Assertions.assertThrows(
+            IllegalStateException.class,
+            () -> new TrinoSpec(478, 0, HiveVersion.HIVE2, -1).getOrStart(null));
+
+    Assertions.assertTrue(e.getMessage().contains("GravitinoServerAware"), e.getMessage());
+  }
+
+  @Test
+  void testTrinoFixtureStartsLazilyByDefault() throws NoSuchMethodException {
+    Assertions.assertEquals(
+        Startup.LAZY, TrinoFixture.class.getMethod("startup").getDefaultValue());
+    Assertions.assertEquals(List.of(), new TrinoSpec(478, 0, HiveVersion.HIVE2, 1).dependencies());
+  }
+
+  @TrinoFixture(workers = 2, hive = HiveVersion.HIVE3)
+  static class TrinoOnly {}
+}
diff --git a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/TrinoCluster.java b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/TrinoCluster.java
new file mode 100644
index 00000000000..6ae9baf0f1d
--- /dev/null
+++ b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/TrinoCluster.java
@@ -0,0 +1,542 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ *  http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.gravitino.integration.test.container;
+
+import static org.testcontainers.shaded.org.awaitility.Awaitility.await;
+
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.ImmutableSet;
+import java.io.File;
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.StandardOpenOption;
+import java.time.Duration;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.UUID;
+import javax.annotation.Nullable;
+import org.apache.gravitino.integration.test.util.ITUtils;
+import org.testcontainers.containers.GenericContainer;
+import org.testcontainers.containers.Network;
+import org.testcontainers.containers.output.OutputFrame;
+import org.testcontainers.containers.wait.strategy.Wait;
+import org.testcontainers.containers.wait.strategy.WaitStrategy;
+import org.testcontainers.utility.MountableFile;
+
+/**
+ * A Trino coordinator with its workers and the Hive, MySQL and PostgreSQL its tests query, all set
+ * up from {@code integration-test-common/docker-script/init}, which the Trino test tool patches per
+ * Trino version; the files are copied in at start.
+ *
+ * 
The cluster owns these services: they are configured for Trino (a {@code trino} database user,
+ * case-insensitive MySQL table names, Hive ACID tables, seed data) and differ from the run's shared
+ * containers. They start in dependency order, each with a readiness check, and stop with the
+ * cluster. Container logs and the Hive and Trino server logs land in {@code
+ * integration-test-common/build/trino-ci-container-log}, which CI uploads.
+ *
+ * 
This object is the coordinator; {@link #serviceUrls()} returns the addresses tests use.
+ */
+public final class TrinoCluster extends BaseContainer {
+
+  /** Which services start next to Trino. */
+  public enum Services {
+    /** Hive, MySQL and PostgreSQL. */
+    ALL,
+    /** MySQL only, for the MySQL-only test sets. */
+    MYSQL
+  }
+
+  public static final int TRINO_PORT = 8080;
+  public static final int DEFAULT_TRINO_VERSION = 478;
+
+  private static final String COORDINATOR_HOST = "trino";
+  private static final String HIVE_HOST = "hive";
+  private static final String MYSQL_HOST = "mysql";
+  private static final String POSTGRESQL_HOST = "postgresql";
+  private static final String DB_PASSWORD = "ds123";
+  // Every Trino node loads the Gravitino connector, which calls the server on the host.
+  private static final Map GRAVITINO_HOST =
+      ImmutableMap.of("host.docker.internal", "host-gateway");
+
+  private final List services;
+  private final List workers;
+  @Nullable private final Network ownNetwork;
+  private final Path logDir;
+
+  private TrinoCluster(
+      GenericContainer coordinator,
+      Map env,
+      Optional network,
+      @Nullable Network ownNetwork,
+      List services,
+      List workers,
+      Path logDir) {
+    super(
+        coordinator,
+        COORDINATOR_HOST,
+        ImmutableSet.of(TRINO_PORT),
+        GRAVITINO_HOST,
+        ImmutableMap.of(),
+        env,
+        network);
+    this.services = services;
+    this.workers = workers;
+    this.ownNetwork = ownNetwork;
+    this.logDir = logDir;
+  }
+
+  public static Builder builder() {
+    return new Builder();
+  }
+
+  // The coordinator's own wait strategy (set by the builder) checks for the gravitino catalog.
+  @Override
+  @Nullable
+  protected WaitStrategy waitStrategy() {
+    return null;
+  }
+
+  @Override
+  protected boolean checkContainerStatus(int retryLimit) {
+    return true;
+  }
+
+  /**
+   * Starts the services, then the workers, then the coordinator, and waits until every worker has
+   * joined. Workers start first and keep retrying the coordinator until it is up.
+   */
+  @Override
+  public void start() {
+    for (BaseContainer service : services) {
+      ContainerSuite.startTimed(service);
+    }
+    for (BaseContainer worker : workers) {
+      ContainerSuite.startTimed(worker);
+    }
+    super.start();
+    if (!workers.isEmpty()) {
+      await()
+          .atMost(Duration.ofMinutes(2))
+          .pollDelay(Duration.ZERO)
+          .pollInterval(Duration.ofSeconds(1))
+          .until(() -> activeWorkers() >= workers.size());
+    }
+  }
+
+  private int activeWorkers() {
+    String out =
+        executeInContainer(
+                "trino",
+                "--execute",
+                "SELECT count(*) FROM system.runtime.nodes WHERE coordinator = false")
+            .getStdout()
+            .replace("\"", "")
+            .trim();
+    return out.isEmpty() ? 0 : Integer.parseInt(out);
+  }
+
+  /**
+   * Returns the addresses tests use, under the keys the compose scripts printed: {@code trino_uri},
+   * {@code hive_uri}, {@code hdfs_uri}, {@code mysql_uri} and {@code postgresql_uri}. Services the
+   * cluster did not start are missing.
+   *
+   * @return the service addresses
+   */
+  public Map serviceUrls() {
+    ImmutableMap.Builder urls = ImmutableMap.builder();
+    urls.put("trino_uri", coordinatorUri());
+    for (BaseContainer service : services) {
+      String ip = service.getContainerIpAddress();
+      switch (service.hostName) {
+        case HIVE_HOST -> {
+          urls.put(
+              "hive_uri", String.format("thrift://%s:%d", ip, HiveContainer.HIVE_METASTORE_PORT));
+          urls.put(
+              "hdfs_uri", String.format("hdfs://%s:%d", ip, HiveContainer.HDFS_DEFAULTFS_PORT));
+        }
+        case MYSQL_HOST -> urls.put("mysql_uri", String.format("jdbc:mysql://%s:3306", ip));
+        case POSTGRESQL_HOST -> urls.put("postgresql_uri", "jdbc:postgresql://" + ip);
+        default -> {}
+      }
+    }
+    return urls.build();
+  }
+
+  /**
+   * Returns the coordinator's HTTP address.
+   *
+   * @return {@code http://:8080}
+   */
+  public String coordinatorUri() {
+    return String.format("http://%s:%d", getContainerIpAddress(), TRINO_PORT);
+  }
+
+  // Copies the server logs for CI first; a failed copy must not keep containers running.
+  @Override
+  public void close() {
+    copyServerLogs();
+    try {
+      super.close();
+    } finally {
+      for (BaseContainer worker : workers) {
+        worker.close();
+      }
+      for (int i = services.size() - 1; i >= 0; i--) {
+        services.get(i).close();
+      }
+      if (ownNetwork != null) {
+        ownNetwork.close();
+      }
+    }
+  }
+
+  private void copyServerLogs() {
+    copyQuietly(this, "/tmp/trino.log", logDir.resolve("trino.log"));
+    for (BaseContainer service : services) {
+      if (HIVE_HOST.equals(service.hostName)) {
+        try {
+          service.executeInContainer(
+              "tar", "czf", "/tmp/hive-logs.tgz", "/usr/local/hadoop/logs", "/tmp/root");
+          copyQuietly(service, "/tmp/hive-logs.tgz", logDir.resolve("hive-logs.tgz"));
+        } catch (RuntimeException e) {
+          LOG.warn("Could not pack the Hive logs for CI", e);
+        }
+      }
+    }
+  }
+
+  private static void copyQuietly(BaseContainer container, String from, Path to) {
+    try {
+      if (container.getContainer().isRunning()) {
+        container.getContainer().copyFileFromContainer(from, to.toString());
+      }
+    } catch (RuntimeException e) {
+      LOG.warn("Could not copy {} for the CI logs", from, e);
+    }
+  }
+
+  /** Builds a {@link TrinoCluster}; {@link #withGravitinoServerPort(int)} is required. */
+  public static final class Builder {
+    private int trinoVersion = DEFAULT_TRINO_VERSION;
+    private int workers;
+    private String hiveRuntimeVersion = HiveContainer.HIVE2;
+    private boolean connectorTest;
+    @Nullable private String connectorDir;
+    private Services services = Services.ALL;
+    private int gravitinoServerPort = -1;
+    private String metalake = "test";
+
+    private Builder() {}
+
+    public Builder withTrinoVersion(int trinoVersion) {
+      this.trinoVersion = trinoVersion;
+      return this;
+    }
+
+    public Builder withWorkers(int workers) {
+      this.workers = workers;
+      return this;
+    }
+
+    public Builder withHiveRuntimeVersion(String hiveRuntimeVersion) {
+      this.hiveRuntimeVersion = hiveRuntimeVersion;
+      return this;
+    }
+
+    /**
+     * Makes Hive create the ACID tables the Trino connector test sets read.
+     *
+     * @param connectorTest whether this cluster runs the Trino connector test sets
+     * @return this builder
+     */
+    public Builder withConnectorTest(boolean connectorTest) {
+      this.connectorTest = connectorTest;
+      return this;
+    }
+
+    /**
+     * Sets the directory holding the Gravitino Trino connector jar for this Trino version.
+     *
+     * @param connectorDir the directory; the 473-478 connector build output by default
+     * @return this builder
+     */
+    public Builder withConnectorDir(String connectorDir) {
+      this.connectorDir = connectorDir;
+      return this;
+    }
+
+    public Builder withServices(Services services) {
+      this.services = services;
+      return this;
+    }
+
+    public Builder withGravitinoServerPort(int gravitinoServerPort) {
+      this.gravitinoServerPort = gravitinoServerPort;
+      return this;
+    }
+
+    public Builder withMetalake(String metalake) {
+      this.metalake = metalake;
+      return this;
+    }
+
+    public TrinoCluster build() {
+      if (gravitinoServerPort <= 0) {
+        throw new IllegalStateException("The Gravitino server port is required");
+      }
+      String root = System.getenv("GRAVITINO_ROOT_DIR");
+      if (root == null || root.isEmpty()) {
+        throw new IllegalStateException("GRAVITINO_ROOT_DIR is not set");
+      }
+      String init = ITUtils.joinPath(root, "integration-test-common", "docker-script", "init");
+      String connector =
+          connectorDir != null
+              ? connectorDir
+              : ITUtils.joinPath(
+                  root, "trino-connector", "trino-connector-473-478", "build", "libs");
+      checkConnectorDir(connector);
+      Path logDir = Path.of(root, "integration-test-common", "build", "trino-ci-container-log");
+      try {
+        // Start from an empty directory, so CI uploads only this run's logs.
+        if (Files.isDirectory(logDir)) {
+          try (java.util.stream.Stream old = Files.list(logDir)) {
+            for (Path file : (Iterable) old::iterator) {
+              if (Files.isRegularFile(file)) {
+                Files.deleteIfExists(file);
+              }
+            }
+          }
+        }
+        Files.createDirectories(logDir);
+      } catch (IOException e) {
+        throw new UncheckedIOException(e);
+      }
+
+      Network shared = ContainerSuite.getInstance().getNetwork();
+      Network ownNetwork = shared == null ? Network.newNetwork() : null;
+      Optional network = Optional.of(shared != null ? shared : ownNetwork);
+
+      List serviceList = new ArrayList<>();
+      if (services == Services.ALL) {
+        serviceList.add(hive(init, network, logDir));
+        serviceList.add(postgresql(init, network, logDir));
+      }
+      serviceList.add(mysql(init, network, logDir));
+
+      String image = "trinodb/trino:" + trinoVersion;
+      List workerList = new ArrayList<>();
+      for (int i = 1; i <= workers; i++) {
+        workerList.add(
+            new Service(
+                trino(image, init, connector, logDir, "trino-worker-" + i),
+                "trino-worker-" + i,
+                GRAVITINO_HOST,
+                trinoEnv("worker"),
+                network));
+      }
+
+      GenericContainer coordinator =
+          trino(image, init, connector, logDir, COORDINATOR_HOST)
+              .waitingFor(
+                  Wait.forSuccessfulCommand(
+                          "trino --execute \"SHOW CATALOGS LIKE 'gravitino'\" | grep -q gravitino")
+                      .withStartupTimeout(Duration.ofMinutes(5)));
+      return new TrinoCluster(
+          coordinator,
+          trinoEnv("coordinator"),
+          network,
+          ownNetwork,
+          List.copyOf(serviceList),
+          List.copyOf(workerList),
+          logDir);
+    }
+
+    private Map trinoEnv(String role) {
+      return ImmutableMap.builder()
+          .put("HADOOP_USER_NAME", "anonymous")
+          .put("GRAVITINO_HOST_IP", "host.docker.internal")
+          .put("GRAVITINO_HOST_PORT", String.valueOf(gravitinoServerPort))
+          .put("GRAVITINO_METALAKE_NAME", metalake)
+          .put("HIVE_HOST_IP", HIVE_HOST)
+          .put("TRINO_WORKER_NUM", String.valueOf(workers))
+          .put("TRINO_ROLE", role)
+          .build();
+    }
+
+    private static GenericContainer trino(
+        String image, String init, String connector, Path logDir, String name) {
+      return new GenericContainer<>(image)
+          .withCopyFileToContainer(
+              MountableFile.forHostPath(ITUtils.joinPath(init, "trino")), "/tmp/trino")
+          .withCopyFileToContainer(
+              MountableFile.forHostPath(connector),
+              TrinoContainer.TRINO_CONTAINER_PLUGIN_GRAVITINO_DIR)
+          .withCreateContainerCmdModifier(
+              cmd -> cmd.withEntrypoint("/bin/bash", "/tmp/trino/init.sh"))
+          .withLogConsumer(fileLog(logDir, name));
+    }
+
+    private BaseContainer hive(String init, Optional network, Path logDir) {
+      GenericContainer hive =
+          new GenericContainer<>(TestImages.HIVE)
+              .withCopyFileToContainer(
+                  MountableFile.forHostPath(ITUtils.joinPath(init, "hive")), "/tmp/hive")
+              .withCreateContainerCmdModifier(
+                  cmd -> cmd.withEntrypoint("/bin/bash", "/tmp/hive/init.sh"))
+              .withLogConsumer(fileLog(logDir, HIVE_HOST))
+              // The metastore answers first; Trino also needs a live HDFS data node.
+              .waitingFor(
+                  Wait.forSuccessfulCommand(
+                          "/tmp/check-status.sh"
+                              + " && hdfs dfsadmin -report 2>/dev/null"
+                              + " | grep -q 'Live datanodes ([1-9]'")
+                      .withStartupTimeout(Duration.ofMinutes(5)));
+      return new Service(
+          hive,
+          HIVE_HOST,
+          ImmutableMap.of(
+              "HADOOP_USER_NAME",
+              "anonymous",
+              HiveContainer.HIVE_RUNTIME_VERSION,
+              hiveRuntimeVersion,
+              "TRINO_CONNECTOR_TEST",
+              String.valueOf(connectorTest)),
+          network);
+    }
+
+    private static BaseContainer postgresql(String init, Optional network, Path logDir) {
+      GenericContainer pg =
+          new GenericContainer<>(TestImages.POSTGRESQL_13)
+              .withCopyFileToContainer(
+                  MountableFile.forHostPath(ITUtils.joinPath(init, "postgres")),
+                  "/docker-entrypoint-initdb.d")
+              .withLogConsumer(fileLog(logDir, POSTGRESQL_HOST))
+              // Once for the init-script server, once for the real one.
+              .waitingFor(
+                  Wait.forLogMessage(".*database system is ready to accept connections.*", 2)
+                      .withStartupTimeout(Duration.ofMinutes(2)));
+      return new Service(
+          pg,
+          POSTGRESQL_HOST,
+          ImmutableMap.of(
+              "POSTGRES_USER", "trino",
+              "POSTGRES_PASSWORD", DB_PASSWORD,
+              "ALLOW_IP_RANGE", "0.0.0.0/0"),
+          network);
+    }
+
+    private static BaseContainer mysql(String init, Optional network, Path logDir) {
+      GenericContainer mysql =
+          new GenericContainer<>(TestImages.MYSQL)
+              .withCopyFileToContainer(
+                  MountableFile.forHostPath(ITUtils.joinPath(init, "mysql")),
+                  "/docker-entrypoint-initdb.d")
+              .withCommand(
+                  "--default-authentication-plugin=mysql_native_password",
+                  "--character-set-server=utf8mb4",
+                  "--collation-server=utf8mb4_general_ci",
+                  "--explicit_defaults_for_timestamp=true",
+                  "--lower_case_table_names=1")
+              .withLogConsumer(fileLog(logDir, MYSQL_HOST))
+              // The init-script server listens on port 0; the real one on 3306.
+              .waitingFor(
+                  Wait.forLogMessage(".*ready for connections.*port: 3306.*", 1)
+                      .withStartupTimeout(Duration.ofMinutes(2)));
+      return new Service(
+          mysql,
+          MYSQL_HOST,
+          ImmutableMap.of(
+              "MYSQL_ROOT_PASSWORD",
+              DB_PASSWORD,
+              "MYSQL_USER",
+              "trino",
+              "MYSQL_PASSWORD",
+              DB_PASSWORD,
+              "MYSQL_DATABASE",
+              "gt_db"),
+          network);
+    }
+
+    private static void checkConnectorDir(String dir) {
+      String[] files = new File(dir).list();
+      if (files == null || files.length == 0) {
+        throw new IllegalStateException(
+            "Gravitino Trino connector directory " + dir + " does not exist or is empty");
+      }
+    }
+
+    // Appends a container's output to /.log for CI.
+    private static java.util.function.Consumer fileLog(Path logDir, String name) {
+      Path file = logDir.resolve(name + ".log");
+      return frame -> {
+        try {
+          Files.writeString(
+              file,
+              frame.getUtf8String(),
+              StandardCharsets.UTF_8,
+              StandardOpenOption.CREATE,
+              StandardOpenOption.APPEND);
+        } catch (IOException e) {
+          // Logging must not break the test.
+        }
+      };
+    }
+  }
+
+  // A service or worker of the cluster: a prepared container with its own readiness check.
+  private static final class Service extends BaseContainer {
+    Service(
+        GenericContainer container,
+        String hostName,
+        Map env,
+        Optional network) {
+      this(container, hostName, ImmutableMap.of(), env, network);
+    }
+
+    Service(
+        GenericContainer container,
+        String hostName,
+        Map extraHosts,
+        Map env,
+        Optional network) {
+      super(container, hostName, ImmutableSet.of(), extraHosts, ImmutableMap.of(), env, network);
+      // Docker answers a reverse lookup of a container's IP with its name, and Hadoop clients in
+      // Trino canonicalize hdfs:// through that lookup. Generated names contain an underscore,
+      // which is not valid in a hostname, so name each container like the old Compose setup did.
+      String name = "trino-ci-" + hostName + "-" + UUID.randomUUID().toString().substring(0, 8);
+      this.container.withCreateContainerCmdModifier(cmd -> cmd.withName(name));
+    }
+
+    @Override
+    @Nullable
+    protected WaitStrategy waitStrategy() {
+      return null;
+    }
+
+    @Override
+    protected boolean checkContainerStatus(int retryLimit) {
+      return true;
+    }
+  }
+}
diff --git a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/TrinoFixture.java b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/TrinoFixture.java
new file mode 100644
index 00000000000..58cc042e3f5
--- /dev/null
+++ b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/TrinoFixture.java
@@ -0,0 +1,80 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ *  http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.gravitino.integration.test.container;
+
+import java.lang.annotation.Documented;
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Inherited;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+import org.junit.jupiter.api.extension.ExtendWith;
+
+/**
+ * Declares that a test class needs a {@link TrinoCluster}: a Trino coordinator, its workers and the
+ * Hive, MySQL and PostgreSQL it queries. The cluster calls the class's Gravitino server, so it
+ * starts lazily: ask for it as a parameter of a {@code @BeforeAll} or test method, which runs after
+ * {@code BaseIT} has started the server.
+ *
+ * 
{@code
+ * @TrinoFixture
+ * class CatalogsPageTest extends BaseWebIT {
+ *   @BeforeAll
+ *   void before(TrinoCluster trino) {
+ *     hiveMetastoreUri = trino.serviceUrls().get("hive_uri");
+ *   }
+ * }
+ * }

+ */
+@Documented
+@Inherited
+@Target({ElementType.TYPE, ElementType.FIELD})
+@Retention(RetentionPolicy.RUNTIME)
+@ExtendWith(ContainerExtension.class)
+@ContainerFixture(TrinoSpec.Factory.class)
+public @interface TrinoFixture {
+
+  /**
+   * Returns the Trino version to run.
+   *
+   * @return the version; 478 by default
+   */
+  int version() default TrinoCluster.DEFAULT_TRINO_VERSION;
+
+  /**
+   * Returns the number of Trino workers besides the coordinator.
+   *
+   * @return the worker count; none by default, so the coordinator runs queries itself
+   */
+  int workers() default 0;
+
+  /**
+   * Returns the Hive version next to Trino.
+   *
+   * @return the Hive version; Hive2 by default
+   */
+  HiveVersion hive() default HiveVersion.HIVE2;
+
+  /**
+   * Returns when the cluster starts.
+   *
+   * @return lazily by default, since the Gravitino server must run first
+   */
+  Startup startup() default Startup.LAZY;
+}
diff --git a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/TrinoITContainers.java b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/TrinoITContainers.java
deleted file mode 100644
index 8c01369cfff..00000000000
--- a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/TrinoITContainers.java
+++ /dev/null
@@ -1,170 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *  http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.gravitino.integration.test.container;
-
-import java.io.File;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.Map;
-import org.apache.gravitino.integration.test.util.CommandExecutor;
-import org.apache.gravitino.integration.test.util.ITUtils;
-import org.apache.gravitino.integration.test.util.ProcessData;
-import org.apache.logging.log4j.util.Strings;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.testcontainers.containers.ContainerLaunchException;
-
-public class TrinoITContainers implements AutoCloseable {
-  private static final Logger LOG = LoggerFactory.getLogger(TrinoITContainers.class);
-
-  private static final String TEST_CONTAINERS = "TEST_CONTAINERS";
-
-  public static String dockerComposeDir;
-
-  Map servicesUri = new HashMap<>();
-  private final Map containerEnv = new HashMap<>();
-
-  TrinoITContainers() {
-    String dir = System.getenv("GRAVITINO_ROOT_DIR");
-    if (Strings.isEmpty(dir)) {
-      throw new RuntimeException("GRAVITINO_ROOT_DIR is not set");
-    }
-
-    dockerComposeDir = ITUtils.joinPath(dir, "integration-test-common", "docker-script");
-  }
-
-  public void launch(int gravitinoServerPort) throws Exception {
-    launch(gravitinoServerPort, "hive2", false, null, null, null, null);
-  }
-
-  public void launch(
-      int gravitinoServerPort,
-      String hiveRuntimeVersion,
-      boolean isTrinoConnectorTest,
-      Integer trinoWorkerNum,
-      Integer trinoVersion,
-      String trinoConnectorDir,
-      String testContainers)
-      throws Exception {
-    servicesUri.clear();
-
-    Map env = new HashMap<>();
-    if (trinoWorkerNum != null) {
-      env.put("TRINO_WORKER_NUM", String.valueOf(trinoWorkerNum));
-    }
-    if (trinoVersion != null) {
-      env.put("TRINO_VERSION", String.valueOf(trinoVersion));
-    }
-    if (trinoConnectorDir != null) {
-      File dir = new File(trinoConnectorDir);
-      if (!dir.exists() || !dir.isDirectory() || dir.list().length == 0) {
-        throw new Exception(
-            "Gravitino trino connector directory %s does not exist or is empty"
-                .formatted(trinoConnectorDir));
-      }
-      env.put("GRAVITINO_TRINO_CONNECTOR_DIR", trinoConnectorDir);
-    }
-    env.put("GRAVITINO_SERVER_PORT", String.valueOf(gravitinoServerPort));
-    env.put("TRINO_CONNECTOR_TEST", String.valueOf(isTrinoConnectorTest));
-    if (hiveRuntimeVersion != null) {
-      env.put("HIVE_RUNTIME_VERSION", hiveRuntimeVersion);
-    }
-    if (Strings.isNotBlank(testContainers)) {
-      env.put(TEST_CONTAINERS, testContainers);
-    }
-    if (System.getProperty("gravitino.log.path") != null) {
-      env.put("GRAVITINO_LOG_PATH", System.getProperty("gravitino.log.path"));
-    }
-    containerEnv.clear();
-    containerEnv.putAll(env);
-
-    shutdown();
-
-    LOG.info("Launching containers with env: {}", env);
-    String command = ITUtils.joinPath(dockerComposeDir, "launch.sh");
-    Object output =
-        CommandExecutor.executeCommandLocalHost(
-            command, false, ProcessData.TypesOfData.STREAMS_MERGED, env);
-    LOG.info("Command {} output:\n{}", command, output);
-
-    String outputString = output.toString();
-    if (Strings.isNotEmpty(outputString)
-        && !outputString.contains("All docker compose service is now available")) {
-      throw new ContainerLaunchException("Failed to start containers:\n " + outputString);
-    }
-
-    resolveServerAddress();
-  }
-
-  private void resolveServerAddress() throws Exception {
-    String command = ITUtils.joinPath(dockerComposeDir, "inspect_ip.sh");
-    Object output =
-        CommandExecutor.executeCommandLocalHost(
-            command, false, ProcessData.TypesOfData.STREAMS_MERGED, containerEnv);
-    LOG.info("Command {} output:\n{}", command, output);
-
-    // expect the output to be like:
-    // trino_uri=http://10.20.30.21:8080
-    // hive_uri=thrift://10.20.30.19:9083
-    // hdfs_uri=hdfs://10.20.30.19:9000
-    // mysql_uri=jdbc:mysql://10.20.30.20:3306
-    // postgresql_uri=jdbc:postgresql://10.20.30.18
-
-    String serviceURLs = output.toString();
-    if (serviceURLs.isEmpty()) {
-      throw new ContainerLaunchException(
-          "Failed to get the container status, the containers have not started");
-    }
-
-    try {
-      String[] serviceInfos = serviceURLs.split("\n");
-      for (String serviceInfo : serviceInfos) {
-        String[] info = serviceInfo.split("=", 2);
-        if (info.length != 2) {
-          continue;
-        }
-
-        servicesUri.put(info[0], info[1]);
-      }
-    } catch (Exception e) {
-      throw new ContainerLaunchException("Unexpected container status :\n" + serviceURLs, e);
-    }
-
-    if (!servicesUri.containsKey("trino_uri")) {
-      throw new ContainerLaunchException("The container for the trino service is not started");
-    }
-  }
-
-  public void shutdown() {
-    String command = ITUtils.joinPath(dockerComposeDir, "shutdown.sh");
-    Object output =
-        CommandExecutor.executeCommandLocalHost(
-            command, false, ProcessData.TypesOfData.STREAMS_MERGED, containerEnv);
-    LOG.info("Command {} output:\n{}", command, output);
-  }
-
-  public Map getServiceUrls() {
-    return Collections.unmodifiableMap(new HashMap<>(servicesUri));
-  }
-
-  @Override
-  public void close() throws Exception {
-    shutdown();
-  }
-}
diff --git a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/TrinoSpec.java b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/TrinoSpec.java
new file mode 100644
index 00000000000..1337faa2d0a
--- /dev/null
+++ b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/TrinoSpec.java
@@ -0,0 +1,68 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ *  http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.gravitino.integration.test.container;
+
+/**
+ * A {@link TrinoCluster} of one Trino version and worker count, talking to one Gravitino server.
+ *
+ * 
Trino needs the Gravitino server's port when it starts, so the spec from {@link TrinoFixture}
+ * has none until {@link #forTestInstance} reads it from a {@link GravitinoServerAware} test; that
+ * is why {@code @TrinoFixture} starts lazily. Each server gets its own cluster: a new port replaces
+ * the running cluster.
+ */
+public record TrinoSpec(int version, int workers, HiveVersion hive, int gravitinoServerPort)
+    implements ContainerSpec {
+
+  private static final int NO_PORT = -1;
+
+  @Override
+  public Class containerType() {
+    return TrinoCluster.class;
+  }
+
+  @Override
+  public ContainerSpec forTestInstance(Object testInstance) {
+    if (testInstance instanceof GravitinoServerAware server) {
+      return new TrinoSpec(version, workers, hive, server.getGravitinoServerPort());
+    }
+    return this;
+  }
+
+  @Override
+  public TrinoCluster getOrStart(ContainerSuite suite) {
+    if (gravitinoServerPort == NO_PORT) {
+      throw new IllegalStateException(
+          "Trino needs a Gravitino server: ask for TrinoCluster in a test or @BeforeAll method of a"
+              + " class that implements GravitinoServerAware, after the server starts");
+    }
+    return suite.startTrinoCluster(this);
+  }
+
+  static final class Factory implements ContainerFixture.SpecFactory {
+    @Override
+    public TrinoSpec specFor(TrinoFixture fixture) {
+      return new TrinoSpec(fixture.version(), fixture.workers(), fixture.hive(), NO_PORT);
+    }
+
+    @Override
+    public Startup startupFor(TrinoFixture fixture) {
+      return fixture.startup();
+    }
+  }
+}
diff --git a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/util/BaseIT.java b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/util/BaseIT.java
index 1db518b1369..c9f6cc821d8 100644
--- a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/util/BaseIT.java
+++ b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/util/BaseIT.java
@@ -62,6 +62,7 @@
 import org.apache.gravitino.integration.test.MiniGravitino;
 import org.apache.gravitino.integration.test.MiniGravitinoContext;
 import org.apache.gravitino.integration.test.container.ContainerSuite;
+import org.apache.gravitino.integration.test.container.GravitinoServerAware;
 import org.apache.gravitino.integration.test.container.MySQLContainer;
 import org.apache.gravitino.integration.test.container.PostgreSQLContainer;
 import org.apache.gravitino.server.GravitinoServer;
@@ -85,7 +86,7 @@
  */
 @ExtendWith({PrintFuncNameExtension.class, CloseContainerExtension.class})
 @TestInstance(TestInstance.Lifecycle.PER_CLASS)
-public class BaseIT {
+public class BaseIT implements GravitinoServerAware {
 
   protected static final ContainerSuite containerSuite = ContainerSuite.getInstance();
 
@@ -139,6 +140,7 @@ public class BaseIT {
 
   protected String originConfig;
 
+  @Override
   public int getGravitinoServerPort() {
     JettyServerConfig jettyServerConfig =
         JettyServerConfig.fromConfig(serverConfig, WEBSERVER_CONF_PREFIX);
diff --git a/trino-connector/integration-test/build.gradle.kts b/trino-connector/integration-test/build.gradle.kts
index 9afdb4aeaa0..353ed2415cc 100644
--- a/trino-connector/integration-test/build.gradle.kts
+++ b/trino-connector/integration-test/build.gradle.kts
@@ -125,13 +125,14 @@ tasks.test {
       // Check whether this module has already built
       val trinoConnectorBuildDir = project(":trino-connector:trino-connector").buildDir
       if (trinoConnectorBuildDir.exists()) {
-        // Check the version Gravitino related jars in build equal to the current project version
+        // Check the version Gravitino related jars in build equal to the current project version.
+        // The build directory can exist without libs: this task needs only the module's classes.
         val invalidGravitinoJars = trinoConnectorBuildDir.resolve("libs").listFiles { _, name -> name.startsWith("gravitino") }?.filter {
           val name = it.name
           !name.endsWith(version + ".jar")
-        }
+        }.orEmpty()
 
-        if (invalidGravitinoJars!!.isNotEmpty()) {
+        if (invalidGravitinoJars.isNotEmpty()) {
           val message = "Mismatched Gravitino JAR versions found in trino-connector/build/libs:\n" +
             "${invalidGravitinoJars.joinToString(", ") { it.name }}\n" +
             "The expected project version is $version. Please clean and rebuild the project."
diff --git a/trino-connector/integration-test/src/test/java/org/apache/gravitino/trino/connector/integration/test/TrinoQueryITBase.java b/trino-connector/integration-test/src/test/java/org/apache/gravitino/trino/connector/integration/test/TrinoQueryITBase.java
index 263dbeab1a5..707af6a94e8 100644
--- a/trino-connector/integration-test/src/test/java/org/apache/gravitino/trino/connector/integration/test/TrinoQueryITBase.java
+++ b/trino-connector/integration-test/src/test/java/org/apache/gravitino/trino/connector/integration/test/TrinoQueryITBase.java
@@ -37,10 +37,11 @@
 import org.apache.gravitino.client.GravitinoAdminClient;
 import org.apache.gravitino.client.GravitinoMetalake;
 import org.apache.gravitino.exceptions.RESTException;
-import org.apache.gravitino.integration.test.container.ContainerSuite;
-import org.apache.gravitino.integration.test.container.TrinoITContainers;
+import org.apache.gravitino.integration.test.container.HiveContainer;
+import org.apache.gravitino.integration.test.container.TrinoCluster;
 import org.apache.gravitino.integration.test.util.BaseIT;
 import org.apache.gravitino.rel.TableCatalog;
+import org.apache.logging.log4j.util.Strings;
 import org.junit.jupiter.api.Assertions;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -65,7 +66,7 @@ public class TrinoQueryITBase {
   public static String postgresqlUri = String.format("jdbc:postgresql://%s", testHost);
 
   protected static GravitinoAdminClient gravitinoClient;
-  protected static TrinoITContainers trinoITContainers;
+  protected static TrinoCluster trinoCluster;
   protected static TrinoQueryRunner trinoQueryRunner;
   protected static Map serviceUrls = new HashMap<>();
 
@@ -111,32 +112,29 @@ private void setEnv() throws Exception {
       gravitinoClient = baseIT.getGravitinoClient();
       gravitinoUri = String.format("http://127.0.0.1:%d", baseIT.getGravitinoServerPort());
 
-      trinoITContainers = ContainerSuite.getTrinoITContainers();
-
-      /**
-       * When connecting to a Hive metastore version 3.x, the Hive connector supports reading from
-       * and writing to insert-only and ACID tables, with full support for partitioning and
-       * bucketing
-       */
-      String hiveRuntimeVersion = "hive3";
-
-      /**
-       * Only Trino connector test will create Hive ACID table when deploy Hive with
-       * `/integration-test-common/docker-script/init/hive/init.sh`
-       */
-      Boolean isTrinoConnectorTest = true;
-
-      trinoITContainers.launch(
-          baseIT.getGravitinoServerPort(),
-          hiveRuntimeVersion,
-          isTrinoConnectorTest,
-          trinoWorkerNum,
-          trinoVersion,
-          trinoConnectorDir,
-          testContainers);
+      // Hive3 creates the ACID tables the connector test sets read; Hive 3 metastores let the
+      // Hive connector read and write insert-only and ACID tables.
+      TrinoCluster.Builder cluster =
+          TrinoCluster.builder()
+              .withGravitinoServerPort(baseIT.getGravitinoServerPort())
+              .withMetalake(metalakeName)
+              .withHiveRuntimeVersion(HiveContainer.HIVE3)
+              .withConnectorTest(true)
+              .withServices(services(testContainers));
+      if (trinoWorkerNum != null) {
+        cluster.withWorkers(trinoWorkerNum);
+      }
+      if (trinoVersion != null) {
+        cluster.withTrinoVersion(trinoVersion);
+      }
+      if (trinoConnectorDir != null) {
+        cluster.withConnectorDir(trinoConnectorDir);
+      }
+      trinoCluster = cluster.build();
+      trinoCluster.start();
 
       serviceUrls.clear();
-      serviceUrls.putAll(trinoITContainers.getServiceUrls());
+      serviceUrls.putAll(trinoCluster.serviceUrls());
       trinoUri = serviceUrls.getOrDefault("trino_uri", trinoUri);
       hiveMetastoreUri = serviceUrls.getOrDefault("hive_uri", hiveMetastoreUri);
       hdfsUri = serviceUrls.getOrDefault("hdfs_uri", hdfsUri);
@@ -153,6 +151,18 @@ private void setEnv() throws Exception {
     }
   }
 
+  // The test sets name their services as the compose scripts did: "mysql" for MySQL only, and
+  // nothing or "default" for every service.
+  private static TrinoCluster.Services services(String testContainers) {
+    if (Strings.isBlank(testContainers) || "default".equals(testContainers)) {
+      return TrinoCluster.Services.ALL;
+    }
+    if ("mysql".equals(testContainers)) {
+      return TrinoCluster.Services.MYSQL;
+    }
+    throw new IllegalArgumentException("Unsupported test containers: " + testContainers);
+  }
+
   public void setup() throws Exception {
     if (started) {
       return;
@@ -172,7 +182,10 @@ public static void cleanup() {
 
     try {
       if (autoStart) {
-        if (trinoITContainers != null) trinoITContainers.shutdown();
+        if (trinoCluster != null) {
+          trinoCluster.close();
+          trinoCluster = null;
+        }
         if (baseIT != null) baseIT.stopIntegrationTest();
       }
     } catch (Exception e) {
diff --git a/trino-connector/integration-test/src/test/java/org/apache/gravitino/trino/connector/integration/test/TrinoQueryTestTool.java b/trino-connector/integration-test/src/test/java/org/apache/gravitino/trino/connector/integration/test/TrinoQueryTestTool.java
index 617c87bfde8..6db877842d4 100644
--- a/trino-connector/integration-test/src/test/java/org/apache/gravitino/trino/connector/integration/test/TrinoQueryTestTool.java
+++ b/trino-connector/integration-test/src/test/java/org/apache/gravitino/trino/connector/integration/test/TrinoQueryTestTool.java
@@ -291,7 +291,11 @@ public static void main(String[] args) throws Exception {
         System.out.println("  Gravitino URI : " + TrinoQueryITBase.gravitinoUri);
         System.out.println("  Trino URI     : " + TrinoQueryITBase.trinoUri);
         System.out.println("Connect to Trino CLI:");
-        System.out.println("  docker exec -it trino-ci-trino trino");
+        String coordinator =
+            TrinoQueryITBase.trinoCluster != null
+                ? TrinoQueryITBase.trinoCluster.getContainer().getContainerId()
+                : "";
+        System.out.println("  docker exec -it " + coordinator + " trino");
         System.out.println("Press Ctrl+C to shutdown the environment.");
         System.out.println("=======================================================");
         try {
diff --git a/trino-connector/integration-test/trino-test-tools/trino_integration_test.sh b/trino-connector/integration-test/trino-test-tools/trino_integration_test.sh
index 166cb305858..ec95bf10770 100755
--- a/trino-connector/integration-test/trino-test-tools/trino_integration_test.sh
+++ b/trino-connector/integration-test/trino-test-tools/trino_integration_test.sh
@@ -28,6 +28,8 @@ cd $GRAVITINO_ROOT_DIR
 
 PID_FILE="$GRAVITINO_ROOT_DIR/integration-test-common/build/trino-test-env.pid"
 
+# The environment's containers belong to the Gradle test JVM: TrinoCluster stops them when the JVM
+# shuts down, and Testcontainers' reaper removes any that are left once it is gone.
 if [ "$1" = "--stop" ]; then
   if [ -f "$PID_FILE" ]; then
     GRADLE_PID=$(cat "$PID_FILE")
@@ -36,21 +38,20 @@ if [ "$1" = "--stop" ]; then
       if [[ -n "$PGID" && "$PGID" =~ ^[0-9]+$ ]]; then
         echo "Stopping environment (PGID: $PGID)..."
         kill -TERM -- "-$PGID" 2>/dev/null
-        for i in $(seq 1 60); do
-          kill -0 "$GRADLE_PID" 2>/dev/null || break
-          sleep 1
-        done
       else
-        echo "Could not determine process group for PID $GRADLE_PID, stopping containers directly..."
+        echo "Could not determine process group for PID $GRADLE_PID, stopping it alone..."
+        kill -TERM "$GRADLE_PID" 2>/dev/null
       fi
+      for i in $(seq 1 60); do
+        kill -0 "$GRADLE_PID" 2>/dev/null || break
+        sleep 1
+      done
     else
-      echo "Environment process ($GRADLE_PID) is no longer running, stopping containers directly..."
+      echo "Environment process ($GRADLE_PID) is no longer running."
     fi
-    "$GRAVITINO_ROOT_DIR/integration-test-common/docker-script/shutdown.sh"
     rm -f "$PID_FILE"
   else
-    echo "No running environment found, stopping containers directly..."
-    "$GRAVITINO_ROOT_DIR/integration-test-common/docker-script/shutdown.sh"
+    echo "No running environment found."
   fi
   exit 0
 fi
diff --git a/web-v2/integration-test/src/test/java/org/apache/gravitino/integration/test/web/ui/CatalogsPageTest.java b/web-v2/integration-test/src/test/java/org/apache/gravitino/integration/test/web/ui/CatalogsPageTest.java
index 193122334e0..5688a1514f3 100644
--- a/web-v2/integration-test/src/test/java/org/apache/gravitino/integration/test/web/ui/CatalogsPageTest.java
+++ b/web-v2/integration-test/src/test/java/org/apache/gravitino/integration/test/web/ui/CatalogsPageTest.java
@@ -30,8 +30,8 @@
 import org.apache.gravitino.NameIdentifier;
 import org.apache.gravitino.client.GravitinoAdminClient;
 import org.apache.gravitino.client.GravitinoMetalake;
-import org.apache.gravitino.integration.test.container.ContainerSuite;
-import org.apache.gravitino.integration.test.container.TrinoITContainers;
+import org.apache.gravitino.integration.test.container.TrinoCluster;
+import org.apache.gravitino.integration.test.container.TrinoFixture;
 import org.apache.gravitino.integration.test.web.ui.pages.CatalogsPage;
 import org.apache.gravitino.integration.test.web.ui.pages.MetalakePage;
 import org.apache.gravitino.integration.test.web.ui.utils.BaseWebIT;
@@ -46,7 +46,6 @@
 import org.apache.gravitino.rel.expressions.transforms.Transform;
 import org.apache.gravitino.rel.types.Types;
 import org.awaitility.Awaitility;
-import org.junit.jupiter.api.AfterAll;
 import org.junit.jupiter.api.Assertions;
 import org.junit.jupiter.api.BeforeAll;
 import org.junit.jupiter.api.MethodOrderer;
@@ -58,11 +57,11 @@
 
 @Tag("gravitino-docker-test")
 @TestMethodOrder(MethodOrderer.OrderAnnotation.class)
+@TrinoFixture
 public class CatalogsPageTest extends BaseWebIT {
   private MetalakePage metalakePage;
   private CatalogsPage catalogsPage;
 
-  protected static TrinoITContainers trinoITContainers;
   protected static GravitinoAdminClient gravitinoClient;
   protected static String gravitinoUri = "http://127.0.0.1:8090";
   protected static String hiveMetastoreUri = "thrift://127.0.0.1:9083";
@@ -114,16 +113,14 @@ public class CatalogsPageTest extends BaseWebIT {
 
   private static String defaultBaseLocation;
 
+  // Asks for the Trino cluster here, after BaseIT has started the Gravitino server it calls.
   @BeforeAll
-  public void before() throws Exception {
+  public void before(TrinoCluster trino) throws Exception {
     gravitinoClient = getGravitinoClient();
 
     gravitinoUri = String.format("http://127.0.0.1:%d", getGravitinoServerPort());
 
-    trinoITContainers = ContainerSuite.getTrinoITContainers();
-    trinoITContainers.launch(getGravitinoServerPort());
-
-    Map serviceUrls = trinoITContainers.getServiceUrls();
+    Map serviceUrls = trino.serviceUrls();
     hiveMetastoreUri = serviceUrls.get("hive_uri");
     hdfsUri = serviceUrls.get("hdfs_uri");
     mysqlUri = serviceUrls.get("mysql_uri");
@@ -202,15 +199,6 @@ private static String storageLocation(String schemaName, String filesetName) {
     return defaultBaseLocation(schemaName) + "/" + filesetName;
   }
 
-  @AfterAll
-  public static void after() {
-    try {
-      if (trinoITContainers != null) trinoITContainers.shutdown();
-    } catch (Exception e) {
-      LOG.error("shutdown trino containers error", e);
-    }
-  }
-
   @Test
   @Order(0)
   public void testDeleteCatalog() throws InterruptedException {
diff --git a/web/integration-test/src/test/java/org/apache/gravitino/integration/test/web/ui/CatalogsPageTest.java b/web/integration-test/src/test/java/org/apache/gravitino/integration/test/web/ui/CatalogsPageTest.java
index 3fe62e4d87f..c40819e26cc 100644
--- a/web/integration-test/src/test/java/org/apache/gravitino/integration/test/web/ui/CatalogsPageTest.java
+++ b/web/integration-test/src/test/java/org/apache/gravitino/integration/test/web/ui/CatalogsPageTest.java
@@ -30,8 +30,8 @@
 import org.apache.gravitino.NameIdentifier;
 import org.apache.gravitino.client.GravitinoAdminClient;
 import org.apache.gravitino.client.GravitinoMetalake;
-import org.apache.gravitino.integration.test.container.ContainerSuite;
-import org.apache.gravitino.integration.test.container.TrinoITContainers;
+import org.apache.gravitino.integration.test.container.TrinoCluster;
+import org.apache.gravitino.integration.test.container.TrinoFixture;
 import org.apache.gravitino.integration.test.web.ui.pages.CatalogsPage;
 import org.apache.gravitino.integration.test.web.ui.pages.MetalakePage;
 import org.apache.gravitino.integration.test.web.ui.utils.BaseWebIT;
@@ -46,7 +46,6 @@
 import org.apache.gravitino.rel.expressions.transforms.Transform;
 import org.apache.gravitino.rel.types.Types;
 import org.awaitility.Awaitility;
-import org.junit.jupiter.api.AfterAll;
 import org.junit.jupiter.api.Assertions;
 import org.junit.jupiter.api.BeforeAll;
 import org.junit.jupiter.api.MethodOrderer;
@@ -58,11 +57,11 @@
 
 @Tag("gravitino-docker-test")
 @TestMethodOrder(MethodOrderer.OrderAnnotation.class)
+@TrinoFixture
 public class CatalogsPageTest extends BaseWebIT {
   private MetalakePage metalakePage;
   private CatalogsPage catalogsPage;
 
-  protected static TrinoITContainers trinoITContainers;
   protected static GravitinoAdminClient gravitinoClient;
   protected static String gravitinoUri = "http://127.0.0.1:8090";
   protected static String hiveMetastoreUri = "thrift://127.0.0.1:9083";
@@ -114,16 +113,14 @@ public class CatalogsPageTest extends BaseWebIT {
 
   private static String defaultBaseLocation;
 
+  // Asks for the Trino cluster here, after BaseIT has started the Gravitino server it calls.
   @BeforeAll
-  public void before() throws Exception {
+  public void before(TrinoCluster trino) throws Exception {
     gravitinoClient = getGravitinoClient();
 
     gravitinoUri = String.format("http://127.0.0.1:%d", getGravitinoServerPort());
 
-    trinoITContainers = ContainerSuite.getTrinoITContainers();
-    trinoITContainers.launch(getGravitinoServerPort());
-
-    Map serviceUrls = trinoITContainers.getServiceUrls();
+    Map serviceUrls = trino.serviceUrls();
     hiveMetastoreUri = serviceUrls.get("hive_uri");
     hdfsUri = serviceUrls.get("hdfs_uri");
     mysqlUri = serviceUrls.get("mysql_uri");
@@ -202,15 +199,6 @@ private static String storageLocation(String schemaName, String filesetName) {
     return defaultBaseLocation(schemaName) + "/" + filesetName;
   }
 
-  @AfterAll
-  public static void after() {
-    try {
-      if (trinoITContainers != null) trinoITContainers.shutdown();
-    } catch (Exception e) {
-      LOG.error("shutdown trino containers error", e);
-    }
-  }
-
   @Test
   @Order(0)
   public void testDeleteCatalog() throws InterruptedException {