Skip to content

Commit 6b272da

Browse files
committed
[#14049] improvement(test): Create Kerberos test principals at container start instead of with kadmin in the container
1 parent 71f9f14 commit 6b272da

6 files changed

Lines changed: 416 additions & 65 deletions

File tree

‎catalogs/catalog-fileset/src/test/java/org/apache/gravitino/catalog/fileset/integration/test/HadoopUserAuthenticationIT.java‎

Lines changed: 19 additions & 56 deletions
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,6 @@
3232
import java.io.IOException;
3333
import java.nio.charset.StandardCharsets;
3434
import java.nio.file.Files;
35-
import java.nio.file.Paths;
3635
import java.util.HashMap;
3736
import java.util.Map;
3837
import org.apache.commons.io.FileUtils;
@@ -61,12 +60,17 @@
6160
import org.junit.jupiter.api.Test;
6261
import org.slf4j.Logger;
6362
import org.slf4j.LoggerFactory;
64-
import org.testcontainers.utility.MountableFile;
6563

66-
// A private Kerberos Hive: the test adds principals with kadmin and changes HDFS permissions,
64+
// A private Kerberos Hive: the test needs principals of its own and changes HDFS permissions,
6765
// which would break the shared Kerberos Hive for later classes.
6866
@Tag("gravitino-docker-test")
69-
@HiveFixture(auth = HiveAuth.KERBEROS, scope = Scope.CLASS)
67+
@HiveFixture(
68+
auth = HiveAuth.KERBEROS,
69+
scope = Scope.CLASS,
70+
kerberosPrincipals = {
71+
HadoopUserAuthenticationIT.HADOOP_SCHEMA_PRINCIPAL,
72+
HadoopUserAuthenticationIT.HADOOP_FILESET_PRINCIPAL
73+
})
7074
public class HadoopUserAuthenticationIT extends BaseIT {
7175
private static final Logger LOG = LoggerFactory.getLogger(HadoopUserAuthenticationIT.class);
7276

@@ -82,14 +86,13 @@ public class HadoopUserAuthenticationIT extends BaseIT {
8286
private static final String HADOOP_CLIENT_PRINCIPAL = "cli@HADOOPKRB";
8387
private static final String HADOOP_CLIENT_KEYTAB = "/client.keytab";
8488

85-
private static final String HADOOP_SCHEMA_PRINCIPAL = "cli_schema";
86-
private static final String HADOOP_FILESET_PRINCIPAL = "cli_fileset";
89+
static final String HADOOP_SCHEMA_PRINCIPAL = "cli_schema";
90+
static final String HADOOP_FILESET_PRINCIPAL = "cli_fileset";
8791

8892
private static final String HADOOP_SCHEMA_KEYTAB = "/cli_schema.keytab";
8993
private static final String HADOOP_FILESET_KEYTAB = "/cli_fileset.keytab";
9094

91-
private static final String REALM = "HADOOPKRB";
92-
private static final String ADMIN_PASSWORD = "Admin12!";
95+
private static final String REALM = HiveContainer.KERBEROS_REALM;
9396

9497
private static String TMP_DIR;
9598

@@ -146,7 +149,7 @@ private static void prepareKerberosConfig() throws IOException {
146149
.getContainer()
147150
.copyFileFromContainer("/gravitino_client.keytab", TMP_DIR + GRAVITINO_CLIENT_KEYTAB);
148151

149-
createKeyTableForSchemaAndFileset();
152+
copyKeytabsOfSchemaAndFileset();
150153

151154
// Keytab of the Gravitino server
152155
kerberosHive
@@ -174,57 +177,17 @@ private static void prepareKerberosConfig() throws IOException {
174177
System.setProperty("sun.security.krb5.debug", "true");
175178
}
176179

177-
private static void createKeyTableForSchemaAndFileset() throws IOException {
178-
String shellContent =
179-
"echo -e \"%s\\n%s\" | kadmin.local -q \"addprinc %s@%s\""
180-
+ "\n"
181-
+ "kadmin.local -q \"xst -k /%s.keytab -norandkey %s@%s\"";
182-
183-
String createSchemaShellFile = String.format("/%s.sh", HADOOP_SCHEMA_PRINCIPAL);
184-
String createFileSetShellFile = String.format("/%s.sh", HADOOP_FILESET_PRINCIPAL);
185-
186-
FileUtils.writeStringToFile(
187-
Paths.get(TMP_DIR + createSchemaShellFile).toFile(),
188-
String.format(
189-
shellContent,
190-
ADMIN_PASSWORD,
191-
ADMIN_PASSWORD,
192-
HADOOP_SCHEMA_PRINCIPAL,
193-
REALM,
194-
HADOOP_SCHEMA_PRINCIPAL,
195-
HADOOP_SCHEMA_PRINCIPAL,
196-
REALM),
197-
StandardCharsets.UTF_8);
180+
// The fixture created both principals and their keytabs when the container started.
181+
private static void copyKeytabsOfSchemaAndFileset() {
198182
kerberosHive
199183
.getContainer()
200-
.copyFileToContainer(
201-
MountableFile.forHostPath(TMP_DIR + createSchemaShellFile), createSchemaShellFile);
202-
kerberosHive.executeInContainer("bash", createSchemaShellFile);
203-
204-
FileUtils.writeStringToFile(
205-
Paths.get(TMP_DIR + createFileSetShellFile).toFile(),
206-
String.format(
207-
shellContent,
208-
ADMIN_PASSWORD,
209-
ADMIN_PASSWORD,
210-
HADOOP_FILESET_PRINCIPAL,
211-
REALM,
212-
HADOOP_FILESET_PRINCIPAL,
213-
HADOOP_FILESET_PRINCIPAL,
214-
REALM),
215-
StandardCharsets.UTF_8);
184+
.copyFileFromContainer(
185+
HiveContainer.kerberosKeytab(HADOOP_SCHEMA_PRINCIPAL), TMP_DIR + HADOOP_SCHEMA_KEYTAB);
216186
kerberosHive
217187
.getContainer()
218-
.copyFileToContainer(
219-
MountableFile.forHostPath(TMP_DIR + createFileSetShellFile), createFileSetShellFile);
220-
kerberosHive.executeInContainer("bash", createFileSetShellFile);
221-
222-
kerberosHive
223-
.getContainer()
224-
.copyFileFromContainer(HADOOP_SCHEMA_KEYTAB, TMP_DIR + HADOOP_SCHEMA_KEYTAB);
225-
kerberosHive
226-
.getContainer()
227-
.copyFileFromContainer(HADOOP_FILESET_KEYTAB, TMP_DIR + HADOOP_FILESET_KEYTAB);
188+
.copyFileFromContainer(
189+
HiveContainer.kerberosKeytab(HADOOP_FILESET_PRINCIPAL),
190+
TMP_DIR + HADOOP_FILESET_KEYTAB);
228191
}
229192

230193
private void addKerberosConfig() {

‎integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/HiveContainer.java‎

Lines changed: 105 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,9 @@
3333
import java.nio.file.Path;
3434
import java.nio.file.Paths;
3535
import java.time.Duration;
36+
import java.util.Collection;
3637
import java.util.Comparator;
38+
import java.util.List;
3739
import java.util.Map;
3840
import java.util.Optional;
3941
import java.util.Set;
@@ -50,6 +52,7 @@
5052
import org.testcontainers.containers.wait.strategy.WaitAllStrategy;
5153
import org.testcontainers.containers.wait.strategy.WaitStrategy;
5254
import org.testcontainers.containers.wait.strategy.WaitStrategyTarget;
55+
import org.testcontainers.images.builder.Transferable;
5356
import org.testcontainers.utility.MountableFile;
5457

5558
public class HiveContainer extends BaseContainer {
@@ -70,6 +73,9 @@ public class HiveContainer extends BaseContainer {
7073
public static final int HDFS_DEFAULTFS_PORT = 9000;
7174
public static final int HIVE_METASTORE_PORT = 9083;
7275
public static final int HIVE_SERVICE_PORT = 10000;
76+
77+
/** The realm of the Kerberos image's KDC. */
78+
public static final String KERBEROS_REALM = "HADOOPKRB";
7379
// The Kerberos image's second metastore, from its hive-site1.xml.
7480
private static final int SECOND_HIVE_METASTORE_PORT = 19083;
7581

@@ -85,8 +91,14 @@ public class HiveContainer extends BaseContainer {
8591
// The image's core-site.xml already lets these users impersonate.
8692
private static final Set<String> USERS_ALLOWED_TO_IMPERSONATE =
8793
ImmutableSet.of("root", "hive", "gravitino");
94+
// Where the container gets the script that creates a test's Kerberos principals.
95+
static final String PRINCIPALS_SCRIPT = "/tmp/gravitino-kerberos-principals.sh";
96+
// The Kerberos image's start script logs in as hive/<host> right after creating its own
97+
// principals and before it configures and starts HDFS; the test's principals go in before it.
98+
static final String PRINCIPALS_HOOK_ANCHOR = "kinit hive/";
8899

89100
private final boolean kerberosEnabled;
101+
private final List<String> kerberosPrincipals;
90102

91103
public static Builder builder() {
92104
return new Builder();
@@ -100,9 +112,26 @@ protected HiveContainer(
100112
Map<String, String> filesToMount,
101113
Map<String, String> envVars,
102114
Optional<Network> network,
103-
boolean kerberosEnabled) {
115+
boolean kerberosEnabled,
116+
List<String> kerberosPrincipals) {
104117
super(image, hostName, ports, extraHosts, filesToMount, envVars, network);
105118
this.kerberosEnabled = kerberosEnabled;
119+
this.kerberosPrincipals = List.copyOf(kerberosPrincipals);
120+
if (!kerberosEnabled && !this.kerberosPrincipals.isEmpty()) {
121+
throw new IllegalArgumentException(
122+
"Kerberos principals need a Kerberos Hive: " + this.kerberosPrincipals);
123+
}
124+
}
125+
126+
/**
127+
* Returns where the Kerberos Hive keeps the keytab of a principal from {@link
128+
* HiveFixture#kerberosPrincipals()}. Tests copy it out with {@code copyFileFromContainer}.
129+
*
130+
* @param principal the principal's name, without realm
131+
* @return the keytab's path in the container
132+
*/
133+
public static String kerberosKeytab(String principal) {
134+
return "/" + principal + ".keytab";
106135
}
107136

108137
public String getHostName() {
@@ -140,6 +169,8 @@ protected void setupContainer() {
140169
public void start() {
141170
if (!kerberosEnabled) {
142171
allowHadoopUserToImpersonate();
172+
} else if (!kerberosPrincipals.isEmpty()) {
173+
createKerberosPrincipals();
143174
}
144175
// Set here rather than in waitStrategy(), which runs before the constructor sets the auth.
145176
container.waitingFor(kerberosEnabled ? kerberosReadiness() : readiness(getHadoopUser()));
@@ -182,6 +213,62 @@ private void allowHadoopUserToImpersonate() {
182213
cmd -> cmd.withEntrypoint("/bin/bash", "-c", script + " && exec /bin/bash " + IMAGE_START));
183214
}
184215

216+
/**
217+
* Creates the test's principals and their keytabs while the container starts.
218+
*
219+
* <p>The KDC's database only exists once the image's start script has created it, so the
220+
* entrypoint hooks a script into the start script, next to the image's own principals and before
221+
* HDFS starts; readiness therefore implies the keytabs exist. A missing hook point or a failed
222+
* principal stops the start script, and with it the container.
223+
*/
224+
private void createKerberosPrincipals() {
225+
container
226+
.withCopyToContainer(
227+
Transferable.of(principalsScript(kerberosPrincipals), 0755), PRINCIPALS_SCRIPT)
228+
.withCreateContainerCmdModifier(
229+
cmd -> cmd.withEntrypoint("/bin/bash", "-c", principalsEntrypoint(IMAGE_START)));
230+
}
231+
232+
/**
233+
* Returns the script that creates each principal in {@link #KERBEROS_REALM} with a keytab at
234+
* {@link #kerberosKeytab(String)}, and fails if a keytab does not hold its principal. {@code
235+
* kadmin.local -q} exits 0 even when its query fails, so the keytab is the check.
236+
*/
237+
static String principalsScript(List<String> principals) {
238+
StringBuilder script = new StringBuilder("set -e\n");
239+
for (String name : principals) {
240+
String principal = name + "@" + KERBEROS_REALM;
241+
String keytab = kerberosKeytab(name);
242+
script
243+
.append(format("kadmin.local -q 'addprinc -randkey %s'\n", principal))
244+
.append(format("kadmin.local -q 'xst -norandkey -k %s %s'\n", keytab, principal))
245+
.append(
246+
format(
247+
"klist -k %1$s | grep -q ' %2$s$'"
248+
+ " || { echo 'No keytab for %2$s at %1$s' >&2; exit 1; }\n",
249+
keytab, principal));
250+
}
251+
return script.toString();
252+
}
253+
254+
/**
255+
* Returns the entrypoint that runs {@link #PRINCIPALS_SCRIPT} inside {@code imageStart}, before
256+
* the line with {@link #PRINCIPALS_HOOK_ANCHOR}, and then runs {@code imageStart}. It fails
257+
* without starting anything if that line is missing.
258+
*/
259+
static String principalsEntrypoint(String imageStart) {
260+
String hook =
261+
format(
262+
"bash %s || { echo \"Cannot create the test Kerberos principals\" >&2; exit 1; }",
263+
PRINCIPALS_SCRIPT);
264+
return format(
265+
"sed -i '/%1$s/i %2$s' %3$s"
266+
+ " && grep -q '^bash %4$s ' %3$s"
267+
+ " || { echo 'Cannot hook the Kerberos principals into %3$s' >&2; exit 1; }"
268+
+ " && exec /bin/bash %3$s",
269+
PRINCIPALS_HOOK_ANCHOR.replace("/", "\\/"), hook, imageStart, PRINCIPALS_SCRIPT);
270+
}
271+
185272
@Override
186273
public void close() {
187274
// A private container is closed by its JUnit store and again by the run's closer at the end of
@@ -407,6 +494,9 @@ private static String httpRequest(String method, String url) {
407494
}
408495

409496
public static class Builder extends BaseContainer.Builder<Builder, HiveContainer> {
497+
// Package-private so that tests can read what a spec asked for.
498+
List<String> kerberosPrincipals = List.of();
499+
410500
private Builder() {
411501
this.image = DEFAULT_IMAGE;
412502
this.hostName = HOST_NAME;
@@ -432,7 +522,20 @@ public HiveContainer build() {
432522
filesToMount,
433523
envVars,
434524
network,
435-
kerberosEnabled);
525+
kerberosEnabled,
526+
kerberosPrincipals);
527+
}
528+
529+
/**
530+
* Sets the extra Kerberos principals a Kerberos Hive creates when it starts.
531+
*
532+
* @param kerberosPrincipals the principal names, without realm; see {@link
533+
* HiveFixture#kerberosPrincipals()}
534+
* @return this builder
535+
*/
536+
public Builder withKerberosPrincipals(Collection<String> kerberosPrincipals) {
537+
this.kerberosPrincipals = List.copyOf(kerberosPrincipals);
538+
return this;
436539
}
437540
}
438541
}

‎integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/HiveFixture.java‎

Lines changed: 11 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -64,9 +64,19 @@
6464
*/
6565
HiveStorage storage() default HiveStorage.HDFS;
6666

67+
/**
68+
* Returns the extra Kerberos principals the Kerberos Hive creates when it starts, before it is
69+
* ready. Each is a name without realm or instance, created in {@link
70+
* HiveContainer#KERBEROS_REALM} with a keytab at {@link HiveContainer#kerberosKeytab(String)}.
71+
* Needs {@code auth = HiveAuth.KERBEROS} and {@code scope = Scope.CLASS} or {@code Scope.METHOD}.
72+
*
73+
* @return the principal names; none by default
74+
*/
75+
String[] kerberosPrincipals() default {};
76+
6777
/**
6878
* Returns how long the Hive container lives. Use {@link Scope#CLASS} or {@link Scope#METHOD} for
69-
* tests that change Hive in ways later tests would see, such as adding Kerberos principals.
79+
* tests that change Hive in ways later tests would see, such as changing HDFS permissions.
7080
*
7181
* @return the scope; the whole run by default, shared with other classes
7282
*/

0 commit comments

Comments
 (0)