Skip to content

Commit 322676b

Browse files
davdhacsclaude
davdhacs
and
claude
committed
Enable UI E2E tests to run on GitHub Actions with PR cluster
This commit enables UI E2E tests to run against ephemeral GKE clusters in GitHub Actions, similar to how Go E2E tests work. Changes: 1. **TEST_MODE deployment support**: - Fix Helm template conditional in secrets.yaml to prevent duplicate secret creation when testMode=true - Generate self-signed certificates at deployment time for TEST_MODE - Delete and recreate secrets to force correct template application - Add test-mode-values.yaml to explicitly set testMode=true 2. **Session secret handling**: - Generate random session secrets for PR cluster deployments - Pass session secret from deploy job to UI E2E test job via outputs - Update Cypress commands to read SESSION_SECRET from environment - Update Makefile to generate and display session secret for local dev 3. **Runtime fixes**: - Handle invalid GCS credentials gracefully in TEST_MODE - Add wait for cluster readiness before deployment 4. **Test improvements**: - Add API error logging to Cypress tests for debugging - Use dynamic session secrets instead of hardcoded local-dev secret - Run UI E2E tests before Go E2E tests (UI tests are faster) - Go E2E tests use port-forward approach like PR 1735 This allows UI E2E tests to authenticate and run against PR clusters that don't have production secrets, enabling automated UI testing in CI. Co-Authored-By: Claude Sonnet 4.5 <noreply@anthropic.com>
1 parent c14ad51 commit 322676b

File tree

8 files changed

+428
-59
lines changed

8 files changed

+428
-59
lines changed

‎.github/workflows/PR.yaml‎

Lines changed: 211 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -66,6 +66,8 @@ jobs:
6666
runs-on: ubuntu-latest
6767
container:
6868
image: quay.io/stackrox-io/apollo-ci:stackrox-test-0.4.9
69+
outputs:
70+
session-secret: ${{ steps.deploy.outputs.session-secret }}
6971
env:
7072
KUBECONFIG: /github/home/artifacts/kubeconfig
7173
INFRA_TOKEN: ${{ secrets.INFRA_TOKEN }}
@@ -100,10 +102,26 @@ jobs:
100102
- name: Download artifacts
101103
run: |
102104
/github/home/.local/bin/infractl artifacts "$CLUSTER_NAME" -d /github/home/artifacts >> "$GITHUB_STEP_SUMMARY"
103-
kubectl get nodes -o wide || true
105+
106+
- name: Wait for cluster to be ready
107+
run: |
108+
echo "Waiting for cluster API server to be ready..."
109+
timeout 300 sh -c 'until kubectl get nodes >/dev/null 2>&1; do
110+
echo "Waiting for cluster..."
111+
sleep 5
112+
done'
113+
echo "Cluster is ready"
114+
kubectl get nodes -o wide
104115
105116
- name: Deploy infra to dev cluster
117+
id: deploy
106118
run: |
119+
# Generate random session secret for JWT signing
120+
# This secret is used by both the server (for verification) and Cypress (for JWT generation)
121+
SESSION_SECRET=$(openssl rand -base64 32 | tr -d '\n')
122+
export SESSION_SECRET
123+
echo "Generated random session secret for this PR cluster deployment"
124+
107125
ENVIRONMENT=development TEST_MODE=true make helm-deploy
108126
sleep 10 # wait for old pods to disappear so the svc port-forward doesn't connect to them
109127
kubectl -n infra port-forward svc/infra-server-service 8443:8443 > /dev/null 2>&1 &
@@ -115,6 +133,11 @@ jobs:
115133
116134
kill %1
117135
136+
# Save session secret for UI E2E tests (job output for next job)
137+
echo "session-secret=$SESSION_SECRET" >> "$GITHUB_OUTPUT"
138+
# Also set as env var for steps in this job
139+
echo "SESSION_SECRET=$SESSION_SECRET" >> "$GITHUB_ENV"
140+
118141
- name: Check the deployment
119142
run: |
120143
kubectl -n infra port-forward svc/infra-server-service 8443:8443 > /dev/null 2>&1 &
@@ -155,9 +178,194 @@ jobs:
155178
run: |
156179
make argo-workflow-lint
157180
158-
- name: Run Go e2e tests
181+
ui-e2e-test-pr-cluster:
182+
needs:
183+
- deploy-and-test
184+
runs-on: ubuntu-latest
185+
# Note: This job does NOT use the apollo-ci container to avoid path issues
186+
env:
187+
KUBECONFIG: /tmp/kubeconfig
188+
INFRA_TOKEN: ${{ secrets.INFRA_TOKEN }}
189+
USE_GKE_GCLOUD_AUTH_PLUGIN: "True"
190+
191+
steps:
192+
- name: Checkout
193+
uses: actions/checkout@v6
194+
with:
195+
fetch-depth: 0
196+
ref: ${{ github.event.pull_request.head.sha }}
197+
path: go/src/github.com/stackrox/infra
198+
199+
- name: Authenticate to GCloud
200+
uses: google-github-actions/auth@v3
201+
with:
202+
credentials_json: ${{ secrets.INFRA_CI_AUTOMATION_GCP_SA }}
203+
204+
- name: Set up Cloud SDK
205+
uses: google-github-actions/setup-gcloud@v3
206+
with:
207+
install_components: "gke-gcloud-auth-plugin"
208+
209+
- name: Download production infractl
210+
uses: stackrox/actions/infra/install-infractl@v1
211+
212+
- name: Get kubeconfig for PR cluster
213+
run: |
214+
echo "Downloading kubeconfig for $CLUSTER_NAME..."
215+
/home/runner/.local/bin/infractl artifacts "$CLUSTER_NAME" -d /tmp/artifacts
216+
cp /tmp/artifacts/kubeconfig "$KUBECONFIG"
217+
218+
echo "Verifying cluster access..."
219+
kubectl get nodes -o wide
220+
221+
- name: Wait for infra-server deployment
222+
run: |
223+
echo "Checking infra-server pods..."
224+
kubectl get pods -n infra
225+
kubectl wait --for=condition=ready pod -l app=infra-server -n infra --timeout=5m
226+
227+
- name: Setup Node.js
228+
uses: actions/setup-node@v4
229+
with:
230+
node-version: '20'
231+
232+
- name: Install UI dependencies
233+
run: |
234+
cd ui
235+
npm install --legacy-peer-deps
236+
237+
- name: Start port-forward to PR cluster
238+
run: |
239+
kubectl -n infra port-forward svc/infra-server-service 8443:8443 >/dev/null 2>&1 &
240+
PORT_FORWARD_PID=$!
241+
echo "PORT_FORWARD_PID=$PORT_FORWARD_PID" >> "$GITHUB_ENV"
242+
echo "Started port-forward with PID: $PORT_FORWARD_PID"
243+
sleep 10
244+
245+
# Verify port-forward is working
246+
echo "Verifying port-forward connectivity..."
247+
timeout 30 sh -c 'until curl -k -f https://localhost:8443/v1/whoami 2>/dev/null; do
248+
echo "Waiting for port-forward..."
249+
sleep 2
250+
done' || {
251+
echo "Port-forward verification failed"
252+
pgrep -a port-forward || true
253+
exit 1
254+
}
255+
echo "Port-forward is working"
256+
257+
- name: Debug - Check flavors API
258+
run: |
259+
echo "Checking if flavors are available..."
260+
261+
# First try without auth (should fail with access denied)
262+
echo "1. Testing without authentication:"
263+
UNAUTH_RESPONSE=$(curl -k -s https://localhost:8443/v1/flavor/list || echo "API call failed")
264+
echo "$UNAUTH_RESPONSE" | jq . || echo "$UNAUTH_RESPONSE"
265+
266+
# Check whoami endpoint
267+
echo ""
268+
echo "2. Testing /v1/whoami:"
269+
WHOAMI=$(curl -k -s https://localhost:8443/v1/whoami || echo "whoami failed")
270+
echo "$WHOAMI" | jq . || echo "$WHOAMI"
271+
272+
# The real issue is the UI itself - let's check if the flavors endpoint
273+
# works at all. The UI must be getting an error from somewhere.
274+
echo ""
275+
echo "3. Checking flavors API (unauthenticated count):"
276+
FLAVOR_COUNT=$(echo "$UNAUTH_RESPONSE" | jq '.flavors | length' 2>/dev/null || echo "0")
277+
echo "Number of flavors available: $FLAVOR_COUNT"
278+
279+
if [ "$FLAVOR_COUNT" = "0" ]; then
280+
echo "NOTE: Flavors API requires authentication"
281+
echo "This is expected - Cypress tests use JWT authentication with randomly generated secret"
282+
fi
283+
284+
- name: Run UI E2E tests
285+
uses: cypress-io/github-action@v6
286+
with:
287+
working-directory: go/src/github.com/stackrox/infra/ui
288+
install: false
289+
start: npm run start
290+
wait-on: 'http://localhost:3001'
291+
wait-on-timeout: 60
292+
command: npm run cypress:run:e2e
159293
env:
160-
INFRA_TOKEN: ${{ secrets.INFRA_TOKEN_DEV }}
294+
BROWSER: none
295+
PORT: 3001
296+
# Backend is the PR cluster deployment accessed via port-forward
297+
# This deployment uses ENVIRONMENT=development with real OIDC (NOT localDeploy=true)
298+
INFRA_API_ENDPOINT: https://localhost:8443
299+
# Session secret for JWT generation (matches what the server uses)
300+
# Retrieved from deploy-and-test job output
301+
CYPRESS_SESSION_SECRET: ${{ needs.deploy-and-test.outputs.session-secret }}
302+
303+
- name: Upload test artifacts
304+
if: failure()
305+
uses: actions/upload-artifact@v4
306+
with:
307+
name: cypress-artifacts-pr-cluster-${{ github.event.pull_request.number }}
308+
path: |
309+
go/src/github.com/stackrox/infra/ui/cypress/videos
310+
go/src/github.com/stackrox/infra/ui/cypress/screenshots
311+
retention-days: 7
312+
313+
- name: Cleanup port-forward
314+
if: always()
315+
run: |
316+
if [ -n "${{ env.PORT_FORWARD_PID }}" ]; then
317+
echo "Cleaning up port-forward (PID: ${{ env.PORT_FORWARD_PID }})..."
318+
kill ${{ env.PORT_FORWARD_PID }} 2>/dev/null || true
319+
fi
320+
pkill -f "kubectl port-forward.*8443:8443" 2>/dev/null || true
321+
322+
go-e2e-test:
323+
needs:
324+
- ui-e2e-test-pr-cluster
325+
runs-on: ubuntu-latest
326+
container:
327+
image: quay.io/stackrox-io/apollo-ci:stackrox-test-0.4.9
328+
env:
329+
KUBECONFIG: /github/home/artifacts/kubeconfig
330+
INFRA_TOKEN: ${{ secrets.INFRA_TOKEN_DEV }}
331+
INFRACTL: bin/infractl -k -e localhost:8443
332+
USE_GKE_GCLOUD_AUTH_PLUGIN: "True"
333+
334+
steps:
335+
- name: Checkout
336+
uses: actions/checkout@v6
337+
with:
338+
fetch-depth: 0
339+
ref: ${{ github.event.pull_request.head.sha }}
340+
path: go/src/github.com/stackrox/infra
341+
342+
- uses: actions/setup-go@v6
343+
with:
344+
go-version-file: go/src/github.com/stackrox/infra/go.mod
345+
346+
- name: Authenticate to GCloud
347+
uses: google-github-actions/auth@v3
348+
with:
349+
credentials_json: ${{ secrets.INFRA_CI_AUTOMATION_GCP_SA }}
350+
351+
- name: Set up Cloud SDK
352+
uses: "google-github-actions/setup-gcloud@v3"
353+
with:
354+
install_components: "gke-gcloud-auth-plugin"
355+
356+
- name: Download production infractl
357+
uses: stackrox/actions/infra/install-infractl@v1
358+
359+
- name: Download artifacts
360+
run: |
361+
/github/home/.local/bin/infractl artifacts "$CLUSTER_NAME" -d /github/home/artifacts >> "$GITHUB_STEP_SUMMARY"
362+
363+
- name: Verify cluster connectivity
364+
run: |
365+
echo "Verifying cluster is accessible..."
366+
kubectl get nodes -o wide
367+
368+
- name: Run Go e2e tests
161369
run: |
162370
kubectl -n infra port-forward svc/infra-server-service 8443:8443 > /dev/null 2>&1 &
163371
sleep 5

‎Makefile‎

Lines changed: 14 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -256,7 +256,15 @@ helm-diff: pre-check helm-dependency-update create-namespaces
256256
## Deploy to local cluster (e.g., Colima) without GCP Secret Manager
257257
.PHONY: deploy-local
258258
deploy-local: helm-dependency-update create-namespaces
259-
TEST_MODE=true ./scripts/deploy/helm.sh deploy-local $(shell make tag) local
259+
@echo "Generating random session secret for local deployment..."
260+
$(eval SESSION_SECRET := $(shell openssl rand -base64 32 | tr -d '\n'))
261+
@echo "SESSION_SECRET generated (use 'export SESSION_SECRET=<value>' for Cypress tests)"
262+
@SESSION_SECRET="$(SESSION_SECRET)" TEST_MODE=true ./scripts/deploy/helm.sh deploy-local $(shell make tag) local
263+
@echo ""
264+
@echo "Deployment complete!"
265+
@echo "To run E2E tests, export the session secret:"
266+
@echo " export SESSION_SECRET='$(SESSION_SECRET)'"
267+
@echo " make test-e2e"
260268

261269
## Run UI E2E tests against local deployment
262270
.PHONY: test-e2e
@@ -276,7 +284,11 @@ test-e2e:
276284
trap cleanup EXIT; \
277285
sleep 5; \
278286
echo "Running Cypress E2E tests..." >&2; \
279-
cd ui && BROWSER=none PORT=3001 INFRA_API_ENDPOINT=http://localhost:8443 npm run test:e2e
287+
if [ -z "$$SESSION_SECRET" ]; then \
288+
echo "WARNING: SESSION_SECRET not set. Using default for local laptop development." >&2; \
289+
echo "If tests fail, make sure you exported SESSION_SECRET from deploy-local output." >&2; \
290+
fi; \
291+
cd ui && BROWSER=none PORT=3001 INFRA_API_ENDPOINT=http://localhost:8443 CYPRESS_SESSION_SECRET="$$SESSION_SECRET" npm run test:e2e
280292

281293
## Bounce pods
282294
.PHONY: bounce-infra-pods

‎chart/infra-server/templates/secrets.yaml‎

Lines changed: 22 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,19 @@
1-
{{- if not .Values.localDeploy }}
1+
{{- if not (or .Values.localDeploy .Values.testMode) }}
2+
---
3+
apiVersion: v1
4+
kind: Secret
5+
type: kubernetes.io/dockerconfigjson
6+
7+
metadata:
8+
name: infra-image-registry-pull-secret
9+
namespace: infra
10+
11+
data:
12+
.dockerconfigjson: {{ template "pull-secret" .Values.pullSecrets.quay }}
13+
14+
---
15+
{{- end }}
16+
{{- if not (or .Values.localDeploy .Values.testMode) }}
217
apiVersion: v1
318
kind: Secret
419

@@ -101,21 +116,9 @@ data:
101116
test-janitor-delete.yaml: |-
102117
{{- tpl (.Files.Get "static/test-janitor-delete.yaml" ) . | b64enc | nindent 4 }}
103118
{{ end }}
104-
105-
---
106-
apiVersion: v1
107-
kind: Secret
108-
type: kubernetes.io/dockerconfigjson
109-
110-
metadata:
111-
name: infra-image-registry-pull-secret
112-
namespace: infra
113-
114-
data:
115-
.dockerconfigjson: {{ template "pull-secret" .Values.pullSecrets.quay }}
116119
{{- end }}
117120

118-
{{- if .Values.localDeploy }}
121+
{{- if or .Values.localDeploy .Values.testMode }}
119122
---
120123
apiVersion: v1
121124
kind: Secret
@@ -127,16 +130,17 @@ metadata:
127130
app.kubernetes.io/name: infra-server
128131

129132
data:
130-
# Minimal config for localDeploy mode
133+
# Minimal config for localDeploy and testMode
131134
infra.yaml: {{ printf "server:\n port: 8443\n cert: /configuration/cert.pem\n key: /configuration/key.pem\n static: /etc/infra/static\n metricsPort: 9101" | b64enc }}
132135

133136
# Minimal OIDC config - uses Google as a valid issuer for initialization
134-
oidc.yaml: {{ printf "issuer: https://accounts.google.com\nclientID: dummy\nclientSecret: dummy\nendpoint: localhost:8443\nsessionSecret: local-dev-secret-min-32-chars-long\ntokenLifetime: 24h" | b64enc }}
137+
# sessionSecret is randomly generated at deployment time for security
138+
oidc.yaml: {{ printf "issuer: https://accounts.google.com\nclientID: dummy\nclientSecret: dummy\nendpoint: localhost:8443\nsessionSecret: %s\ntokenLifetime: 24h" (required ".Values.sessionSecret is required for localDeploy and testMode" .Values.sessionSecret) | b64enc }}
135139

136-
# Empty Google credentials - not used in localDeploy mode
140+
# Empty Google credentials - not used in localDeploy or testMode
137141
google-credentials.json: {{ "{}" | b64enc }}
138142

139-
# Empty BigQuery credentials - not used in localDeploy mode
143+
# Empty BigQuery credentials - not used in localDeploy or testMode
140144
bigquery-sa.json: {{ "{}" | b64enc }}
141145

142146
# Self-signed TLS certificate for local development

‎chart/infra-server/test-mode-values.yaml‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
# Values for TEST_MODE deployments (PR clusters, CI testing)
2+
# This file explicitly sets testMode to true, overriding any values from GCloud secrets
3+
4+
testMode: true

‎cmd/infra-server/main.go‎

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -69,13 +69,19 @@ func mainCmd() error {
6969

7070
// Initialize GCS signer for signed URLs and artifact downloads.
7171
// Only create signer if GOOGLE_APPLICATION_CREDENTIALS is set (production/development).
72-
// Local deployments skip GCS signing entirely.
72+
// Local deployments and test mode skip GCS signing entirely.
7373
var gcsSigner *signer.Signer
74+
testMode := os.Getenv("TEST_MODE") == "true"
7475
if _, hasGCSCredentials := os.LookupEnv("GOOGLE_APPLICATION_CREDENTIALS"); hasGCSCredentials {
7576
var err error
7677
gcsSigner, err = signer.NewFromEnv()
7778
if err != nil {
78-
return errors.Wrapf(err, "failed to load GCS signing credentials")
79+
if testMode {
80+
log.Log(logging.WARN, "GCS signing disabled in TEST_MODE: invalid credentials", "error", err.Error())
81+
gcsSigner = &signer.Signer{} // Empty signer for test mode with invalid credentials
82+
} else {
83+
return errors.Wrapf(err, "failed to load GCS signing credentials")
84+
}
7985
}
8086
} else {
8187
log.Log(logging.INFO, "GCS signing disabled: GOOGLE_APPLICATION_CREDENTIALS not set")

0 commit comments

Comments
 (0)