Add retry logic for flaky connectivity tests in boutique #57

Workflow file for this run

.github/workflows/integration.yml at e956524

	name: Integration Tests

	on:
	push:
	branches: [main]
	pull_request:
	branches: [main]
	workflow_dispatch:

	jobs:
	build:
	runs-on: new
	steps:
	- uses: actions/checkout@v6

	- uses: actions/setup-go@v6
	with:
	go-version-file: go.mod

	- name: Build netalchemy binary
	run: go build -o netalchemy ./cmd/netalchemy

	- name: Upload binary
	uses: actions/upload-artifact@v4
	with:
	name: netalchemy-binary
	path: netalchemy

	basic:
	needs: build
	runs-on: new
	steps:
	- uses: actions/checkout@v6

	- name: Download netalchemy binary
	uses: actions/download-artifact@v5
	with:
	name: netalchemy-binary

	- name: Make binary executable
	run: \|
	chmod +x netalchemy
	echo "${{ github.workspace }}" >> $GITHUB_PATH

	- uses: actions/setup-python@v6
	with:
	python-version: "3.12"

	- name: Install NCA
	run: pip install network-config-analyzer

	- name: Configure Docker Hub registry auth
	run: \|
	sudo mkdir -p /etc/rancher/k3s
	cat <<EOF \| sudo tee /etc/rancher/k3s/registries.yaml
	mirrors:
	docker.io:
	endpoint:
	- https://mirror.gcr.io
	configs:
	docker.io:
	auth:
	username: bfenski
	password: ${{ secrets.DOCKER }}
	EOF

	- name: Create k3s cluster
	uses: fenio/setup-k3s@v5

	- name: Deploy manifests
	run: kubectl apply -f examples/kind-lab/manifests/

	- name: Wait for pods to be ready
	run: \|
	kubectl wait --for=condition=Ready pods --all -n data --timeout=120s
	kubectl wait --for=condition=Ready pods --all -n app --timeout=120s
	kubectl wait --for=condition=Ready pods --all -n ingress-nginx --timeout=120s

	- name: Test connectivity (before hardening)
	working-directory: examples/kind-lab
	run: ./test-connectivity.sh \|\| true

	- name: Harden with netalchemy
	working-directory: examples/kind-lab
	run: echo "y" \| ./harden.sh

	- name: Wait for NetworkPolicy propagation
	run: sleep 5

	- name: Test connectivity (after hardening)
	working-directory: examples/kind-lab
	run: \|
	output=$(./test-connectivity.sh 2>&1) \|\| true
	echo "$output"
	if echo "$output" \| grep -q FAIL; then
	echo "::error::Connectivity test failed after hardening"
	exit 1
	fi

	- name: Verify policy matches cluster
	working-directory: examples/kind-lab
	run: netalchemy verify -p policy.yaml

	- name: Break a NetworkPolicy
	run: \|
	ALL=$(kubectl get networkpolicies -A --no-headers \| grep -v default-deny)
	LINE=$(echo "$ALL" \| shuf -n1)
	NS=$(echo "$LINE" \| awk '{print $1}')
	POLICY=$(echo "$LINE" \| awk '{print $2}')
	echo "Deleting NetworkPolicy: $POLICY in namespace $NS"
	kubectl delete networkpolicy "$POLICY" -n "$NS"

	- name: Verify drift detected
	working-directory: examples/kind-lab
	run: \|
	if netalchemy verify -p policy.yaml; then
	echo "::error::verify should have detected drift but reported no violations"
	exit 1
	else
	echo "Drift correctly detected after breaking a NetworkPolicy"
	fi

	- name: Upload policy artifacts
	if: failure()
	uses: actions/upload-artifact@v4
	with:
	name: basic-policies
	path: \|
	examples/kind-lab/policy.yaml
	examples/kind-lab/generated-policies/

	- name: Dump cluster state
	if: failure()
	run: \|
	echo "=== Pods ==="
	kubectl get pods -A
	echo "=== Events ==="
	kubectl get events -A --sort-by=.lastTimestamp \| tail -50
	echo "=== NetworkPolicies ==="
	kubectl get networkpolicies -A -o yaml

	boutique:
	needs: build
	runs-on: new
	steps:
	- uses: actions/checkout@v6

	- name: Download netalchemy binary
	uses: actions/download-artifact@v5
	with:
	name: netalchemy-binary

	- name: Make binary executable
	run: \|
	chmod +x netalchemy
	echo "${{ github.workspace }}" >> $GITHUB_PATH

	- uses: actions/setup-python@v6
	with:
	python-version: "3.12"

	- name: Install NCA
	run: pip install network-config-analyzer

	- name: Configure Docker Hub registry auth
	run: \|
	sudo mkdir -p /etc/rancher/k3s
	cat <<EOF \| sudo tee /etc/rancher/k3s/registries.yaml
	mirrors:
	docker.io:
	endpoint:
	- https://mirror.gcr.io
	configs:
	docker.io:
	auth:
	username: bfenski
	password: ${{ secrets.DOCKER }}
	EOF

	- name: Create k3s cluster
	uses: fenio/setup-k3s@v5

	- name: Deploy Online Boutique
	run: \|
	kubectl create namespace online-boutique
	curl -sL https://raw.githubusercontent.com/GoogleCloudPlatform/microservices-demo/v0.10.2/release/kubernetes-manifests.yaml -o kubernetes-manifests.yaml
	kubectl apply -n online-boutique -f kubernetes-manifests.yaml

	- name: Wait for pods to be ready
	run: kubectl wait --for=condition=Ready pods --all -n online-boutique --timeout=300s

	- name: Test connectivity (before hardening)
	working-directory: examples/kind-lab-boutique
	run: ./test-connectivity.sh \|\| true

	- name: Harden with netalchemy
	working-directory: examples/kind-lab-boutique
	run: echo "y" \| ./harden.sh

	- name: Wait for NetworkPolicy propagation
	run: sleep 15

	- name: Test connectivity (after hardening)
	working-directory: examples/kind-lab-boutique
	run: \|
	output=$(./test-connectivity.sh 2>&1) \|\| true
	echo "$output"
	if echo "$output" \| grep -q FAIL; then
	echo "::error::Connectivity test failed after hardening"
	exit 1
	fi

	- name: Verify policy matches cluster
	working-directory: examples/kind-lab-boutique
	run: netalchemy verify -p policy.yaml

	- name: Break a NetworkPolicy
	run: \|
	ALL=$(kubectl get networkpolicies -n online-boutique --no-headers \| grep -v default-deny)
	LINE=$(echo "$ALL" \| shuf -n1)
	POLICY=$(echo "$LINE" \| awk '{print $1}')
	echo "Deleting NetworkPolicy: $POLICY in namespace online-boutique"
	kubectl delete networkpolicy "$POLICY" -n online-boutique

	- name: Verify drift detected
	working-directory: examples/kind-lab-boutique
	run: \|
	if netalchemy verify -p policy.yaml; then
	echo "::error::verify should have detected drift but reported no violations"
	exit 1
	else
	echo "Drift correctly detected after breaking a NetworkPolicy"
	fi

	- name: Upload policy artifacts
	if: failure()
	uses: actions/upload-artifact@v4
	with:
	name: boutique-policies
	path: \|
	examples/kind-lab-boutique/policy.yaml
	examples/kind-lab-boutique/generated-policies/

	- name: Dump cluster state
	if: failure()
	run: \|
	echo "=== Pods ==="
	kubectl get pods -A
	echo "=== Events ==="
	kubectl get events -A --sort-by=.lastTimestamp \| tail -50
	echo "=== NetworkPolicies ==="
	kubectl get networkpolicies -A -o yaml

	advanced:
	needs: build
	runs-on: new
	steps:
	- uses: actions/checkout@v6

	- name: Download netalchemy binary
	uses: actions/download-artifact@v5
	with:
	name: netalchemy-binary

	- name: Make binary executable
	run: \|
	chmod +x netalchemy
	echo "${{ github.workspace }}" >> $GITHUB_PATH

	- uses: actions/setup-python@v6
	with:
	python-version: "3.12"

	- name: Install NCA
	run: pip install network-config-analyzer

	- name: Configure Docker Hub registry auth
	run: \|
	sudo mkdir -p /etc/rancher/k3s
	cat <<EOF \| sudo tee /etc/rancher/k3s/registries.yaml
	mirrors:
	docker.io:
	endpoint:
	- https://mirror.gcr.io
	configs:
	docker.io:
	auth:
	username: bfenski
	password: ${{ secrets.DOCKER }}
	EOF

	- name: Create k3s cluster
	uses: fenio/setup-k3s@v5

	- name: Deploy namespaces
	run: kubectl apply -f examples/kind-lab-advanced/manifests/namespaces.yaml

	- name: Deploy infra
	run: kubectl apply -f examples/kind-lab-advanced/manifests/infra.yaml

	- name: Deploy tenants
	run: \|
	kubectl apply -f examples/kind-lab-advanced/manifests/tenant-a.yaml
	kubectl apply -f examples/kind-lab-advanced/manifests/tenant-b.yaml

	- name: Deploy monitoring
	run: kubectl apply -f examples/kind-lab-advanced/manifests/monitoring.yaml

	- name: Deploy ingress
	run: kubectl apply -f examples/kind-lab-advanced/manifests/ingress.yaml

	- name: Wait for pods to be ready
	run: \|
	kubectl wait --for=condition=Ready pods --all -n infra --timeout=180s
	kubectl wait --for=condition=Ready pods --all -n tenant-a --timeout=180s
	kubectl wait --for=condition=Ready pods --all -n tenant-b --timeout=180s
	kubectl wait --for=condition=Ready pods --all -n monitoring --timeout=180s
	kubectl wait --for=condition=Ready pods --all -n ingress --timeout=180s

	- name: Test connectivity (before hardening)
	working-directory: examples/kind-lab-advanced
	run: ./test-connectivity.sh \|\| true

	- name: Harden with netalchemy
	working-directory: examples/kind-lab-advanced
	run: ./harden.sh

	- name: Wait for NetworkPolicy propagation
	run: sleep 5

	- name: Test connectivity (after hardening)
	working-directory: examples/kind-lab-advanced
	run: \|
	output=$(./test-connectivity.sh 2>&1) \|\| true
	echo "$output"
	if echo "$output" \| grep -q FAIL; then
	echo "::error::Connectivity test failed after hardening"
	exit 1
	fi

	- name: Verify policy matches cluster
	working-directory: examples/kind-lab-advanced
	run: netalchemy verify -p policy-refined.yaml

	- name: Break a NetworkPolicy
	run: \|
	ALL=$(kubectl get networkpolicies -A --no-headers \| grep -v default-deny)
	LINE=$(echo "$ALL" \| shuf -n1)
	NS=$(echo "$LINE" \| awk '{print $1}')
	POLICY=$(echo "$LINE" \| awk '{print $2}')
	echo "Deleting NetworkPolicy: $POLICY in namespace $NS"
	kubectl delete networkpolicy "$POLICY" -n "$NS"

	- name: Verify drift detected
	working-directory: examples/kind-lab-advanced
	run: \|
	if netalchemy verify -p policy-refined.yaml; then
	echo "::error::verify should have detected drift but reported no violations"
	exit 1
	else
	echo "Drift correctly detected after breaking a NetworkPolicy"
	fi

	- name: Upload policy artifacts
	if: failure()
	uses: actions/upload-artifact@v4
	with:
	name: advanced-policies
	path: \|
	examples/kind-lab-advanced/policy.yaml
	examples/kind-lab-advanced/policy-refined.yaml
	examples/kind-lab-advanced/baseline.yaml
	examples/kind-lab-advanced/observed.yaml
	examples/kind-lab-advanced/generated-policies/

	- name: Dump cluster state
	if: failure()
	run: \|
	echo "=== Pods ==="
	kubectl get pods -A
	echo "=== Events ==="
	kubectl get events -A --sort-by=.lastTimestamp \| tail -50
	echo "=== NetworkPolicies ==="
	kubectl get networkpolicies -A -o yaml

	scan:
	needs: build
	runs-on: new
	steps:
	- uses: actions/checkout@v6

	- name: Download netalchemy binary
	uses: actions/download-artifact@v5
	with:
	name: netalchemy-binary

	- name: Make binary executable
	run: \|
	chmod +x netalchemy
	echo "${{ github.workspace }}" >> $GITHUB_PATH

	- uses: actions/setup-python@v6
	with:
	python-version: "3.12"

	- name: Install NCA
	run: pip install network-config-analyzer

	- name: Configure Docker Hub registry auth
	run: \|
	sudo mkdir -p /etc/rancher/k3s
	cat <<EOF \| sudo tee /etc/rancher/k3s/registries.yaml
	mirrors:
	docker.io:
	endpoint:
	- https://mirror.gcr.io
	configs:
	docker.io:
	auth:
	username: bfenski
	password: ${{ secrets.DOCKER }}
	EOF

	- name: Create k3s cluster
	uses: fenio/setup-k3s@v5

	- name: Deploy manifests
	run: kubectl apply -f examples/kind-lab-observe/manifests/

	- name: Wait for pods to be ready
	run: \|
	kubectl wait --for=condition=Ready pods --all -n data --timeout=120s
	kubectl wait --for=condition=Ready pods --all -n app --timeout=120s
	kubectl wait --for=condition=Ready pods --all -n ingress-nginx --timeout=120s

	- name: Install kubectl-gadget
	uses: fenio/setup-krew@v1
	with:
	plugins: gadget

	- name: Deploy Inspektor Gadget
	run: \|
	kubectl gadget deploy
	kubectl wait --for=condition=Ready pods --all -n gadget --timeout=120s

	- name: Test connectivity (before hardening)
	working-directory: examples/kind-lab-observe
	run: ./test-connectivity.sh \|\| true

	- name: Harden with netalchemy (observe approach)
	working-directory: examples/kind-lab-observe
	run: echo "y" \| ./harden.sh

	- name: Wait for NetworkPolicy propagation
	run: sleep 5

	- name: Test connectivity (after hardening)
	working-directory: examples/kind-lab-observe
	run: \|
	output=$(./test-connectivity.sh 2>&1) \|\| true
	echo "$output"
	if echo "$output" \| grep -q FAIL; then
	echo "::error::Connectivity test failed after hardening"
	exit 1
	fi

	- name: Verify policy matches cluster
	working-directory: examples/kind-lab-observe
	run: netalchemy verify -p policy.yaml

	- name: Break a NetworkPolicy
	run: \|
	ALL=$(kubectl get networkpolicies -A --no-headers \| grep -v default-deny)
	LINE=$(echo "$ALL" \| shuf -n1)
	NS=$(echo "$LINE" \| awk '{print $1}')
	POLICY=$(echo "$LINE" \| awk '{print $2}')
	echo "Deleting NetworkPolicy: $POLICY in namespace $NS"
	kubectl delete networkpolicy "$POLICY" -n "$NS"

	- name: Verify drift detected
	working-directory: examples/kind-lab-observe
	run: \|
	if netalchemy verify -p policy.yaml; then
	echo "::error::verify should have detected drift but reported no violations"
	exit 1
	else
	echo "Drift correctly detected after breaking a NetworkPolicy"
	fi

	- name: Upload policy artifacts
	if: failure()
	uses: actions/upload-artifact@v4
	with:
	name: scan-policies
	path: \|
	examples/kind-lab-observe/policy.yaml
	examples/kind-lab-observe/observed.yaml
	examples/kind-lab-observe/generated-policies/

	- name: Dump cluster state
	if: failure()
	run: \|
	echo "=== Pods ==="
	kubectl get pods -A
	echo "=== Events ==="
	kubectl get events -A --sort-by=.lastTimestamp \| tail -50
	echo "=== NetworkPolicies ==="
	kubectl get networkpolicies -A -o yaml

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

Add retry logic for flaky connectivity tests in boutique #57

Workflow file

Add retry logic for flaky connectivity tests in boutique #57

Uh oh!

Workflow file for this run