diff --git a/Makefile b/Makefile index 76b2d3a66a4..8b5d1b7394c 100644 --- a/Makefile +++ b/Makefile @@ -376,7 +376,7 @@ endif # For all possible env variable that later can be handle by a Dockerfile as a build-arg-file of # a given package, we create a target of the form lk-extra-opt// -LK_POSSIBLE_PKG_PLUS_BUILD_ARGS := pillar/IMM_PROFILING pillar/ARTIFICIAL_LEAK pillar/COVER +LK_POSSIBLE_PKG_PLUS_BUILD_ARGS := pillar/IMM_PROFILING pillar/ARTIFICIAL_LEAK pillar/COVER pillar/FAULT_INJECTION LK_POSSIBLE_BUILD_ARG_TARGETS := $(addprefix lk-extra-opt/,$(LK_POSSIBLE_PKG_PLUS_BUILD_ARGS)) .PHONY: lk-extra-opt/% diff --git a/pkg/pillar/Dockerfile b/pkg/pillar/Dockerfile index 13983ad72f6..74a2f745036 100644 --- a/pkg/pillar/Dockerfile +++ b/pkg/pillar/Dockerfile @@ -89,6 +89,8 @@ ARG IMM_PROFILING=n ENV IMM_PROFILING=${IMM_PROFILING} ARG ARTIFICIAL_LEAK=n ENV ARTIFICIAL_LEAK=${ARTIFICIAL_LEAK} +ARG FAULT_INJECTION=n +ENV FAULT_INJECTION=${FAULT_INJECTION} ARG COVER=n # we need zfs files during build @@ -109,13 +111,14 @@ ARG GOPKGVERSION # make LK build args affect a layer, so value change will invalidate cache RUN echo "IMM_PROFILING: ${IMM_PROFILING}" RUN echo "ARTIFICIAL_LEAK: ${ARTIFICIAL_LEAK}" +RUN echo "FAULT_INJECTION: ${FAULT_INJECTION}" RUN echo "COVER: ${COVER}" # hadolint ignore=DL4006 RUN --mount=type=cache,target=/root/.cache/go-build echo "Running go vet" && make HV="$HV" vet && \ echo "Running go fmt" && ERR="$(find . -name \*.go | grep -v /vendor/ | xargs gofmt -d -e -l -s)" && \ if [ -n "$ERR" ] ; then printf 'go fmt Failed - ERR: %s' "$ERR" ; exit 1 ; fi && \ - make ZARCH=${TARGETARCH} HV="$HV" DEV="$DEV" COVER="$COVER" RSTATS=$RSTATS RSTATS_ENDPOINT=$RSTATS_ENDPOINT RSTATS_TAG=$RSTATS_TAG IMM_PROFILING=$IMM_PROFILING ARTIFICIAL_LEAK=$ARTIFICIAL_LEAK DISTDIR=/final/opt/zededa/bin BUILD_VERSION=${GOPKGVERSION} build + make ZARCH=${TARGETARCH} HV="$HV" DEV="$DEV" COVER="$COVER" RSTATS=$RSTATS RSTATS_ENDPOINT=$RSTATS_ENDPOINT RSTATS_TAG=$RSTATS_TAG IMM_PROFILING=$IMM_PROFILING ARTIFICIAL_LEAK=$ARTIFICIAL_LEAK FAULT_INJECTION=$FAULT_INJECTION DISTDIR=/final/opt/zededa/bin BUILD_VERSION=${GOPKGVERSION} build # Generate the list of non-persistent pubsub topics from the sources being # built; upgradeconverter removes the matching directories under diff --git a/pkg/pillar/Makefile b/pkg/pillar/Makefile index f91f3c030ab..eb4875a8807 100644 --- a/pkg/pillar/Makefile +++ b/pkg/pillar/Makefile @@ -45,6 +45,9 @@ endif ifeq ($(COVER),y) TAGS+=cover endif +ifeq ($(FAULT_INJECTION),y) + TAGS+=faultinjection +endif ifneq ($(TAGS),) TAGS:=-tags "$(TAGS)" endif diff --git a/pkg/pillar/kubeapi/faultinjection_volumedelete.go b/pkg/pillar/kubeapi/faultinjection_volumedelete.go new file mode 100644 index 00000000000..0162a90ba99 --- /dev/null +++ b/pkg/pillar/kubeapi/faultinjection_volumedelete.go @@ -0,0 +1,42 @@ +// Copyright (c) 2026 Zededa, Inc. +// SPDX-License-Identifier: Apache-2.0 + +//go:build k && faultinjection + +package kubeapi + +import ( + "os" +) + +// Fault-injection markers for the volume-delete path, mirroring the +// file-existence idiom of DrainStatusFaultInjectionWait. This file is compiled +// in only under the faultinjection build tag (FAULT_INJECTION=y), so production +// images never carry it; the no-op variant in +// faultinjection_volumedelete_disabled.go is used otherwise. When present, a +// test creates a marker to make a delete fail at a chosen layer, then removes it +// to observe recovery. Two markers target different layers so a test can pin +// down which operation failed: +// - VolumeDestroyFaultPath makes volumeHandlerCSI.DestroyVolume fail for ANY +// volume, including replicated ones (which otherwise skip the PVC delete), +// so it exercises volumemgr's delete-retry loop directly. +// - DeletePVCFaultPath makes kubeapi.DeletePVC fail, reproducing a realistic +// k8s PVC-delete error on the non-replicated path. +const ( + VolumeDestroyFaultPath = "/tmp/VolumeDestroy_FaultInjection_Fail" + DeletePVCFaultPath = "/tmp/DeletePVC_FaultInjection_Fail" +) + +// VolumeDestroyFaultInjected reports whether the DestroyVolume fault marker is +// present; while it is, DestroyVolume returns a synthetic error. +func VolumeDestroyFaultInjected() bool { + _, err := os.Stat(VolumeDestroyFaultPath) + return err == nil +} + +// DeletePVCFaultInjected reports whether the DeletePVC fault marker is present; +// while it is, DeletePVC returns a synthetic error. +func DeletePVCFaultInjected() bool { + _, err := os.Stat(DeletePVCFaultPath) + return err == nil +} diff --git a/pkg/pillar/kubeapi/faultinjection_volumedelete_disabled.go b/pkg/pillar/kubeapi/faultinjection_volumedelete_disabled.go new file mode 100644 index 00000000000..5bc90d6be08 --- /dev/null +++ b/pkg/pillar/kubeapi/faultinjection_volumedelete_disabled.go @@ -0,0 +1,18 @@ +// Copyright (c) 2026 Zededa, Inc. +// SPDX-License-Identifier: Apache-2.0 + +//go:build k && !faultinjection + +package kubeapi + +// No-op variant used when the faultinjection build tag is absent (production +// images). The real, file-existence-driven implementation lives in +// faultinjection_volumedelete.go and is compiled in only under FAULT_INJECTION=y. +// Callers invoke these unconditionally; here they always report "no fault", so +// the delete-path fault branches are dead code the compiler drops. + +// VolumeDestroyFaultInjected always reports false in production builds. +func VolumeDestroyFaultInjected() bool { return false } + +// DeletePVCFaultInjected always reports false in production builds. +func DeletePVCFaultInjected() bool { return false } diff --git a/pkg/pillar/kubeapi/vitoapiserver.go b/pkg/pillar/kubeapi/vitoapiserver.go index 78f0292455f..b7f4789e4e5 100644 --- a/pkg/pillar/kubeapi/vitoapiserver.go +++ b/pkg/pillar/kubeapi/vitoapiserver.go @@ -170,6 +170,13 @@ func checkLonghornSchedulable(nodeName string) error { // DeletePVC : deletes PVC of the given name. func DeletePVC(pvcName string, log *base.LogObject) error { + // Fault injection: while the marker file exists, fail the PVC delete to + // simulate a k8s-API-level delete failure. See DeletePVCFaultPath. + if DeletePVCFaultInjected() { + err := fmt.Errorf("fault-injected DeletePVC failure for %s", pvcName) + log.Warnf("%v", err) + return err + } // Get the Kubernetes clientset clientset, err := GetClientSet() if err != nil { diff --git a/pkg/pillar/volumehandlers/csihandler.go b/pkg/pillar/volumehandlers/csihandler.go index 6a640dbaacd..6b71f9b0002 100644 --- a/pkg/pillar/volumehandlers/csihandler.go +++ b/pkg/pillar/volumehandlers/csihandler.go @@ -282,6 +282,15 @@ func (handler *volumeHandlerCSI) CreateVolume() (string, error) { func (handler *volumeHandlerCSI) DestroyVolume() (string, error) { pvcName := handler.status.GetPVCName() handler.log.Noticef("DestroyVolume called for PVC %s", pvcName) + // Fault injection: while the marker file exists, fail the destroy to + // simulate an unreachable owner node at delete time. Fires before the + // replicated skip so it can exercise the retry loop for replicated volumes + // too. See kubeapi.VolumeDestroyFaultPath. + if kubeapi.VolumeDestroyFaultInjected() { + err := fmt.Errorf("fault-injected DestroyVolume failure for PVC %s", pvcName) + handler.log.Warnf("DestroyVolume: %v", err) + return pvcName, err + } // if this is a replicated volume, do not delete PVC on this node. if !handler.status.IsReplicated { err := kubeapi.DeletePVC(pvcName, handler.log)