volumemgr: build-tagged fault injection for volume delete

Makefile

@@ -376,7 +376,7 @@ endif
 
 # For all possible env variable that later can be handle by a Dockerfile as a build-arg-file of
 # a given package, we create a target of the form lk-extra-opt/<pkg-name>/<env-var-name>
-LK_POSSIBLE_PKG_PLUS_BUILD_ARGS := pillar/IMM_PROFILING pillar/ARTIFICIAL_LEAK pillar/COVER
+LK_POSSIBLE_PKG_PLUS_BUILD_ARGS := pillar/IMM_PROFILING pillar/ARTIFICIAL_LEAK pillar/COVER pillar/FAULT_INJECTION
 LK_POSSIBLE_BUILD_ARG_TARGETS := $(addprefix lk-extra-opt/,$(LK_POSSIBLE_PKG_PLUS_BUILD_ARGS))
 
 .PHONY: lk-extra-opt/%

pkg/pillar/Dockerfile

@@ -89,6 +89,8 @@ ARG IMM_PROFILING=n
 ENV IMM_PROFILING=${IMM_PROFILING}
 ARG ARTIFICIAL_LEAK=n
 ENV ARTIFICIAL_LEAK=${ARTIFICIAL_LEAK}
+ARG FAULT_INJECTION=n
+ENV FAULT_INJECTION=${FAULT_INJECTION}
 ARG COVER=n
 
 # we need zfs files during build
@@ -109,13 +111,14 @@ ARG GOPKGVERSION
 # make LK build args affect a layer, so value change will invalidate cache
 RUN echo "IMM_PROFILING: ${IMM_PROFILING}"
 RUN echo "ARTIFICIAL_LEAK: ${ARTIFICIAL_LEAK}"
+RUN echo "FAULT_INJECTION: ${FAULT_INJECTION}"
 RUN echo "COVER: ${COVER}"
 
 # hadolint ignore=DL4006
 RUN --mount=type=cache,target=/root/.cache/go-build echo "Running go vet" && make HV="$HV" vet && \
     echo "Running go fmt" && ERR="$(find . -name \*.go | grep -v /vendor/ | xargs gofmt -d -e -l -s)" && \
        if [ -n "$ERR" ] ; then printf 'go fmt Failed - ERR: %s' "$ERR" ; exit 1 ; fi && \
-       make ZARCH=${TARGETARCH} HV="$HV" DEV="$DEV" COVER="$COVER" RSTATS=$RSTATS RSTATS_ENDPOINT=$RSTATS_ENDPOINT RSTATS_TAG=$RSTATS_TAG IMM_PROFILING=$IMM_PROFILING ARTIFICIAL_LEAK=$ARTIFICIAL_LEAK DISTDIR=/final/opt/zededa/bin BUILD_VERSION=${GOPKGVERSION} build
+       make ZARCH=${TARGETARCH} HV="$HV" DEV="$DEV" COVER="$COVER" RSTATS=$RSTATS RSTATS_ENDPOINT=$RSTATS_ENDPOINT RSTATS_TAG=$RSTATS_TAG IMM_PROFILING=$IMM_PROFILING ARTIFICIAL_LEAK=$ARTIFICIAL_LEAK FAULT_INJECTION=$FAULT_INJECTION DISTDIR=/final/opt/zededa/bin BUILD_VERSION=${GOPKGVERSION} build
 
 # Generate the list of non-persistent pubsub topics from the sources being
 # built; upgradeconverter removes the matching directories under

pkg/pillar/Makefile

@@ -45,6 +45,9 @@ endif
 ifeq ($(COVER),y)
 	TAGS+=cover
 endif
+ifeq ($(FAULT_INJECTION),y)
+	TAGS+=faultinjection
+endif
 ifneq ($(TAGS),)
 	TAGS:=-tags "$(TAGS)"
 endif

pkg/pillar/kubeapi/faultinjection_volumedelete.go

@@ -0,0 +1,42 @@
+// Copyright (c) 2026 Zededa, Inc.
+// SPDX-License-Identifier: Apache-2.0
+
+//go:build k && faultinjection
+
+package kubeapi
+
+import (
+	"os"
+)
+
+// Fault-injection markers for the volume-delete path, mirroring the
+// file-existence idiom of DrainStatusFaultInjectionWait. This file is compiled
+// in only under the faultinjection build tag (FAULT_INJECTION=y), so production
+// images never carry it; the no-op variant in
+// faultinjection_volumedelete_disabled.go is used otherwise. When present, a
+// test creates a marker to make a delete fail at a chosen layer, then removes it
+// to observe recovery. Two markers target different layers so a test can pin
+// down which operation failed:
+//   - VolumeDestroyFaultPath makes volumeHandlerCSI.DestroyVolume fail for ANY
+//     volume, including replicated ones (which otherwise skip the PVC delete),
+//     so it exercises volumemgr's delete-retry loop directly.
+//   - DeletePVCFaultPath makes kubeapi.DeletePVC fail, reproducing a realistic
+//     k8s PVC-delete error on the non-replicated path.
+const (
+	VolumeDestroyFaultPath = "/tmp/VolumeDestroy_FaultInjection_Fail"
+	DeletePVCFaultPath     = "/tmp/DeletePVC_FaultInjection_Fail"
+)
+
+// VolumeDestroyFaultInjected reports whether the DestroyVolume fault marker is
+// present; while it is, DestroyVolume returns a synthetic error.
+func VolumeDestroyFaultInjected() bool {
+	_, err := os.Stat(VolumeDestroyFaultPath)
+	return err == nil
+}
+
+// DeletePVCFaultInjected reports whether the DeletePVC fault marker is present;
+// while it is, DeletePVC returns a synthetic error.
+func DeletePVCFaultInjected() bool {
+	_, err := os.Stat(DeletePVCFaultPath)
+	return err == nil
+}

pkg/pillar/kubeapi/faultinjection_volumedelete_disabled.go

@@ -0,0 +1,18 @@
+// Copyright (c) 2026 Zededa, Inc.
+// SPDX-License-Identifier: Apache-2.0
+
+//go:build k && !faultinjection
+
+package kubeapi
+
+// No-op variant used when the faultinjection build tag is absent (production
+// images). The real, file-existence-driven implementation lives in
+// faultinjection_volumedelete.go and is compiled in only under FAULT_INJECTION=y.
+// Callers invoke these unconditionally; here they always report "no fault", so
+// the delete-path fault branches are dead code the compiler drops.
+
+// VolumeDestroyFaultInjected always reports false in production builds.
+func VolumeDestroyFaultInjected() bool { return false }
+
+// DeletePVCFaultInjected always reports false in production builds.
+func DeletePVCFaultInjected() bool { return false }

pkg/pillar/kubeapi/vitoapiserver.go

@@ -170,6 +170,13 @@ func checkLonghornSchedulable(nodeName string) error {
 
 // DeletePVC : deletes PVC of the given name.
 func DeletePVC(pvcName string, log *base.LogObject) error {
+	// Fault injection: while the marker file exists, fail the PVC delete to
+	// simulate a k8s-API-level delete failure. See DeletePVCFaultPath.
+	if DeletePVCFaultInjected() {
+		err := fmt.Errorf("fault-injected DeletePVC failure for %s", pvcName)
+		log.Warnf("%v", err)
+		return err
+	}
 	// Get the Kubernetes clientset
 	clientset, err := GetClientSet()
 	if err != nil {

pkg/pillar/volumehandlers/csihandler.go

@@ -282,6 +282,15 @@ func (handler *volumeHandlerCSI) CreateVolume() (string, error) {
 func (handler *volumeHandlerCSI) DestroyVolume() (string, error) {
 	pvcName := handler.status.GetPVCName()
 	handler.log.Noticef("DestroyVolume called for PVC %s", pvcName)
+	// Fault injection: while the marker file exists, fail the destroy to
+	// simulate an unreachable owner node at delete time. Fires before the
+	// replicated skip so it can exercise the retry loop for replicated volumes
+	// too. See kubeapi.VolumeDestroyFaultPath.
+	if kubeapi.VolumeDestroyFaultInjected() {
+		err := fmt.Errorf("fault-injected DestroyVolume failure for PVC %s", pvcName)
+		handler.log.Warnf("DestroyVolume: %v", err)
+		return pvcName, err
+	}
 	// if this is a replicated volume, do not delete PVC on this node.
 	if !handler.status.IsReplicated {
 		err := kubeapi.DeletePVC(pvcName, handler.log)