Deep Review: 20260412-003842-pr-295

// Reconcile handles App condition changes, Docker watcher lifecycle, container
// spec.state transitions, and finalizer processing for mirror resources.
func (r *EngineReconciler) Reconcile(ctx context.Context, _ ctrl.Request) (ctrl.Result, error) {
	log := logf.FromContext(ctx)

	var app appv1alpha1.App
	if err := r.Get(ctx, client.ObjectKey{Name: appName}, &app); err != nil {
		return ctrl.Result{}, client.IgnoreNotFound(err)
	}

	running := meta.IsStatusConditionTrue(app.Status.Conditions, "Running")
	engineIsDocker := app.Spec.ContainerEngine.Name == engineMoby

	r.watcherMu.Lock()
	watcherRunning := r.watcher != nil
	watcherDied := watcherRunning && !r.watcher.alive()
	if watcherDied {
		r.watcher.stop()
		r.watcher = nil
		watcherRunning = false
	}
	r.watcherMu.Unlock()

	// Watcher died unexpectedly (Docker socket gone, engine restarted, etc.).
	// Clean up mirror resources and set the condition to Disconnected.
	if watcherDied {
		log.Info("Docker watcher died, cleaning up mirror resources")
		if err := r.cleanupMirrorResources(ctx); err != nil {
			log.Error(err, "Failed to clean up mirror resources")
			return ctrl.Result{}, err
		}
		if err := r.setEngineCondition(ctx, &app, metav1.ConditionFalse, "Disconnected", "Container engine connection lost"); err != nil {
			return ctrl.Result{}, err
		}
		return ctrl.Result{}, nil
	}

	// The watcher should only run when the App is Running with the moby
	// backend. In every other state (stopped, containerd, both) we stop
	// the watcher and sweep mirror resources. Cleanup runs on every
	// reconcile in that state so transient errors are retried on the

The engine reconciler decides the watcher lifecycle purely from the App's Running condition; it never consults app.Spec.ContainerEngine.Name. Meanwhile lima-template.yaml:47-52 explicitly disables docker.service when PARAM_CONTAINER_ENGINE=containerd, and newDockerWatcher unconditionally pings the Docker socket. So for a containerd-backed VM, the reconciler enters an endless ConnectFailed loop.

    # Write the selected container engine; docker.service and containerd.service
    mkdir -p /etc/sysconfig
    echo "CONTAINER_ENGINE=${PARAM_CONTAINER_ENGINE}" >/etc/sysconfig/rancher-desktop

    # Enable the requested container engine and disable the other.
    # containerd.service has a Conflicts=docker, so we must not start both.
    CONTAINERD_ACTION=$([[ $PARAM_CONTAINER_ENGINE = containerd ]] && echo enable || echo disable)
    DOCKER_ACTION=$([[ $PARAM_CONTAINER_ENGINE = moby ]] && echo enable || echo disable)
    systemctl "$CONTAINERD_ACTION" containerd.service buildkitd.service
    systemctl "$DOCKER_ACTION" docker.service docker.socket

    if [[ $PARAM_KUBERNETES_ENABLED == "true" ]]; then
        export INSTALL_K3S_VERSION="v${PARAM_KUBERNETES_VERSION}+k3s1"
        INSTALLED_VERSION=""
        [[ -x /usr/local/bin/k3s ]] && INSTALLED_VERSION=$(/usr/local/bin/k3s --version | awk '{print $3; exit}')

This intersects directly with the new rdd set wait path: the help text at cmd/rdd/set.go:70-72 advertises rdd set running=true containerEngine.name=containerd as a supported example, and waitForDesiredState at cmd/rdd/set.go:450-454 waits for ContainerEngineReady=True. In containerd mode that condition can never become True, so the command always hangs until the 300-second timeout.

			runtime. If the App resource does not exist, it is created with
			default settings before applying the specified values.

			By default, rdd set waits for the desired state: when running=true
			is set, it waits for the container engine to be ready; when
			running=false, it waits for the VM to stop. Use --timeout=0 to
			skip waiting.

			Examples:
			  rdd set running=true
			  rdd set running=true containerEngine.name=containerd
			  rdd set kubernetes.enabled=true kubernetes.version=1.32.2
			  rdd set --dry-run running=true

	runningVal, ok := properties["running"]
	if !ok {
		return nil
	}

	ctx, cancel := context.WithTimeout(ctx, timeout)
	defer cancel()

	if runningVal == "true" {
		logrus.Info("Waiting for container engine to be ready")
		return watchCondition(ctx, config, func(obj *unstructured.Unstructured) bool {
			return conditionStatus(obj, "ContainerEngineReady") == metav1.ConditionTrue
		})
	}
	logrus.Info("Waiting for the VM to stop")

Fix: gate the watcher on app.Spec.ContainerEngine.Name == "moby" (or any other Docker-compatible engine you add later). For containerd, either short-circuit ContainerEngineReady to a distinct terminal reason (e.g. Unsupported / NotApplicable) so the CLI can detect "nothing to wait for," or skip the condition entirely and drive the CLI off the App's Running condition instead. Either way, the "waiting for container engine" branch in waitForDesiredState needs a matching adjustment.

	var ns corev1.Namespace
	if err := w.k8s.Get(ctx, client.ObjectKey{Name: apiNamespace}, &ns); err != nil {
		if !apierrors.IsNotFound(err) {
			return err
		}
		ns = corev1.Namespace{
			ObjectMeta: metav1.ObjectMeta{Name: apiNamespace},
		}
		if err := w.k8s.Create(ctx, &ns); err != nil && !apierrors.IsAlreadyExists(err) {
			return fmt.Errorf("failed to create namespace %s: %w", apiNamespace, err)
		}
	}
	return nil
}

// syncContainerNamespace creates the "moby" ContainerNamespace resource.
// Unlike Container / Image / Volume mirrors, this resource has no mirror

syncContainerNamespace now adds the engine.rancherdesktop.io/docker-mirror finalizer to the moby ContainerNamespace.

processFinalizers at engine_reconciler.go:266-281 only covers Containers, Images, and Volumes, and SetupWithManager at lines 354-361 does not watch ContainerNamespaces.

				}
				return err
			}
			if !removeFinalizer(latest, mirrorFinalizer) {
				return nil
			}
			return r.Update(ctx, latest)
		})
		if retryErr != nil {
			errs = append(errs, fmt.Errorf("failed to remove finalizer from %s/%s: %w",
				kind, obj.GetName(), retryErr))
			continue
		}
		if err := client.IgnoreNotFound(r.Delete(ctx, obj)); err != nil {
			errs = append(errs, fmt.Errorf("failed to delete %s/%s: %w",
				kind, obj.GetName(), err))
		}
	}
	return errors.Join(errs...)
}

// reconcileContainerSpecs handles container spec.state changes by calling
// Docker start/stop.
func (r *EngineReconciler) reconcileContainerSpecs(ctx context.Context) error {
	r.watcherMu.Lock()
	w := r.watcher

The existing ValidateDelete webhook at pkg/controllers/containers/containernamespace/container_namespace_webhooks.go:24-29 returns only a warning, so users can still issue kubectl delete containernamespace/moby. The object gets a deletion timestamp, retains the mirror finalizer, and is never unstuck until the next VM stop (when cleanupMirrorResources strips finalizers wholesale). The pre-existing "namespace delete TODO" just graduated into a visible dead end.

func (c *deleteValidator) ValidateCreate(context.Context, *v1alpha1.ContainerNamespace) (ctrlwebhookadmission.Warnings, error) {
	return nil, errors.New("webhook does not implement create")
}

// ValidateDelete implements [ctrlwebhookadmission.Validator].
func (c *deleteValidator) ValidateDelete(context.Context, *v1alpha1.ContainerNamespace) (ctrlwebhookadmission.Warnings, error) {
	// TODO: We should validate that:
	// The namespace to be deleted must be empty before allowing the delete.
	// Additionally, when using moby backend: we do not delete the `moby` namespace.
	return ctrlwebhookadmission.Warnings{"namespace delete validation not implemented"}, nil
}

// ValidateUpdate implements [ctrlwebhookadmission.Validator].
func (c *deleteValidator) ValidateUpdate(context.Context, *v1alpha1.ContainerNamespace, *v1alpha1.ContainerNamespace) (ctrlwebhookadmission.Warnings, error) {
	return nil, errors.New("webhook does not implement update")
}

Fix: either stop adding the mirror finalizer to ContainerNamespace at all (it isn't reverse-managed and cleanupMirrorResources can still sweep it at stop time), or add a dedicated ContainerNamespace watch + finalizer handler and tighten the delete webhook to reject user-initiated deletes of system-managed namespaces up front.

	}
	return errors.Join(errs...)
}

// reconcileContainerSpecs handles container spec.state changes by calling
// Docker start/stop.
func (r *EngineReconciler) reconcileContainerSpecs(ctx context.Context) error {
	r.watcherMu.Lock()
	w := r.watcher
	r.watcherMu.Unlock()
	if w == nil {
		return nil
	}

	var containers containersv1alpha1.ContainerList
	if err := r.List(ctx, &containers, client.InNamespace(apiNamespace)); err != nil {
		return err
	}

	for i := range containers.Items {
		c := &containers.Items[i]
		if c.DeletionTimestamp != nil {
			continue
		}
		if err := w.reconcileContainerState(ctx, c); err != nil {
			logf.FromContext(ctx).Error(err, "Failed to reconcile container state",
				"container", c.Name)
		}
	}
	return nil
}

// processFinalizers handles resources with a deletion timestamp by deleting
// the corresponding Docker object and removing the finalizer.
func (r *EngineReconciler) processFinalizers(ctx context.Context) error {
	r.watcherMu.Lock()
	w := r.watcher
	r.watcherMu.Unlock()
	if w == nil {
		return nil
	}

	if err := r.processContainerFinalizers(ctx, w); err != nil {
		return err
	}
	if err := r.processImageFinalizers(ctx, w); err != nil {
		return err
	}
	return r.processVolumeFinalizers(ctx, w)
}

// processContainerFinalizers deletes the Docker-side container for every
// K8s Container pending deletion and only strips the mirror finalizer
// when the Docker delete succeeds. Per-item errors are collected so one
// stuck container does not block the rest; the reconciler retries the
// remaining stuck items on the next reconcile.
func (r *EngineReconciler) processContainerFinalizers(ctx context.Context, w *dockerWatcher) error {
	var containers containersv1alpha1.ContainerList
	if err := r.List(ctx, &containers, client.InNamespace(apiNamespace)); err != nil {
		return err
	}
	var errs []error
	for i := range containers.Items {
		c := &containers.Items[i]
		if c.DeletionTimestamp == nil || !hasFinalizer(c, mirrorFinalizer) {
			continue
		}
		if err := w.deleteContainer(ctx, c.Name); err != nil {

deleteContainer / deleteImage / deleteVolume at docker_watcher.go:273-304 swallow all Docker errors (V(1) log only). The finalizer callers (processContainerFinalizers, processImageFinalizers, processVolumeFinalizers) then unconditionally strip the finalizer and delete the K8s mirror.

	// desired != actual is guaranteed by the early return above, so each
	// branch just dispatches to Docker. ContainerStop handles paused /
	// restarting containers natively — filtering by actual == running
	// would silently drop the user's intent in those states.
	switch desired {
	case containersv1alpha1.ContainerStatusRunning:
		log.Info("Starting container", "id", c.Name)
		_, err := w.cli.ContainerStart(ctx, c.Name, dockerclient.ContainerStartOptions{})
		return err
	case containersv1alpha1.ContainerStatusCreated:
		log.Info("Stopping container", "id", c.Name)
		_, err := w.cli.ContainerStop(ctx, c.Name, dockerclient.ContainerStopOptions{})
		return err
	}
	return nil
}

// deleteContainer removes a container from Docker. NotFound errors are
// treated as success (the container is already gone); all other errors
// are returned so the caller keeps the mirror finalizer in place and
// retries on the next reconcile.
func (w *dockerWatcher) deleteContainer(ctx context.Context, id string) error {
	_, err := w.cli.ContainerRemove(ctx, id, dockerclient.ContainerRemoveOptions{Force: true})
	if cerrdefs.IsNotFound(err) {
		return nil
	}
	return err
}

// deleteImage removes an image from Docker. See deleteContainer for the
// error-handling contract.
func (w *dockerWatcher) deleteImage(ctx context.Context, img *containersv1alpha1.Image) error {
	// Use the tag if available, otherwise the raw image ID.
	ref := img.Status.ID
	if img.Status.RepoTag != "" {
		ref = img.Status.RepoTag
	}
	_, err := w.cli.ImageRemove(ctx, ref, dockerclient.ImageRemoveOptions{})
	if cerrdefs.IsNotFound(err) {
		return nil
	}
	return err

This directly violates the contract in docs/design/api_containers.md:56-59: "the finalizer handler deletes the corresponding object in the container engine, then removes the finalizer to allow K8s garbage collection." A transient Docker daemon error, permission error, or conflict (container still running, image in use, volume mounted) leaves the real Docker object in place while the Kubernetes mirror is gone for good. Eventual divergence is guaranteed unless the next fullSync rebuilds the mirror — and a user who tried to delete it via the API will be surprised that it has been silently recreated.

controller removes all mirror resources and sets `ContainerEngineReady` to
`False`.

### Finalizer lifecycle

Each mirror resource carries the `engine.rancherdesktop.io/docker-mirror`
finalizer.  When a user deletes a resource through the K8s API, the finalizer
handler deletes the corresponding object in the container engine, then removes
the finalizer to allow K8s garbage collection.

When the container engine deletes an object (the user ran `docker rm` or
`nerdctl rm`, for example), the engine controller strips the finalizer and
deletes the K8s resource directly, so no reverse engine call occurs.

Fix: make the delete helpers return errors. Ignore only the "not found" / already-gone cases (e.g. errdefs.IsNotFound), treat all other errors as a failure, and leave the finalizer in place so processFinalizers retries on the next reconcile. If the error is persistently terminal (e.g. "container is in use"), surface it on the Container's conditions so the user can see why deletion is blocked.

}

// waitForDesiredState waits for the App to reach the state implied by the
// properties that were set. If running=true was set, it waits for the
// container engine to be ready. If running=false was set, it waits for the
// App's Running condition to go to False — i.e. the VM has actually
// stopped, which is stricter than "container engine disconnected".
// Other property changes do not wait.
//
// TODO: changes that trigger a VM restart without changing "running" (e.g.
// containerEngine.name) are not waited on. This requires a "Reconciled"
// condition with observedGeneration on the App resource so the CLI can
// detect when the full reconcile chain has settled.
func waitForDesiredState(ctx context.Context, config *rest.Config, properties map[string]string, timeout time.Duration) error {
	runningVal, ok := properties["running"]
	if !ok {
		return nil
	}

	ctx, cancel := context.WithTimeout(ctx, timeout)
	defer cancel()

	if runningVal == "true" {
		logrus.Info("Waiting for container engine to be ready")
		return watchCondition(ctx, config, func(obj *unstructured.Unstructured) bool {
			return conditionStatus(obj, "ContainerEngineReady") == metav1.ConditionTrue
		})
	}
	logrus.Info("Waiting for the VM to stop")
	return watchCondition(ctx, config, func(obj *unstructured.Unstructured) bool {
		// Running=False means the VM has stopped. Running absent means
		// the VM was never started (nothing to wait for).
		return conditionStatus(obj, "Running") != metav1.ConditionTrue
	})
}

// watchCondition watches the App singleton until the supplied predicate

The help text at cmd/rdd/set.go:65-67 promises that running=false "waits for the VM to stop", but waitForDesiredState returns as soon as ContainerEngineReady is absent or False. That condition is already False on the engine controller's ConnectFailed path, can be False while the App controller is still propagating running=false to the LimaVM, and can flip to False long before the LimaVM actually finishes shutting down. In those windows the command reports success while the VM is still running.

			Properties are specified as PROPERTY=VALUE pairs. Property names use
			dot notation for nested fields (e.g., containerEngine.name).

			Valid property names and types are derived from the App CRD at
			runtime. If the App resource does not exist, it is created with
			default settings before applying the specified values.

			By default, rdd set waits for the desired state: when running=true
			is set, it waits for the container engine to be ready; when
			running=false, it waits for the VM to stop. Use --timeout=0 to
			skip waiting.

Fix: for running=false, wait on the App's Running condition going to False (or the LimaVM's equivalent stopped condition). Reserve ContainerEngineReady for the running=true / Docker branch. This also sidesteps the I1 containerd hang on the running=true side once the gating there is fixed.

// spec.state transitions, and finalizer processing for mirror resources.
func (r *EngineReconciler) Reconcile(ctx context.Context, _ ctrl.Request) (ctrl.Result, error) {
	log := logf.FromContext(ctx)

	var app appv1alpha1.App
	if err := r.Get(ctx, client.ObjectKey{Name: appName}, &app); err != nil {
		return ctrl.Result{}, client.IgnoreNotFound(err)
	}

	running := meta.IsStatusConditionTrue(app.Status.Conditions, "Running")
	engineIsDocker := app.Spec.ContainerEngine.Name == engineMoby

	r.watcherMu.Lock()
	watcherRunning := r.watcher != nil
	watcherDied := watcherRunning && !r.watcher.alive()
	if watcherDied {
		r.watcher.stop()
		r.watcher = nil
		watcherRunning = false
	}
	r.watcherMu.Unlock()

	// Watcher died unexpectedly (Docker socket gone, engine restarted, etc.).
	// Clean up mirror resources and set the condition to Disconnected.
	if watcherDied {
		log.Info("Docker watcher died, cleaning up mirror resources")
		if err := r.cleanupMirrorResources(ctx); err != nil {
			log.Error(err, "Failed to clean up mirror resources")
			return ctrl.Result{}, err
		}
		if err := r.setEngineCondition(ctx, &app, metav1.ConditionFalse, "Disconnected", "Container engine connection lost"); err != nil {
			return ctrl.Result{}, err
		}
		return ctrl.Result{}, nil
	}

	// The watcher should only run when the App is Running with the moby
	// backend. In every other state (stopped, containerd, both) we stop
	// the watcher and sweep mirror resources. Cleanup runs on every
	// reconcile in that state so transient errors are retried on the
	// next requeue without relying on the in-memory watcher pointer as
	// the retry trigger.
	wantWatcher := running && engineIsDocker
	if !wantWatcher {
		if watcherRunning {
			log.Info("Stopping Docker watcher",
				"running", running, "engine", app.Spec.ContainerEngine.Name)
			r.stopWatcher()
		}
		if err := r.cleanupMirrorResources(ctx); err != nil {
			log.Error(err, "Failed to clean up mirror resources")
			return ctrl.Result{}, err
		}
		switch {
		case running && !engineIsDocker:
			return ctrl.Result{}, r.setEngineCondition(ctx, &app, metav1.ConditionTrue, "NotApplicable",
				"Engine mirroring is only supported with the moby backend")
		default:
			return ctrl.Result{}, r.setEngineCondition(ctx, &app, metav1.ConditionFalse, "Stopped",

Three stacked weaknesses turn any transient problem during cleanup into lasting damage:

1. Inner loop has no Conflict retry Claude. deleteAllOfType calls r.Update(ctx, obj) to strip finalizers with no retry.RetryOnConflict wrapper. A stale cache or concurrent modification during shutdown returns Conflict, which propagates up as a generic error.
2. First error short-circuits the remaining types Claude. cleanupMirrorResources at engine_reconciler.go:190-207 calls deleteAllOfType four times in sequence for Container / Volume / Image / ContainerNamespace. Any error in the first call returns before the remaining three run, leaving a half-cleaned state across kinds.
3. Outer retry is edge-triggered on in-memory watcher state Gemini. stopWatcher() sets r.watcher = nil before cleanupMirrorResources runs at engine_reconciler.go:115-124. When cleanup returns an error the reconcile requeues — but on the retry, watcherRunning is now false, the !running && watcherRunning branch is skipped entirely, and no further cleanup attempt happens until the user stops the VM again.

	// backend. In every other state (stopped, containerd, both) we stop
	// the watcher and sweep mirror resources. Cleanup runs on every
	// reconcile in that state so transient errors are retried on the
	// next requeue without relying on the in-memory watcher pointer as
	// the retry trigger.
	wantWatcher := running && engineIsDocker
	if !wantWatcher {
		if watcherRunning {
			log.Info("Stopping Docker watcher",
				"running", running, "engine", app.Spec.ContainerEngine.Name)
			r.stopWatcher()
		}
		if err := r.cleanupMirrorResources(ctx); err != nil {
			log.Error(err, "Failed to clean up mirror resources")
			return ctrl.Result{}, err
		}
		switch {
		case running && !engineIsDocker:
			return ctrl.Result{}, r.setEngineCondition(ctx, &app, metav1.ConditionTrue, "NotApplicable",
				"Engine mirroring is only supported with the moby backend")

	r.watcherMu.Lock()
	w := r.watcher
	r.watcher = nil
	r.watcherMu.Unlock()

	if w != nil {
		w.stop()
	}
}

// setEngineCondition updates the ContainerEngineReady condition on the App resource.
func (r *EngineReconciler) setEngineCondition(ctx context.Context, app *appv1alpha1.App, status metav1.ConditionStatus, reason, message string) error {
	changed := meta.SetStatusCondition(&app.Status.Conditions, metav1.Condition{
		Type:               conditionContainerEngineReady,
		Status:             status,
		Reason:             reason,
		Message:            message,
		ObservedGeneration: app.Generation,
	})
	if !changed {
		return nil
	}
	return r.Status().Update(ctx, app)
}

// cleanupMirrorResources removes every mirror resource the engine
// controller owns. Errors are collected across all four kinds with
// errors.Join so one stuck resource type does not block the rest — the

A successful next start partially self-heals via the syncAllContainers / syncAllImages / syncAllVolumes stale-item pruning, so this is not permanent orphaning — but between stop and that next start, the resources are visible to rdd ctl get and violate the BATS assertion at bats/tests/32-app-controllers/engine.bats:160-168. Objects mid-deletion (finalizer already stripped, Delete not yet issued) can also land in a broken state.

    assert_output "engine.rancherdesktop.io/docker-mirror"

    # Remove the container so Docker permits the image removal. The
    # next reconcile's finalizer retry succeeds and the K8s Image is
    # finally collected.
    docker rm --force test-inuse
    rdd ctl wait --for=delete --namespace="${NAMESPACE}" \
        "${image_ref}" --timeout=30s
}

# --- Container state transitions via spec ---

@test "patching spec.state=created stops Docker container" {
    docker run -d --name test-state busybox sleep 3600

    run -0 docker inspect test-state --format '{{.Id}}'
    cid=${output}

    rdd ctl wait --for=jsonpath='{.status.status}'=running \

Fix: (1) wrap the inner Update in retry.RetryOnConflict with the default backoff; (2) collect errors across the four types with errors.Join instead of returning on the first; (3) decouple "cleanup needed" from the in-memory watcher pointer — run cleanup whenever !running regardless of watcherRunning, or persist a short-lived "cleanup pending" condition on the App.

// concern on loaded machines, parallelise with errgroup + a small
// worker pool here rather than redesigning the sync.
func (w *dockerWatcher) syncAllContainers(ctx context.Context) error {
	log := logf.FromContext(ctx).WithName("docker-watcher")

	listResult, err := w.cli.ContainerList(ctx, dockerclient.ContainerListOptions{All: true})
	if err != nil {
		return fmt.Errorf("failed to list containers: %w", err)
	}

	// Track which container IDs exist in Docker.
	dockerIDs := make(map[string]bool, len(listResult.Items))

	var errs []error
	for _, dc := range listResult.Items {
		dockerIDs[dc.ID] = true
		if err := w.syncContainer(ctx, dc.ID); err != nil {

syncAllContainers calls ContainerList, then loops and calls ContainerInspect once per container. syncAllImages does the same with ImageList + ImageInspect. A developer laptop with several hundred images (not uncommon on a long-lived machine) hits the Docker socket hundreds of times sequentially before the watcher goroutine enters its event loop, and every hit goes through the Lima port-forwarded Unix socket. Observed startup latency for rdd set running=true will grow linearly with the image count, and the operation is entirely serial — newDockerWatcher holds watcherMu the whole time.

This also intersects with I1: even in the correct Docker case, any developer with a bloated local cache pays this cost on every VM restart.

Fix: seed the initial mirror from List responses where the shape is sufficient (image list has RepoTags, Size, Labels, etc.), and defer Inspect to event-driven updates or lazy per-resource fetches when a user actually queries a field that only Inspect exposes. If an Inspect is still needed for full status, do it concurrently with a small worker pool so startup isn't bounded by serial round-trips.

			log.Error(err, "Failed to clean up mirror resources")
			return ctrl.Result{}, err
		}
		switch {
		case running && !engineIsDocker:
			return ctrl.Result{}, r.setEngineCondition(ctx, &app, metav1.ConditionTrue, "NotApplicable",
				"Engine mirroring is only supported with the moby backend")
		default:
			return ctrl.Result{}, r.setEngineCondition(ctx, &app, metav1.ConditionFalse, "Stopped",
				"Container engine stopped")
		}
	}

	if !watcherRunning {
		log.Info("App is running, starting Docker watcher")
		if err := r.startWatcher(ctx); err != nil {
			log.Error(err, "Failed to start Docker watcher")
			_ = r.setEngineCondition(ctx, &app, metav1.ConditionFalse, "ConnectFailed", err.Error())
			return ctrl.Result{}, err
		}
		if err := r.setEngineCondition(ctx, &app, metav1.ConditionTrue, "Connected", "Container engine synced"); err != nil {
			return ctrl.Result{}, err
		}

Every reconcile with running=true executes four unfiltered list operations:

• List(ContainerList) in reconcileContainerSpecs at line 247
• List(ContainerList) in processContainerFinalizers at line 285
• List(ImageList) in processImageFinalizers at line 305
• List(VolumeList) in processVolumeFinalizers at line 325

		return err
	}

	items, err := meta.ExtractList(list)
	if err != nil {
		return err
	}

	var errs []error
	for _, item := range items {
		obj := item.(client.Object)

}

// reconcileContainerSpecs handles container spec.state changes by calling
// Docker start/stop.
func (r *EngineReconciler) reconcileContainerSpecs(ctx context.Context) error {
	r.watcherMu.Lock()
	w := r.watcher
	r.watcherMu.Unlock()
	if w == nil {
		return nil
	}

	var containers containersv1alpha1.ContainerList
	if err := r.List(ctx, &containers, client.InNamespace(apiNamespace)); err != nil {
		return err
	}

	for i := range containers.Items {
		c := &containers.Items[i]
		if c.DeletionTimestamp != nil {
			continue
		}
		if err := w.reconcileContainerState(ctx, c); err != nil {
			logf.FromContext(ctx).Error(err, "Failed to reconcile container state",
				"container", c.Name)
		}
	}
	return nil
}

// processFinalizers handles resources with a deletion timestamp by deleting
// the corresponding Docker object and removing the finalizer.
func (r *EngineReconciler) processFinalizers(ctx context.Context) error {
	r.watcherMu.Lock()
	w := r.watcher
	r.watcherMu.Unlock()
	if w == nil {
		return nil
	}

	if err := r.processContainerFinalizers(ctx, w); err != nil {
		return err
	}
	if err := r.processImageFinalizers(ctx, w); err != nil {
		return err
	}
	return r.processVolumeFinalizers(ctx, w)
}

// processContainerFinalizers deletes the Docker-side container for every
// K8s Container pending deletion and only strips the mirror finalizer

The reconcile fires on every Container / Image / Volume watch event via Watches(&containersv1alpha1.Container{}, enqueueApp) at engine_reconciler.go:358-360, and each status update the Docker watcher writes back generates one such event. So every container status change on a host with N mirrored containers performs 3·N K8s reads and a nested reconcileContainerState sweep that calls ContainerStart / ContainerStop against Docker for every container with a non-unknown spec. On a busy system with dozens of containers cycling the multiplier is immediate and user-visible.

		}
	}
	return errors.Join(errs...)
}

func (r *EngineReconciler) processImageFinalizers(ctx context.Context, w *dockerWatcher) error {
	var images containersv1alpha1.ImageList
	if err := r.List(ctx, &images, client.InNamespace(apiNamespace)); err != nil {
		return err
	}
	var errs []error
	for i := range images.Items {
		img := &images.Items[i]

The same architecture also made it easy to miss ContainerNamespace coverage entirely (see I2) — the App-level sweep approach has no compile-time guarantee that every mirrored kind has a finalizer handler.

Fix: split the engine controller into (a) an App-level session manager that only owns the Docker watcher lifecycle and (b) per-resource reconcilers (Container, Image, Volume, ContainerNamespace) that handle their own finalizer and spec updates with For() predicates filtering to "deletion timestamp set" or "spec generation changed." That removes the O(N) sweeps and makes coverage explicit per kind.

	}
}

// removeMirrorResource removes the finalizer from a mirror resource and deletes
// it. This is used when Docker has already deleted the object.
//
// The finalizer-strip Update is wrapped in retry.RetryOnConflict to
// survive a stale cache during concurrent reconciles; NotFound means
// the mirror is already gone and is treated as success. Any other
// Update error propagates so the event handler retries on the next
// reconcile instead of stripping the mirror while leaving a stale
// object behind.
func (w *dockerWatcher) removeMirrorResource(ctx context.Context, obj client.Object, name string) error {
	key := client.ObjectKey{Name: name, Namespace: apiNamespace}
	retryErr := retry.RetryOnConflict(retry.DefaultRetry, func() error {
		if err := w.k8s.Get(ctx, key, obj); err != nil {
			if apierrors.IsNotFound(err) {
				return nil
			}
			return err
		}
		if !removeFinalizer(obj, mirrorFinalizer) {

client.IgnoreNotFound returns nil only for IsNotFound errors and passes everything else through — but here it is being used as a generic "swallow if it fails" on Update, which is wrong. Conflict on the finalizer-strip Update (the common optimistic-concurrency outcome when two goroutines race on the same mirror object) is silently treated as success, and the subsequent Delete then runs against a stale snapshot. The only expected benign error on Update here is NotFound (object deleted out from under us); everything else should surface.

This path is reachable for every Docker Destroy event through handleContainerEvent / handleImageEvent / handleVolumeEvent, which makes it a frequent trigger in normal use. Combined with I5 above, a Conflict here leads to a finalizer that was never actually removed plus a Delete that appears to succeed locally — exactly the kind of divergence the finalizer contract is meant to prevent.

Fix: drop IgnoreNotFound on the Update and either (a) propagate the error so the event loop retries, or (b) wrap in retry.RetryOnConflict with a bounded backoff and only propagate on terminal errors. Keep IgnoreNotFound on the final Delete — that one is correct.