Deep Review: 20260414-192417-pr-309

// run is the main watcher goroutine that processes Docker events. The
// since parameter is the "Since" filter passed to the Docker events
// endpoint, captured before the initial fullSync so events that
// happened during the sync window are replayed once the stream opens.
func (w *dockerWatcher) run(ctx context.Context, since string) {
	log := logf.FromContext(ctx).WithName("docker-watcher")
	// Wake the engine reconciler after this goroutine exits so it observes
	// the dead watcher via alive() and starts a fresh one. Registered
	// before close(w.done) so that LIFO defer ordering guarantees the
	// enqueue fires AFTER w.done is closed — otherwise the reconciler
	// could wake, see alive()==true on the about-to-exit goroutine, and
	// skip the restart.
	defer w.enqueueReconcile()
	defer close(w.done)
	// Recover from panics in event handling so an unexpected Docker
	// event shape cannot crash the whole app-controller process.
	defer func() {
		if r := recover(); r != nil {
			log.Error(nil, "panic in Docker watcher goroutine",
				"recovered", r, "stack", string(debug.Stack()))
		}
	}()

	eventFilter := dockerclient.Filters{}.
		Add("type", string(events.ContainerEventType)).
		Add("type", string(events.ImageEventType)).
		Add("type", string(events.VolumeEventType))

	result := w.cli.Events(ctx, dockerclient.EventsListOptions{
		Since:   since,
		Filters: eventFilter,
	})

	for {
		select {
		case <-ctx.Done():
			log.Info("Docker watcher stopping")
			return
		case err := <-result.Err:
			if ctx.Err() != nil {
				return
			}
			log.Error(err, "Docker event stream error")
			return
		case msg, ok := <-result.Messages:
			if !ok {
				// The daemon closed the event stream without writing to
				// result.Err. The deferred enqueueReconcile wakes the
				// engine reconciler after w.done is closed, so it
				// observes the dead watcher and starts a fresh one.
				log.Info("Docker event stream closed")
				return
			}
			// A transient handleEvent error (apiserver 503, SSA conflict
			// past its internal retry) is logged and the event is
			// dropped. Container events tend to self-heal because the
			// next state change (start/stop/die) triggers another sync

	// which blocks on <-w.done and cli.Close() — would stall any
	// concurrent reconcile the moment MaxConcurrentReconciles > 1.
	var diedWatcher *dockerWatcher
	r.watcherMu.Lock()
	watcherRunning := r.watcher != nil
	if watcherRunning && !r.watcher.alive() {
		diedWatcher = r.watcher
		r.watcher = nil
		watcherRunning = false
	}
	r.watcherMu.Unlock()

			}
		}
		return ctrl.Result{}, r.setEngineCondition(ctx, &app, terminalStatus, terminalReason, terminalMessage)
	}

	if !watcherRunning {
		log.Info("App is running, starting Docker watcher")
		if err := r.startWatcher(ctx); err != nil {
			log.Error(err, "Failed to start Docker watcher")
			if condErr := r.setEngineCondition(ctx, &app, metav1.ConditionFalse, "ConnectFailed", err.Error()); condErr != nil {
				log.Error(condErr, "Failed to update ContainerEngineReady to ConnectFailed")
			}
			return ctrl.Result{}, err
		}
	}

	// Re-assert the Connected condition on every steady-state reconcile
	// so its observedGeneration tracks the App's current generation. When
	// the spec is patched without cycling the watcher (e.g. `rdd set
	// running=true kubernetes.enabled=true` from an already-running App),

 func (w *dockerWatcher) run(ctx context.Context, since string) {
 	log := logf.FromContext(ctx).WithName("docker-watcher")
 	defer close(w.done)
+	defer w.enqueueReconcile()
 	defer func() {
 		if r := recover(); r != nil {
 			log.Error(nil, "panic in Docker watcher goroutine",
 				"recovered", r, "stack", string(debug.Stack()))
-			w.enqueueReconcile()
 		}
 	}()
 	...
 		case err := <-result.Err:
 			if ctx.Err() != nil {
 				return
 			}
 			log.Error(err, "Docker event stream error")
-			w.enqueueReconcile()
 			return
 		case msg, ok := <-result.Messages:
 			if !ok {
 				log.Info("Docker event stream closed")
-				w.enqueueReconcile()
 				return
 			}

	// every Container/Image/Volume watch event triggers a reconcile via
	// SetupWithManager below. Cost is therefore O(N) per child event.
	// The long-term fix is to split these into per-resource reconcilers
	// with watch predicates ("deletion timestamp set", "spec.state
	// changed"); until then the sweep runs on every event.
	//
	// Run both passes unconditionally and join their errors. An early
	// return on specsErr would stall every mirror's finalizer behind a
	// single stuck Container.spec.state actuation (for example a
	// container whose ContainerStart keeps failing): processFinalizers
	// would never reach the Image/Volume finalizers, and the next
	// reconcile would hit the same broken container first and skip
	// them again.
	var specsErr, finErr error
	if err := r.reconcileContainerSpecs(ctx); err != nil {
		specsErr = fmt.Errorf("failed to reconcile container specs: %w", err)
	}
	if err := r.processFinalizers(ctx); err != nil {

-	if err := r.reconcileContainerSpecs(ctx); err != nil {
-		return ctrl.Result{}, fmt.Errorf("failed to reconcile container specs: %w", err)
-	}
-	if err := r.processFinalizers(ctx); err != nil {
-		return ctrl.Result{}, fmt.Errorf("failed to process finalizers: %w", err)
-	}
-	return ctrl.Result{}, nil
+	specsErr := r.reconcileContainerSpecs(ctx)
+	if specsErr != nil {
+		specsErr = fmt.Errorf("failed to reconcile container specs: %w", specsErr)
+	}
+	finErr := r.processFinalizers(ctx)
+	if finErr != nil {
+		finErr = fmt.Errorf("failed to process finalizers: %w", finErr)
+	}
+	return ctrl.Result{}, errors.Join(specsErr, finErr)

}

// SetupWithManager sets up the controller with the Manager.
func (r *EngineReconciler) SetupWithManager(mgr ctrl.Manager) error {
	r.reconcileChan = make(chan event.GenericEvent, 1)

	// Map any Container/Image/Volume event to a reconcile of the App singleton.
	enqueueApp := handler.EnqueueRequestsFromMapFunc(
		func(_ context.Context, _ client.Object) []reconcile.Request {
			return []reconcile.Request{{NamespacedName: types.NamespacedName{Name: appName}}}
		},
	)

	return ctrl.NewControllerManagedBy(mgr).
		For(&appv1alpha1.App{}).
		Named("engine-reconciler").
		WatchesRawSource(source.Channel(r.reconcileChan, enqueueApp)).
		Watches(&containersv1alpha1.Container{}, enqueueApp).

The engine creates both the rancher-desktop Kubernetes namespace (sync_volumes.go:40-53 — ensureNamespace) and the containernamespace/moby mirror (sync_volumes.go:62-67 — syncContainerNamespace) exactly once, inside fullSync() at watcher startup. SetupWithManager watches App, Container, Image, and Volume — nothing on corev1.Namespace or containersv1alpha1.ContainerNamespace.

	return fmt.Sprintf("vol-%x", sum)
}

// ensureNamespace creates the Kubernetes namespace that holds the
// mirror resources if it doesn't exist.
func (w *dockerWatcher) ensureNamespace(ctx context.Context) error {
	var ns corev1.Namespace
	if err := w.k8s.Get(ctx, client.ObjectKey{Name: apiNamespace}, &ns); err != nil {
		if !apierrors.IsNotFound(err) {
			return err
		}
		ns = corev1.Namespace{
			ObjectMeta: metav1.ObjectMeta{Name: apiNamespace},
		}
		if err := w.k8s.Create(ctx, &ns); err != nil && !apierrors.IsAlreadyExists(err) {
			return fmt.Errorf("failed to create namespace %s: %w", apiNamespace, err)
		}
	}
	return nil
}

// syncContainerNamespace creates the "moby" `ContainerNamespace` mirror.
// Unlike `Container` / `Image` / `Volume` mirrors, this resource has no
// mirror finalizer: Docker has no corresponding engine object to delete
// on the reverse path, and cleanupMirrorResources sweeps it
// unconditionally on VM stop, so a finalizer with no handler would only
// trap user-initiated deletes in Terminating until the next bounce.
func (w *dockerWatcher) syncContainerNamespace(ctx context.Context) error {
	applyConfig := containersv1alpha1apply.ContainerNamespace(containerNamespace, apiNamespace)

	return w.k8s.Apply(ctx, applyConfig,
		client.ForceOwnership, client.FieldOwner(controllerName))
}

// syncAllVolumes lists all Docker volumes and creates/updates `Volume`
// mirrors, then removes stale ones.
func (w *dockerWatcher) syncAllVolumes(ctx context.Context) error {
	log := logf.FromContext(ctx).WithName("docker-watcher")

While the App is running, deleting either of those bootstrap resources produces no watch event that will re-enter fullSync(). The engine.bats test at line 362 (deleting containernamespace/moby completes without a finalizer hang) only works because the next test does rdd set running=true which cycles the watcher; once the file runs, the intervening state is "moby namespace is gone and nobody notices". The namespace case is worse: once namespace/rancher-desktop is deleted, every subsequent mirror apply inside handleEvent fails with a NotFound, and docker_watcher.go:203-206 explicitly logs-and-drops those errors until a watcher restart happens. The controller is wedged in a broken steady state with no self-heal path.

			// on the same ID; image pull and volume create events fire
			// only once, so a dropped apply leaves the mirror missing
			// until the next full resync (watcher restart or engine
			// reconnect). The bounded-window fix is a periodic
			// fullSync tick, deferred until the rest of the mirroring
			// work lands.
			if err := w.handleEvent(ctx, msg); err != nil {
				log.Error(err, "Failed to handle Docker event",
					"type", msg.Type, "action", msg.Action, "actor", msg.Actor.ID)
			}
		}
	}
}

// errors.Join and returned so controller-runtime requeues with backoff
// — matching the pattern in processContainerFinalizers. Without the
// return, a failed start/stop would get exactly one attempt (the watch
// event from the original spec.state patch) and then sit forever.
func (r *EngineReconciler) reconcileContainerSpecs(ctx context.Context) error {
	r.watcherMu.Lock()
	w := r.watcher
	r.watcherMu.Unlock()
	if w == nil {
		return nil
	}

Fix needs both pieces: watch the bootstrap resources so a delete enqueues a reconcile, AND move ensureNamespace() / syncContainerNamespace() out of startup-only and into the steady-state reconcile path so the wake actually rebuilds them.

 return ctrl.NewControllerManagedBy(mgr).
     For(&appv1alpha1.App{}).
     Named("engine-reconciler").
     WatchesRawSource(source.Channel(r.reconcileChan, enqueueApp)).
+    Watches(&corev1.Namespace{}, enqueueApp /* filter name == apiNamespace */).
+    Watches(&containersv1alpha1.ContainerNamespace{}, enqueueApp).
     Watches(&containersv1alpha1.Container{}, enqueueApp).
     Watches(&containersv1alpha1.Image{}, enqueueApp).
     Watches(&containersv1alpha1.Volume{}, enqueueApp).
     Complete(r)

Combined with calling ensureNamespace + syncContainerNamespace near the top of the steady-state reconcile (before any sub-resource apply) so the reconciler that the watch enqueues can actually recover.

//
// TODO: changes that trigger a VM restart without changing "running" (e.g.
// containerEngine.name alone) are still not waited on. A dedicated
// "Reconciled" condition on App would let the CLI detect when the full
// reconcile chain has settled for any property change.
func waitForDesiredState(ctx context.Context, config *rest.Config, properties map[string]string, timeout time.Duration, minGen int64) error {
	runningVal, ok := properties["running"]
	if !ok {
		return nil
	}

	ctx, cancel := context.WithTimeout(ctx, timeout)
	defer cancel()

	if runningVal == "true" {
		logrus.Info("Waiting for container engine to be ready")
		return watchCondition(ctx, config, func(obj *unstructured.Unstructured) bool {
			status, gen, present := conditionInfo(obj, "ContainerEngineReady")
			return present && gen >= minGen && status == metav1.ConditionTrue
		})
	}
	logrus.Info("Waiting for the VM to stop")
	return watchCondition(ctx, config, func(obj *unstructured.Unstructured) bool {
		// Running absent means the VM was never started (nothing to
		// wait for). Otherwise wait until the condition is refreshed
		// with our generation and reports a non-True status.
		status, gen, present := conditionInfo(obj, "Running")
		if !present {
			return true
		}
		return gen >= minGen && status != metav1.ConditionTrue
	})
}

// watchCondition watches the App singleton until the supplied predicate
// returns true or the context expires. It uses watchtools.UntilWithSync
// so benign watch-channel closures (API server timeouts, proxy
// disconnects, resource-version compaction) re-list transparently

The new wait path treats ContainerEngineReady as the completion signal for every running=true command. But engine_reconciler.go:184-186 re-asserts ContainerEngineReady=True/Connected on every steady-state reconcile solely to advance observedGeneration, with a comment explicitly stating that this "is the only place that stamps the new generation into ContainerEngineReady, and rdd set's wait predicate depends on it".

	// running=true kubernetes.enabled=true` from an already-running App),
	// this is the only place that stamps the new generation into
	// ContainerEngineReady, and rdd set's wait predicate depends on it.
	// setEngineCondition no-ops when nothing changed, so the extra call
	// is free on stable reconciles.
	if err := r.setEngineCondition(ctx, &app, metav1.ConditionTrue, "Connected", "Container engine synced"); err != nil {
		return ctrl.Result{}, err
	}

	// reconcileContainerSpecs + processFinalizers each issue one or more
	// List() calls across every mirrored object on every reconcile, and
	// every Container/Image/Volume watch event triggers a reconcile via
	// SetupWithManager below. Cost is therefore O(N) per child event.

That self-stamping is correct for this code path's purpose, but it decouples the signal from actual convergence. When the user runs rdd set running=true kubernetes.enabled=true or rdd set running=true containerEngine.name=containerd against an already-running App:

1. The patch lands; App generation becomes N+1.
2. AppReconciler.Reconcile fires, sees the template ConfigMap differs, patches the template ConfigMap at app_controller.go:274-290, and returns with Requeue: true. It does not wait for Lima to pick up the change.
3. EngineReconciler.Reconcile fires in parallel on the same App change. running=true, engineIsDocker=true, watcher is still alive → it re-stamps ContainerEngineReady=Connected/True with ObservedGeneration=N+1.
4. rdd set's watchCondition observes status=True && observedGeneration >= minGen → returns success.
5. Lima still has to pick up the template-ConfigMap change, mark restartNeeded, stop the VM, start it with the new template. The engine controller will eventually sweep mirrors and set ContainerEngineReady=Stopped, but the user has already seen the command succeed and moved on — docker ps shows a different engine's state than the one the user asked for.

		} else {
			desired, err := applySpecToTemplate(r.LimaTemplateData, app.Spec)
			if err != nil {
				return ctrl.Result{}, fmt.Errorf("failed to apply spec to template: %w", err)
			}
			if templateCM.Data[limav1alpha1.TemplateConfigMapKey] != desired {
				patch := client.MergeFrom(templateCM.DeepCopy())
				if templateCM.Data == nil {
					templateCM.Data = make(map[string]string)
				}
				templateCM.Data[limav1alpha1.TemplateConfigMapKey] = desired
				if err := r.Patch(ctx, templateCM, patch); err != nil {
					return ctrl.Result{}, fmt.Errorf("failed to update template ConfigMap: %w", err)
				}
				log.Info("Updated template ConfigMap",
					"containerEngine", app.Spec.ContainerEngine.Name,
					"kubernetesEnabled", app.Spec.Kubernetes.Enabled,
					"kubernetesVersion", app.Spec.Kubernetes.Version)
				// ConfigMaps are not watched, so requeue to let the
				// reconciler evaluate remaining spec fields (e.g. running).
				return ctrl.Result{Requeue: true}, nil
			}
		}
	}

	if !limaVM.Spec.Running && app.Spec.Running {
		limaVM.Spec.Running = true

The existing TODO at set.go:463-466 acknowledges the gap for "running not changed" cases; this finding extends the same logic to running=true combined with a template-affecting field.

// so a stale ContainerEngineReady=True from a previous backend (or a
// stale Running=False from a prior stop) cannot satisfy the wait before
// the reconciler has observed the new spec. EngineReconciler and
// AppReconciler both stamp ObservedGeneration when they update conditions.
//
// TODO: changes that trigger a VM restart without changing "running" (e.g.
// containerEngine.name alone) are still not waited on. A dedicated
// "Reconciled" condition on App would let the CLI detect when the full
// reconcile chain has settled for any property change.
func waitForDesiredState(ctx context.Context, config *rest.Config, properties map[string]string, timeout time.Duration, minGen int64) error {
	runningVal, ok := properties["running"]
	if !ok {
		return nil
	}

Fix: condition the wait target on what actually changed. Three feasible shapes:

• For any patch that touches a template input (containerEngine.*, kubernetes.*), wait on LimaVM.status.observedTemplateResourceVersion (already maintained for this purpose, per pkg/apis/lima/v1alpha1 and docs/design/api_lima.md) rather than ContainerEngineReady.
• Or introduce the "Reconciled" condition on App mentioned in the TODO comment and key the wait on that.
• Or, as a short-term fix that does not need a new condition, stop re-stamping ContainerEngineReady purely to advance observedGeneration in the steady-state path — but that breaks the current wait semantics for the cold-start-from-stopped path.

The first option is cheapest: rdd set knows which fields it patched, so it can pick the right wait predicate per patch.

	}

	// Track which `Volume` mirror names we create.
	activeNames := make(map[string]bool, len(volumeList.Items))

	// Per-item apply failures are logged and skipped rather than failing
	// the whole startup, matching the containers/images pattern: a
	// single permanently-broken Apply (transient API 503, SSA conflict
	// past its internal retry) must not pin ContainerEngineReady at
	// ConnectFailed. Structural errors below (k8s list, stale-mirror
	// cleanup) are still fatal.
	var errs []error
	for _, v := range volumeList.Items {
		mirrorName := volumeMirrorName(v.Name)
		activeNames[mirrorName] = true
		if err := w.applyVolume(ctx, v); err != nil {
			log.Error(err, "Skipping volume during full sync", "name", v.Name)
		}

syncAllContainers (sync_containers.go:57-63) and syncAllImages (sync_images.go:86-93) both log a per-item apply failure and continue, with an explicit comment calling out that transient failures must not "prevent every other healthy container from being mirrored, nor pin the ContainerEngineReady condition at ConnectFailed". syncAllVolumes does not follow that contract — it accumulates per-item errors into errs and returns them.

	// failing the whole startup: a single permanently-broken Inspect
	// (engine-side corruption, transient race) must not prevent every
	// other healthy container from being mirrored, nor pin the
	// ContainerEngineReady condition at ConnectFailed. Structural
	// errors (k8s list, stale-mirror cleanup) are still fatal.
	var errs []error
	for _, dc := range listResult.Items {
		dockerIDs[dc.ID] = true
		if err := w.syncContainer(ctx, dc.ID); err != nil {
			log.Error(err, "Skipping container during full sync", "id", dc.ID)
		}
	}

	// Remove stale Container mirrors.
	var containerMirrors containersv1alpha1.ContainerList
	if err := w.k8s.List(ctx, &containerMirrors, client.InNamespace(apiNamespace)); err != nil {
		return fmt.Errorf("failed to list Containers: %w", err)

	var errs []error
	for _, summary := range listResult.Items {
		for _, n := range imageMirrorNames(summary.ID, summary.RepoTags) {
			activeNames[n] = true
		}
		if err := w.syncImageFromSummary(ctx, summary); err != nil {
			log.Error(err, "Skipping image during full sync", "id", summary.ID)
		}
	}

	// Remove stale Image mirrors.
	var imageMirrors containersv1alpha1.ImageList
	if err := w.k8s.List(ctx, &imageMirrors, client.InNamespace(apiNamespace)); err != nil {
		return fmt.Errorf("failed to list Images: %w", err)
	}
	for i := range imageMirrors.Items {
		img := &imageMirrors.Items[i]
		if !activeNames[img.Name] {

The downstream chain amplifies this: fullSync returns errors.Join(...), newDockerWatcher forwards that error to startWatcher, which sets ContainerEngineReady=False/ConnectFailed and returns. A single transient API-server 503 on one Volume apply fails the entire watcher startup until the condition clears.

Fix: match the containers/images pattern exactly — log the error and continue.

 var errs []error
 for _, v := range volumeList.Items {
     mirrorName := volumeMirrorName(v.Name)
     activeNames[mirrorName] = true
     if err := w.applyVolume(ctx, v); err != nil {
-        errs = append(errs, err)
+        log.Error(err, "Skipping volume during full sync", "name", v.Name)
     }
 }

The stale-mirror sweep below still uses errs — leave that branch untouched. If the mirror is missing, the next event will pick it up; if the volume is permanently broken, nothing in either pattern catches it anyway.

# containers, images, and volumes into Container, Image, and Volume
# resources, and that deletions and spec.state changes are forwarded
# to Docker.

NAMESPACE="rancher-desktop"
DOCKER_HOST="unix://$(rdd svc paths docker_socket)"
export DOCKER_HOST

local_setup_file() {
    # The Docker socket access pattern used by these tests is not yet wired
    # up for Windows/WSL2.
    skip_on_windows

The $(rdd svc paths short_dir) substitution runs when BATS loads the file, before local_setup_file has a chance to run skip_on_windows or establish any test context. If rdd is missing, unbuildable, or returns an error, the failure surfaces as a file-load-time error rather than a clean per-test failure, and the value silently becomes unix:///docker.sock (no error, no indication which test file broke). The repo's BATS style rules (per ~/.claude/memory/bats-style.md) explicitly prefer run -0 for commands whose output is captured, so failures are both visible and attributable.

Fix: move the assignment into local_setup_file, using run -0 so a failure is reported against the file rather than silently eaten.

 NAMESPACE="rancher-desktop"
-DOCKER_HOST="unix://$(rdd svc paths short_dir)/docker.sock"
-export DOCKER_HOST

 local_setup_file() {
     skip_on_windows
     rdd svc delete
     rdd set running=true
+    run -0 rdd svc paths short_dir
+    DOCKER_HOST="unix://${output}/docker.sock"
+    export DOCKER_HOST
 }

Reconciliation note: Gemini and Codex both raised this; I kept it as Important because the failure mode (silent substitution during file load) is the exact class the BATS style rule exists to prevent, even though Codex filed it as a Suggestion.

// removeImagesByID finds and removes all `Image` mirrors for a given Docker image ID.
//
// Matching is by status.id, which leaves a narrow orphan window: if
// applyImage's metadata Apply succeeded but the status SubResource Apply
// failed transiently, the mirror exists with an empty status.id and this
// function will not match it. The orphan is swept by syncAllImages'
// stale-mirror pass on the next watcher restart, which is the existing
// self-heal path; the window is too narrow to justify a fallback lookup
// by computed dangling-ID name here.
func (w *dockerWatcher) removeImagesByID(ctx context.Context, id string) error {
	var images containersv1alpha1.ImageList
	if err := w.k8s.List(ctx, &images, client.InNamespace(apiNamespace)); err != nil {
		return err
	}
	var errs []error
	for i := range images.Items {
		if images.Items[i].Status.ID == id {
			if err := w.removeMirrorResource(ctx, &images.Items[i], images.Items[i].Name); err != nil {
				errs = append(errs, err)
			}
		}
	}
	return errors.Join(errs...)
}

applyImage (sync_images.go:191-212) issues two sequential calls — the metadata Apply and the status SubResource Apply. If the first succeeds and the second fails transiently (for example a 503 or a SSA conflict past its internal retry), the Image mirror exists with an empty status.id. A subsequent Docker ActionDelete event for the underlying image ID routes through removeImagesByID, which iterates by status.id == id — the empty-status mirror never matches and is left as an orphan. The next full sync at watcher restart prunes it via syncAllImages' stale-mirror sweep, but during normal operation the orphan persists.

// applyImage creates or updates a single `Image` mirror and its status.
func (w *dockerWatcher) applyImage(
	ctx context.Context,
	image *containersv1alpha1apply.ImageApplyConfiguration,
	status *containersv1alpha1apply.ImageStatusApplyConfiguration,
) error {
	err := w.k8s.Apply(ctx, image,
		client.ForceOwnership, client.FieldOwner(controllerName))
	if err != nil {
		return fmt.Errorf("failed to apply image %s: %w", *image.GetName(), err)
	}

	err = w.k8s.SubResource("status").Apply(ctx,
		containersv1alpha1apply.Image(*image.GetName(), *image.GetNamespace()).
			WithStatus(status),
		client.ForceOwnership, client.FieldOwner(controllerName))
	if err != nil {
		return fmt.Errorf("failed to apply image status %s: %w", *image.GetName(), err)
	}

	return nil
}

// reconcileImageByID re-inspects a Docker image and reconciles every
// `Image` mirror whose status.id matches. Tags still present are
// re-applied; Image mirrors for tags that are no longer present are
// deleted. If the image has been fully removed from Docker, all mirrors
// with that status.id are deleted instead.
//
// This is the path for events that carry an image ID but not the tag
// name — notably Docker's untag events, where the event payload only

The symmetric applyImageFromInspect path for dangling images is also affected: a dangling image created and deleted faster than the status apply completes produces an orphan mirror.

Fix: as a fallback when no mirror matches by status.id, resolve the expected mirror name(s) with imageMirrorNames(id, nil) — the same helper used to create the mirror name for a dangling image — and try to remove by computed name too. The dangling-ID name is deterministic from the Docker ID alone, so this cleanly handles the empty-status case without needing to know the tags.

matched := make(map[string]bool)
for i := range images.Items {
    if images.Items[i].Status.ID == id {
        matched[images.Items[i].Name] = true
        if err := w.removeMirrorResource(ctx, &images.Items[i], images.Items[i].Name); err != nil {
            errs = append(errs, err)
        }
    }
}
for _, name := range imageMirrorNames(id, nil) {
    if matched[name] {
        continue
    }
    if err := w.removeMirrorResource(ctx, &containersv1alpha1.Image{}, name); err != nil && !apierrors.IsNotFound(err) {
        errs = append(errs, err)
    }
}

// created branch always dispatches Stop from any non-stopped state. The
// running branch handles paused explicitly because Docker rejects
// ContainerStart on a paused container. isStopped() treats
// created/exited/dead as terminal stopped states: the user's intent
// ("do not run") is already satisfied and redispatching Stop would be
// wasted work on every unrelated reconcile event.
func (w *dockerWatcher) reconcileContainerState(ctx context.Context, c *containersv1alpha1.Container) error {
	desired := c.Spec.State
	if desired == containersv1alpha1.ContainerStatusUnknown {
		return nil
	}

	actual := c.Status.Status
	log := logf.FromContext(ctx).WithName("docker-watcher")

	switch desired {
	case containersv1alpha1.ContainerStatusRunning:
		if actual == containersv1alpha1.ContainerStatusRunning {
			return nil
		}
		if actual == containersv1alpha1.ContainerStatusPaused {
			log.Info("Unpausing container", "id", c.Name)
			_, err := w.cli.ContainerUnpause(ctx, c.Name, dockerclient.ContainerUnpauseOptions{})
			return err
		}
		log.Info("Starting container", "id", c.Name)
		_, err := w.cli.ContainerStart(ctx, c.Name, dockerclient.ContainerStartOptions{})
		return err
	case containersv1alpha1.ContainerStatusCreated:
		if isStopped(actual) {
			return nil
		}
		log.Info("Stopping container", "id", c.Name)
		_, err := w.cli.ContainerStop(ctx, c.Name, dockerclient.ContainerStopOptions{})
		return err
	}
	return nil
}

// isStopped reports whether a container status represents a terminal
// non-running state. Every state listed here satisfies a desired state
// of "created": the container is not running, cannot transition without
// a user action (start/rm), and dispatching ContainerStop again would
// be a no-op.
func isStopped(s containersv1alpha1.ContainerStatusValue) bool {
	switch s {
	case containersv1alpha1.ContainerStatusCreated,
		containersv1alpha1.ContainerStatusExited,
		containersv1alpha1.ContainerStatusDead:
		return true
	}
	return false
}

// deleteContainer removes a container from Docker. NotFound errors are
// treated as success (the container is already gone); all other errors
// are returned so the caller keeps the mirror finalizer in place and

	// +optional
	Labels map[string]string `json:"labels"`
	// Status of the container.
	//
	// +required
	// +kubebuilder:default:=unknown
	Status ContainerStatusValue `json:"status"`
	// Pid is the process identifier for the main process in the container.
	//
	// +optional
	Pid int32 `json:"pid"`

// from Docker). The webhook restricts desired to {unknown, created,
// running}; actual can be any value in the CRD enum.
//
//	desired \ actual  | running | paused   | created | exited | dead | pausing/restarting | removing | unknown
//	------------------+---------+----------+---------+--------+------+--------------------+----------+--------
//	running           | nil     | unpause  | start   | start  | start| start              | start    | start
//	created           | stop    | stop     | nil     | nil    | nil  | stop               | stop     | stop
//
// ContainerStop handles paused/restarting containers natively, so the
// created branch always dispatches Stop from any non-stopped state. The
// running branch handles paused explicitly because Docker rejects

NAMESPACE="rancher-desktop"
DOCKER_HOST="unix://$(rdd svc paths docker_socket)"
export DOCKER_HOST

local_setup_file() {
    # The Docker socket access pattern used by these tests is not yet wired
    # up for Windows/WSL2.
    skip_on_windows
    # Deliberately skip setup_rdd_control_plane here: `rdd set` internally
    # calls ensureServiceRunning, which is exactly the CLI path we want to
    # exercise — the engine controller is part of the default controller
    # set, so no explicit --controllers selection is needed.
    rdd svc delete
    rdd set running=true
}

}

// getAppKubeClient returns a controller-runtime client with the App scheme
// registered, and the raw REST config for creating other clients.
func getAppKubeClient(ctx context.Context) (client.Client, *rest.Config, error) {
	if err := ensureServiceRunning(ctx); err != nil {
		return nil, nil, err
	}
	config, err := service.GetKubeRestConfig()
	if err != nil {
		return nil, nil, fmt.Errorf("failed to get kubeconfig: %w", err)

// errors.Join and returned so controller-runtime requeues with backoff
// — matching the pattern in processContainerFinalizers. Without the
// return, a failed start/stop would get exactly one attempt (the watch
// event from the original spec.state patch) and then sit forever.
func (r *EngineReconciler) reconcileContainerSpecs(ctx context.Context) error {
	r.watcherMu.Lock()
	w := r.watcher
	r.watcherMu.Unlock()
	if w == nil {
		return nil
	}

// conditionInfo returns the status, observedGeneration, and presence of
// the named condition on an App. present is false when the condition is
// not on the object; callers must check it before trusting the other
// return values.
func conditionInfo(obj *unstructured.Unstructured, condType string) (status metav1.ConditionStatus, observedGen int64, present bool) {
	conditions, found, err := unstructured.NestedSlice(obj.Object, "status", "conditions")
	if err != nil || !found {
		return "", 0, false
	}
	for _, c := range conditions {
		cond, ok := c.(map[string]any)
		if !ok {
			continue
		}
		if cond["type"] != condType {
			continue
		}
		s, _ := cond["status"].(string)
		gen, _, _ := unstructured.NestedInt64(cond, "observedGeneration")
		return metav1.ConditionStatus(s), gen, true
	}
	return "", 0, false
}

// getAppKubeClient returns a controller-runtime client with the App scheme
// registered, and the raw REST config for creating other clients.
func getAppKubeClient(ctx context.Context) (client.Client, *rest.Config, error) {
	if err := ensureServiceRunning(ctx); err != nil {

	if runningVal == "true" {
		logrus.Info("Waiting for container engine to be ready")
		return watchCondition(ctx, config, func(obj *unstructured.Unstructured) bool {
			status, gen, present := conditionInfo(obj, "ContainerEngineReady")
			return present && gen >= minGen && status == metav1.ConditionTrue
		})
	}
	logrus.Info("Waiting for the VM to stop")
	return watchCondition(ctx, config, func(obj *unstructured.Unstructured) bool {
		// Running absent means the VM was never started (nothing to