Deep Review: 20260415-103429-pr-309

		func(_ context.Context, _ client.Object) []reconcile.Request {
			return []reconcile.Request{{NamespacedName: types.NamespacedName{Name: appName}}}
		},
	)

	return ctrl.NewControllerManagedBy(mgr).
		For(&appv1alpha1.App{}).
		Named("engine-reconciler").
		WatchesRawSource(source.Channel(r.reconcileChan, enqueueApp)).
		Watches(&containersv1alpha1.Container{}, enqueueApp).
		Watches(&containersv1alpha1.Image{}, enqueueApp).
		Watches(&containersv1alpha1.Volume{}, enqueueApp).
		Complete(r)
}

syncContainerNamespace is only called from fullSync (docker_watcher.go:436-438), and the steady-state reconcile path in Reconcile never touches the ContainerNamespace mirror. The controller graph also does not Watches it, so deleting containernamespace/moby fires no event, triggers no reconcile, and the mirror stays gone until the watcher cycles (Docker daemon restart or VM restart). The API stops reflecting the running engine's namespace in the meantime.

		return fmt.Errorf("failed to ensure namespace: %w", err)
	}

	var errs []error

	if err := w.syncContainerNamespace(ctx); err != nil {
		errs = append(errs, fmt.Errorf("failed to sync container namespace: %w", err))
	}
	if err := w.syncAllContainers(ctx); err != nil {
		errs = append(errs, fmt.Errorf("failed to sync containers: %w", err))
	}
	if err := w.syncAllImages(ctx); err != nil {
		errs = append(errs, fmt.Errorf("failed to sync images: %w", err))

The new BATS suite actually locks this behavior in: engine.bats:375-381 deletes containernamespace/moby and then asserts refute_output on the list, which passes only because the mirror never comes back.

    rdd set running=true
    rdd ctl wait --for=create --namespace="${NAMESPACE}" \
        containernamespace/moby --timeout=10s
}

@test "deleting containernamespace/moby completes without a finalizer hang" {
    # moby ContainerNamespace has no mirror finalizer, so a user delete
    # must return promptly rather than get trapped in Terminating.
    rdd ctl delete containernamespace/moby --namespace="${NAMESPACE}" --timeout=10s
    run -0 rdd ctl get containernamespaces --namespace="${NAMESPACE}" --output=name
    refute_output
}

# --- containerd backend ---

@test "containerd backend reports ContainerEngineReady=NotApplicable and skips mirroring" {
    # Stop first so there is no stale True/Connected from moby to

@test "deleting containernamespace/moby completes without a finalizer hang" {
    rdd ctl delete containernamespace/moby --namespace="${NAMESPACE}" --timeout=10s
    run -0 rdd ctl get containernamespaces --namespace="${NAMESPACE}" --output=name
    refute_output
}

Fix: add ContainerNamespace to the controller's Watches list, and re-apply containernamespace/moby as part of the steady-state reconcile (or gate it behind wantWatcher the same way reconcileContainerSpecs is). Flip the BATS test to rdd ctl wait --for=create containernamespace/moby --timeout=10s so the self-heal is verified instead of masked.

// SPDX-FileCopyrightText: The Rancher Desktop Authors

// Command containers-controller implements the containers API.
package main

import (
	"os"

	// Import app controller packages to trigger init() functions.
	_ "github.com/rancher-sandbox/rancher-desktop-daemon/pkg/controllers/containers"
	"github.com/rancher-sandbox/rancher-desktop-daemon/pkg/external"
)

func main() {
	os.Exit(external.RunControllers("containers"))
}

The engine controller declares itself part of the containers API group (pkg/controllers/app/engine/controller.go:38) with the explicit rationale that it watches Container, Image, and Volume. The embedded path in pkg/service/cmd/service.go:59 blank-imports pkg/controllers/app/engine, but cmd/containers-controller/main.go only imports pkg/controllers/containers, so the external binary never sees the engine controller's init() and base.RegisterController never runs. external.RunControllers("containers") therefore exposes the containers CRDs without any Docker mirroring.

	controllerName = "engine"
	// apiGroup is "containers" because the reconciler watches
	// Container, Image, and Volume from that group and needs
	// their CRDs at startup. Grouping engine with its dependencies
	// keeps --controllers selections from splitting the two apart.
	apiGroup = "containers"
)

type controller struct{}

func newController() base.Controller {

	"github.com/rancher-sandbox/rancher-desktop-daemon/pkg/cli/help"
	// Import controller packages to trigger init() functions for embedded mode.
	_ "github.com/rancher-sandbox/rancher-desktop-daemon/pkg/controllers/app/app"
	_ "github.com/rancher-sandbox/rancher-desktop-daemon/pkg/controllers/app/demo"
	_ "github.com/rancher-sandbox/rancher-desktop-daemon/pkg/controllers/app/engine"
	"github.com/rancher-sandbox/rancher-desktop-daemon/pkg/controllers/base"
	// Import built-in system controllers.
	_ "github.com/rancher-sandbox/rancher-desktop-daemon/pkg/controllers/builtin/namespace"
	// Import controller packages to trigger init() functions for embedded mode.
	_ "github.com/rancher-sandbox/rancher-desktop-daemon/pkg/controllers/containers"

Either the grouping is right and the import is missing, or the grouping should change to app and the import should move to cmd/app-controller/main.go. Picking one and making the import graph match is the fix.

 import (
     "os"

-    _ "github.com/rancher-sandbox/rancher-desktop-daemon/pkg/controllers/containers"
+    _ "github.com/rancher-sandbox/rancher-desktop-daemon/pkg/controllers/app/engine"
+    _ "github.com/rancher-sandbox/rancher-desktop-daemon/pkg/controllers/containers"
     "github.com/rancher-sandbox/rancher-desktop-daemon/pkg/external"
 )

	if err := scm.registerDiscovery(ctx); err != nil {
		log.Error(err, "Failed to register controller manager for discovery")
		// Don't fail startup for discovery registration errors
	}

	// Mark the control plane ready after registerDiscovery so clients
	// waiting on the ready annotation see both CRDs installed and
	// controller registration written, not just CRDs.
	if err := MarkControlPlaneReady(ctx, scm.discovery.client); err != nil {
		return fmt.Errorf("failed to mark control plane as ready: %w", err)
	}

	// Ensure cleanup on shutdown with a timeout to avoid blocking if apiserver is gone
	defer func() {
		cleanupCtx, cleanupCancel := context.WithTimeout(context.Background(), 5*time.Second)
		defer cleanupCancel()

MarkControlPlaneReady at line 201 runs well before mgr.Start(ctx) at line 284. Between those two points, Start still iterates scm.registrations to call RegisterWithManager (line 223-235) and then runs webhook Setup in parallel goroutines (line 241-261). mgr.Start is where controller-runtime actually starts the reconcile loops and the webhook HTTP server. A client that observes the ready annotation at this point and immediately issues a request can:

	// Register all controllers before launching webhook goroutines.
	// RegisterWithManager calls AddToScheme, which writes to the scheme map.
	// Webhook setup reads the scheme via client.Apply. Running both concurrently
	// causes a fatal "concurrent map read and map write" crash.
	for _, registration := range scm.registrations {
		klog.InfoS("Registering controller with shared manager", "controller", registration.GetName())

		// If the controller needs the webhook port, provide it
		if webhookController, ok := registration.(base.WebhookController); ok {
			klog.InfoS("Setting webhook port for controller", "controller", registration.GetName(), "port", scm.webhookPort)
			webhookController.SetWebhookPort(scm.webhookPort)
		}

		if err := registration.RegisterWithManager(mgr); err != nil {
			return fmt.Errorf("failed to register controller %s: %w", registration.GetName(), err)
		}
	}

	// Now that all scheme mutations are done, set up webhooks in parallel.
	var webhookWaitGroup sync.WaitGroup
	webhookErrors := make(chan error, len(scm.registrations))

	for _, registration := range scm.registrations {
		if webhookController, ok := registration.(base.WebhookController); ok {
			managers := webhookController.GetWebhookManagers()
			for i, manager := range managers {
				if manager == nil {
					continue
				}
				name := registration.GetName()
				webhookWaitGroup.Add(1)
				go func() {
					defer webhookWaitGroup.Done()
					klog.Infof("Starting webhook setup for controller %s (webhook %d)", name, i)
					if err := manager.Setup(); err != nil {
						webhookErrors <- fmt.Errorf("controller %s webhook %d: %w", name, i, err)
					}
				}()
			}
		}
	}

	webhookWaitGroup.Wait()
	close(webhookErrors)

	var errs []error
	for err := range webhookErrors {
		errs = append(errs, err)

		"controllers", len(scm.registrations),
		"metricsPort", scm.metricsPort,
		"healthPort", scm.healthPort)

	// Start the manager (this blocks until context is cancelled)
	if err := mgr.Start(ctx); err != nil {
		return fmt.Errorf("failed to start shared controller manager: %w", err)
	}

	return nil
}

• hit an admission webhook whose HTTPS server is not yet listening, returning "connection refused" from the apiserver's webhook call, or
• land on a reconciler that has not registered yet and see a spec change that is never reconciled until the next event fires.

The window is short in practice because rdd set's own ensureServiceRunning + waitForFreshDiscoveryConfigMap gives the manager time to finish startup, but the annotation's documented contract ("every enabled controller has installed its CRDs") is stronger than what the code currently guarantees, and the whole point of the annotation is to let clients skip ad-hoc waits.

Fix: move MarkControlPlaneReady into a manager.RunnableFunc added to mgr so it runs only after Start has initialized controllers and webhooks. A RunnableFunc added before mgr.Start fires once the manager is up.

		return fmt.Errorf("failed to set up ready check: %w", err)
	}

	// Register controller manager in cluster for service discovery before
	// running the controllers.
	if err := scm.registerDiscovery(ctx); err != nil {
		log.Error(err, "Failed to register controller manager for discovery")
		// Don't fail startup for discovery registration errors
	}

	// Mark the control plane ready after registerDiscovery so clients
	// waiting on the ready annotation see both CRDs installed and
	// controller registration written, not just CRDs.
	if err := MarkControlPlaneReady(ctx, scm.discovery.client); err != nil {
		return fmt.Errorf("failed to mark control plane as ready: %w", err)
	}

	// Ensure cleanup on shutdown with a timeout to avoid blocking if apiserver is gone
	defer func() {
		cleanupCtx, cleanupCancel := context.WithTimeout(context.Background(), 5*time.Second)
		defer cleanupCancel()

If registerDiscovery fails at 193, execution falls through and MarkControlPlaneReady stamps the ready annotation on a discovery ConfigMap that has no controller entries. rdd set's isEngineControllerEnabled (cmd/rdd/set.go:523-537) then reads an empty controller list, decides the engine controller is not enabled, and falls back to waiting on Running=True — which returns before the Docker full sync has finished. The user sees rdd set running=true complete while the engine is still catching up.

	})
}

// isEngineControllerEnabled reports whether the engine controller
// is listed in the discovery ConfigMap's EnabledControllers.
func isEngineControllerEnabled(ctx context.Context, config *rest.Config) (bool, error) {
	discovery, err := controllers.NewControllerManagerDiscovery(config)
	if err != nil {
		return false, fmt.Errorf("failed to create discovery client: %w", err)
	}
	enabled, err := discovery.GetEnabledControllers(ctx)
	if err != nil {
		return false, fmt.Errorf("failed to list enabled controllers: %w", err)
	}
	for _, name := range enabled {
		if name == engineControllerName {
			return true, nil
		}
	}
	return false, nil
}

// watchCondition watches the App singleton until the predicate
// returns true or the context expires. watchtools.UntilWithSync
// re-lists transparently on benign watch-channel closures (API

Fix: either make registerDiscovery failure fatal (it already returns an error), or skip MarkControlPlaneReady when it fails. The "don't fail startup" comment pre-dates the new ready-annotation contract and is now at odds with it.

-	if err := scm.registerDiscovery(ctx); err != nil {
-		log.Error(err, "Failed to register controller manager for discovery")
-		// Don't fail startup for discovery registration errors
-	}
-
-	if err := MarkControlPlaneReady(ctx, scm.discovery.client); err != nil {
+	if err := scm.registerDiscovery(ctx); err != nil {
+		return fmt.Errorf("failed to register controller manager for discovery: %w", err)
+	}
+	if err := MarkControlPlaneReady(ctx, scm.discovery.client); err != nil {