Deep Review: 20260408-163819-pr-280

When the hostagent exits while shouldRun is false, the watcher is stopped at line 267 but the host-switch goroutine is not. The host-switch holds a vsock listener, a gvisor-tap-vsock virtual network, and an HTTP server — all permanently leaked. If the VM restarts later, startHostSwitch at line 116 overwrites the hostSwitchStates entry, making the old goroutine unreachable.

Every other teardown path calls stopHostSwitch: handleDeletion (line 40), crash recovery in phaseStopped && shouldRun (line 213), handleRestartNeeded (line 333), stopInstance (line 505), and shutdownAllHostagents (line 543). This branch is the sole gap.

		if inst == nil {
			return ctrl.Result{}, errors.New("instance not found")
		}
		return r.stopInstance(ctx, limaVM, inst)

	default:
		// phase == phaseStopped && !shouldRun
		r.stopWatcher(limaVM.Name)
		if !base.HasConditionWithReason(limaVM.Status.Conditions, ConditionRunning, metav1.ConditionFalse, ReasonStopped) {
			if err := r.updateCondition(ctx, limaVM, ConditionRunning, metav1.ConditionFalse, ReasonStopped, "Lima instance is stopped"); err != nil {
				logger.Error(err, "Failed to update stopped condition")
				return ctrl.Result{}, err
			}
		}
		return ctrl.Result{}, nil
	}
}

Fix:

 	default:
 		// phase == phaseStopped && !shouldRun
 		r.stopWatcher(limaVM.Name)
+		r.stopHostSwitch(limaVM.Name)
 		if !base.HasConditionWithReason(limaVM.Status.Conditions, ConditionRunning, metav1.ConditionFalse, ReasonStopped) {

getVMGUID closes found at line 333 after the per-VM attemptHandshake goroutines finish (wg.Wait()), but attemptHandshake spawns a nested goroutine every second that is not tracked by the WaitGroup. When context cancellation causes attemptHandshake to return, wg.Wait() completes and close(found) fires while nested goroutines are still in-flight. Any goroutine that passed the ctx.Err() check at line 363 before cancellation, then completes readSignature after close(found), will send on a closed channel and panic.

In Go, sending on a closed channel panics even inside a select with a default case. The one-second ticker guarantees multiple in-flight goroutines during normal operation, making this race plausible during startup.

	found := make(chan hvsock.GUID, len(names))
	var wg sync.WaitGroup

	for _, name := range names {
		vmGUID, err := hvsock.GUIDFromString(name)
		if err != nil {
			logger.V(1).Info("Skipping invalid VM GUID", "name", name, "error", err)
			continue
		}
		wg.Add(1)
		go func() {
			defer wg.Done()
			attemptHandshake(ctx, logger, vmGUID, found)
		}()
	}

	// Close found channel when all handshake goroutines finish, so the
	// select below can detect "no match" without blocking forever.
	go func() {
		wg.Wait()
		close(found)
	}()

	select {
	case vmGUID, ok := <-found:
		if !ok {
			return hvsock.GUIDZero, errors.New("no VM matched the signature phrase")
		}
		return vmGUID, nil
	case <-ctx.Done():
		return hvsock.GUIDZero, fmt.Errorf("VM GUID discovery timed out: %w", ctx.Err())
	}
}

// attemptHandshake polls a single VM GUID once per second, trying to match
// the signature phrase.
func attemptHandshake(ctx context.Context, logger logr.Logger, vmGUID hvsock.GUID, found chan<- hvsock.GUID) {
	ticker := time.NewTicker(time.Second)
	defer ticker.Stop()

	for {
		select {
		case <-ctx.Done():
			return
		case <-ticker.C:
			// Dial in a goroutine to avoid blocking on hvsock.Dial timeout.
			go func() {
				conn, err := getVsockConnection(vmGUID, vsockHandshakePort)
				if ctx.Err() != nil {
					if conn != nil {
						conn.Close()
					}
					return
				}
				if err != nil {
					logger.V(1).Info("Handshake dial failed", "vmGUID", vmGUID.String(), "error", err)
					return
				}

				sig, err := readSignature(conn)
				conn.Close()
				if err != nil {
					logger.V(1).Info("Handshake read failed", "vmGUID", vmGUID.String(), "error", err)
					return
				}
				if sig == signaturePhrase {
					logger.V(1).Info("Signature matched", "vmGUID", vmGUID.String())
					select {
					case found <- vmGUID:
					default:
					}
				}
			}()
		}
	}
}

Fix: Remove the nested goroutine and perform the dial synchronously within the ticker loop. This limits concurrency to one outstanding probe per VM, which also addresses the goroutine accumulation issue (see I3).

         case <-ticker.C:
-            go func() {
-                conn, err := getVsockConnection(vmGUID, vsockHandshakePort)
-                if ctx.Err() != nil {
-                    if conn != nil {
-                        conn.Close()
-                    }
-                    return
-                }
+                conn, err := getVsockConnection(vmGUID, vsockHandshakePort)
+                if ctx.Err() != nil {
+                    if conn != nil {
+                        conn.Close()
+                    }
+                    continue
+                }

io.ReadFull at line 395 has no read deadline. If a VM accepts the connection but sends no data (or sends partial data), the goroutine blocks forever. Combined with the nested goroutine spawning in attemptHandshake, multiple goroutines can accumulate over the 5-minute handshake timeout — up to 300 per VM GUID.

// readSignature reads the signature phrase from the peer.
func readSignature(conn net.Conn) (string, error) {
	buf := make([]byte, len(signaturePhrase))
	if _, err := io.ReadFull(conn, buf); err != nil {
		return "", err
	}
	return string(buf), nil
}

Fix: Set a read deadline before the read:

 func readSignature(conn net.Conn) (string, error) {
+    if err := conn.SetReadDeadline(time.Now().Add(5 * time.Second)); err != nil {
+        return "", err
+    }
     buf := make([]byte, len(signaturePhrase))

If runHostSwitch fails after startup (handshake timeout, virtual network failure, listener error), it logs the error and exits silently. Unlike the hostagent watcher, which enqueues reconciles on state changes, the host-switch has no mechanism to notify the controller. The guest's network-setup.service blocks on the handshake during boot, so the VM hangs permanently without triggering a restart.

	})

	logger.Info("Host-switch running", "subnet", subnet.SubnetCIDR, "gateway", subnet.GatewayIP)

	if err := g.Wait(); err != nil {
		logger.Error(err, "Host-switch exited with error")
	} else {
		logger.Info("Host-switch stopped")
	}
}

Fix: When runHostSwitch exits unexpectedly (context not cancelled), enqueue a reconcile so the controller can restart the host-switch or the entire instance.

signalListenerReady calls hvsock.Dial (via getVsockConnection at line 403) without any timeout or context. The handshakeTimeout applied to getVMGUID at line 268 does not cover this call. If the VM becomes unresponsive between discovery and signaling, hvsock.Dial can hang indefinitely. This blocks runHostSwitch from exiting, which blocks stopHostSwitch on <-state.done, which stalls the controller's reconcile loop.

// signalListenerReady tells the guest that the data listener is ready.
func signalListenerReady(vmGUID hvsock.GUID) error {
	conn, err := getVsockConnection(vmGUID, vsockHandshakePort)
	if err != nil {
		return err
	}
	defer conn.Close()
	_, err = conn.Write([]byte(readySignal))
	return err
}

Fix: Pass a context and enforce a timeout:

-func signalListenerReady(vmGUID hvsock.GUID) error {
-    conn, err := getVsockConnection(vmGUID, vsockHandshakePort)
+func signalListenerReady(ctx context.Context, vmGUID hvsock.GUID) error {
+    type result struct {
+        err error
+    }
+    ch := make(chan result, 1)
+    go func() {
+        conn, err := getVsockConnection(vmGUID, vsockHandshakePort)
+        if err != nil {
+            ch <- result{err}
+            return
+        }
+        defer conn.Close()
+        _, err = conn.Write([]byte(readySignal))
+        ch <- result{err}
+    }()
+    select {
+    case r := <-ch:
+        return r.err
+    case <-ctx.Done():
+        return ctx.Err()
+    }
+}