Add recovery for orphaned jobs

internal/server/ws/handler.go

@@ -357,10 +357,26 @@ func dispatchAgentMessage(ctx context.Context, c *Conn, hostID string, env api.E
 		}
 
 	case api.MsgCommandResult:
-		// TODO(P2): persist command.result acks for "did the agent
-		// accept the dispatch?" forensics. Currently the job lifecycle
-		// (job.started → job.finished) is sufficient signal.
-		slog.Debug("ws msg not yet handled", "type", env.Type, "host_id", hostID)
+		var p api.CommandResultPayload
+		if err := env.UnmarshalPayload(&p); err != nil {
+			slog.Warn("ws: decode command result", "host_id", hostID, "err", err)
+			break
+		}
+		// A cancel for a job unknown to the current agent process can never
+		// produce job.finished. Terminalise the orphan server-side.
+		if !p.Accepted && p.Error == "job_not_found" && p.JobID != "" {
+			job, err := deps.Store.GetJob(ctx, p.JobID)
+			if err != nil || job.HostID != hostID {
+				slog.Warn("ws: reject orphan result for foreign or missing job", "job_id", p.JobID, "host_id", hostID)
+				break
+			}
+			if job.Status == string(api.JobQueued) || job.Status == string(api.JobRunning) {
+				if err := deps.Store.MarkJobFinished(ctx, p.JobID, string(api.JobCancelled), -1, nil,
+					"agent reported job not found during cancellation", time.Now().UTC()); err != nil {
+					slog.Warn("ws: terminalise orphaned job", "job_id", p.JobID, "err", err)
+				}
+			}
+		}
 
 	case api.MsgTreeListResult:
 		// Reply to a synchronous tree.list RPC. Route to the waiter

internal/server/ws/handler_test.go

@@ -36,6 +36,54 @@ func seedHostWS(t *testing.T, s *store.Store, hostID string) {
 func int64ptrWS(v int64) *int64 { return &v }
 func boolptrWS(v bool) *bool    { return &v }
 
+func TestUnknownCancelResultTerminalisesOrphanedJob(t *testing.T) {
+	t.Parallel()
+	s := openWSTestStore(t)
+	ctx := context.Background()
+	const hostID = "h-orphan-ws"
+	seedHostWS(t, s, hostID)
+	jobID := "j-orphan-ws"
+	started := time.Now().UTC().Add(-time.Hour)
+	if err := s.CreateJob(ctx, store.Job{ID: jobID, HostID: hostID, Kind: "backup", ActorKind: "user", CreatedAt: started}); err != nil {
+		t.Fatalf("create job: %v", err)
+	}
+	if err := s.MarkJobStarted(ctx, jobID, started); err != nil {
+		t.Fatalf("start job: %v", err)
+	}
+	env, _ := api.Marshal(api.MsgCommandResult, jobID, api.CommandResultPayload{
+		JobID: jobID, Accepted: false, Error: "job_not_found",
+	})
+	dispatchAgentMessage(ctx, nil, hostID, env, HandlerDeps{Store: s})
+	job, _ := s.GetJob(ctx, jobID)
+	if job.Status != "cancelled" || job.FinishedAt == nil {
+		t.Fatalf("orphan not terminalised: %+v", job)
+	}
+}
+
+func TestUnknownCancelResultCannotTerminaliseAnotherHostsJob(t *testing.T) {
+	t.Parallel()
+	s := openWSTestStore(t)
+	ctx := context.Background()
+	seedHostWS(t, s, "reporting-host")
+	seedHostWS(t, s, "job-host")
+	const jobID = "j-foreign-ws"
+	started := time.Now().UTC().Add(-time.Hour)
+	if err := s.CreateJob(ctx, store.Job{ID: jobID, HostID: "job-host", Kind: "backup", ActorKind: "user", CreatedAt: started}); err != nil {
+		t.Fatalf("create job: %v", err)
+	}
+	if err := s.MarkJobStarted(ctx, jobID, started); err != nil {
+		t.Fatalf("start job: %v", err)
+	}
+	env, _ := api.Marshal(api.MsgCommandResult, jobID, api.CommandResultPayload{
+		JobID: jobID, Accepted: false, Error: "job_not_found",
+	})
+	dispatchAgentMessage(ctx, nil, "reporting-host", env, HandlerDeps{Store: s})
+	job, _ := s.GetJob(ctx, jobID)
+	if job.Status != "running" || job.FinishedAt != nil {
+		t.Fatalf("foreign job was modified: %+v", job)
+	}
+}
+
 func TestRepoStatsReportPersisted(t *testing.T) {
 	t.Parallel()
 	s := openWSTestStore(t)