Add recovery for orphaned jobs

internal/server/ws/handler_test.go

@@ -36,6 +36,54 @@ func seedHostWS(t *testing.T, s *store.Store, hostID string) {
 func int64ptrWS(v int64) *int64 { return &v }
 func boolptrWS(v bool) *bool    { return &v }
 
+func TestUnknownCancelResultTerminalisesOrphanedJob(t *testing.T) {
+	t.Parallel()
+	s := openWSTestStore(t)
+	ctx := context.Background()
+	const hostID = "h-orphan-ws"
+	seedHostWS(t, s, hostID)
+	jobID := "j-orphan-ws"
+	started := time.Now().UTC().Add(-time.Hour)
+	if err := s.CreateJob(ctx, store.Job{ID: jobID, HostID: hostID, Kind: "backup", ActorKind: "user", CreatedAt: started}); err != nil {
+		t.Fatalf("create job: %v", err)
+	}
+	if err := s.MarkJobStarted(ctx, jobID, started); err != nil {
+		t.Fatalf("start job: %v", err)
+	}
+	env, _ := api.Marshal(api.MsgCommandResult, jobID, api.CommandResultPayload{
+		JobID: jobID, Accepted: false, Error: "job_not_found",
+	})
+	dispatchAgentMessage(ctx, nil, hostID, env, HandlerDeps{Store: s})
+	job, _ := s.GetJob(ctx, jobID)
+	if job.Status != "cancelled" || job.FinishedAt == nil {
+		t.Fatalf("orphan not terminalised: %+v", job)
+	}
+}
+
+func TestUnknownCancelResultCannotTerminaliseAnotherHostsJob(t *testing.T) {
+	t.Parallel()
+	s := openWSTestStore(t)
+	ctx := context.Background()
+	seedHostWS(t, s, "reporting-host")
+	seedHostWS(t, s, "job-host")
+	const jobID = "j-foreign-ws"
+	started := time.Now().UTC().Add(-time.Hour)
+	if err := s.CreateJob(ctx, store.Job{ID: jobID, HostID: "job-host", Kind: "backup", ActorKind: "user", CreatedAt: started}); err != nil {
+		t.Fatalf("create job: %v", err)
+	}
+	if err := s.MarkJobStarted(ctx, jobID, started); err != nil {
+		t.Fatalf("start job: %v", err)
+	}
+	env, _ := api.Marshal(api.MsgCommandResult, jobID, api.CommandResultPayload{
+		JobID: jobID, Accepted: false, Error: "job_not_found",
+	})
+	dispatchAgentMessage(ctx, nil, "reporting-host", env, HandlerDeps{Store: s})
+	job, _ := s.GetJob(ctx, jobID)
+	if job.Status != "running" || job.FinishedAt != nil {
+		t.Fatalf("foreign job was modified: %+v", job)
+	}
+}
+
 func TestRepoStatsReportPersisted(t *testing.T) {
 	t.Parallel()
 	s := openWSTestStore(t)