Add recovery for orphaned jobs

2026-08-22 13:42:42 +01:00
parent 3ccefc254a
commit ec448157d9
9 changed files with 409 additions and 5 deletions
@@ -0,0 +1,93 @@
+package http
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	stdhttp "net/http"
+	"testing"
+	"time"
+
+	"github.com/oklog/ulid/v2"
+
+	"gitea.dcglab.co.uk/steve/restic-manager/internal/alert"
+	"gitea.dcglab.co.uk/steve/restic-manager/internal/store"
+)
+
+func seedRunningJob(t *testing.T, st *store.Store, hostID string) string {
+	t.Helper()
+	id := ulid.Make().String()
+	when := time.Now().UTC().Add(-24 * time.Hour)
+	if err := st.CreateJob(context.Background(), store.Job{
+		ID: id, HostID: hostID, Kind: "backup", ActorKind: "user", CreatedAt: when,
+	}); err != nil {
+		t.Fatalf("create job: %v", err)
+	}
+	if err := st.MarkJobStarted(context.Background(), id, when); err != nil {
+		t.Fatalf("start job: %v", err)
+	}
+	return id
+}
+
+func TestAbandonJobMakesOrphanTerminalAndAudits(t *testing.T) {
+	t.Parallel()
+	_, ts, st := rawTestServer(t)
+	hostID := makeHost(t, st, "abandon-host")
+	jobID := seedRunningJob(t, st, hostID)
+	req, _ := stdhttp.NewRequest(stdhttp.MethodPost, ts.URL+"/api/jobs/"+jobID+"/abandon", nil)
+	req.AddCookie(loginAsAdmin(t, st))
+	res, err := stdhttp.DefaultClient.Do(req)
+	if err != nil {
+		t.Fatalf("abandon: %v", err)
+	}
+	defer res.Body.Close()
+	if res.StatusCode != stdhttp.StatusOK {
+		t.Fatalf("status: got %d", res.StatusCode)
+	}
+	job, _ := st.GetJob(context.Background(), jobID)
+	if job.Status != "cancelled" || job.FinishedAt == nil || job.Error == nil {
+		t.Fatalf("job not terminalised: %+v", job)
+	}
+	var audits int
+	_ = st.DB().QueryRow(`SELECT COUNT(*) FROM audit_log WHERE action = 'job.abandon' AND target_id = ?`, jobID).Scan(&audits)
+	if audits != 1 {
+		t.Fatalf("audit rows: got %d", audits)
+	}
+}
+
+func TestBulkAbandonTargetsOnlyAlertedStuckJobs(t *testing.T) {
+	t.Parallel()
+	_, ts, st := rawTestServer(t)
+	hostID := makeHost(t, st, "bulk-abandon-host")
+	stuckID := seedRunningJob(t, st, hostID)
+	unalertedID := seedRunningJob(t, st, hostID)
+	_, _, err := st.RaiseOrTouch(context.Background(), hostID, alert.KindJobStuck, stuckID,
+		"warning", "stuck", time.Now().UTC())
+	if err != nil {
+		t.Fatalf("raise alert: %v", err)
+	}
+	req, _ := stdhttp.NewRequest(stdhttp.MethodPost, ts.URL+"/api/jobs/abandon-stuck", bytes.NewReader(nil))
+	req.AddCookie(loginAsAdmin(t, st))
+	res, err := stdhttp.DefaultClient.Do(req)
+	if err != nil {
+		t.Fatalf("bulk abandon: %v", err)
+	}
+	defer res.Body.Close()
+	if res.StatusCode != stdhttp.StatusOK {
+		t.Fatalf("status: got %d", res.StatusCode)
+	}
+	var out map[string]int
+	_ = json.NewDecoder(res.Body).Decode(&out)
+	if out["abandoned"] != 1 {
+		t.Fatalf("abandoned: %+v", out)
+	}
+	stuck, _ := st.GetJob(context.Background(), stuckID)
+	unalerted, _ := st.GetJob(context.Background(), unalertedID)
+	if stuck.Status != "cancelled" || unalerted.Status != "running" {
+		t.Fatalf("statuses: stuck=%s unalerted=%s", stuck.Status, unalerted.Status)
+	}
+	open, _ := st.ListAlerts(context.Background(), store.AlertFilter{Status: "open"})
+	if len(open) != 0 {
+		t.Fatalf("stuck alerts remained open: %+v", open)
+	}
+}