Add recovery for orphaned jobs

2026-08-22 13:42:42 +01:00
parent 3ccefc254a
commit ec448157d9
9 changed files with 409 additions and 5 deletions
@@ -0,0 +1,143 @@
+package http
+
+import (
+	"context"
+	"errors"
+	stdhttp "net/http"
+	"time"
+
+	"github.com/go-chi/chi/v5"
+	"github.com/oklog/ulid/v2"
+
+	"gitea.dcglab.co.uk/steve/restic-manager/internal/alert"
+	"gitea.dcglab.co.uk/steve/restic-manager/internal/api"
+	"gitea.dcglab.co.uk/steve/restic-manager/internal/store"
+)
+
+func (s *Server) abandonJob(ctx context.Context, userID *string, jobID, reason string) error {
+	job, err := s.deps.Store.GetJob(ctx, jobID)
+	if err != nil {
+		return err
+	}
+	switch api.JobStatus(job.Status) {
+	case api.JobSucceeded, api.JobFailed, api.JobCancelled:
+		return errJobTerminal
+	}
+	when := time.Now().UTC()
+	if err := s.deps.Store.MarkJobFinished(ctx, jobID, string(api.JobCancelled), -1, nil, reason, when); err != nil {
+		return err
+	}
+	_ = s.deps.Store.AppendAudit(ctx, store.AuditEntry{
+		ID: ulid.Make().String(), UserID: userID, Actor: "user",
+		Action: "job.abandon", TargetKind: ptr("job"), TargetID: &jobID,
+		TS: when,
+	})
+	return nil
+}
+
+var errJobTerminal = errors.New("job already terminal")
+
+func (s *Server) handleAbandonJob(w stdhttp.ResponseWriter, r *stdhttp.Request) {
+	user, ok := s.requireUser(r)
+	if !ok {
+		writeJSONError(w, stdhttp.StatusUnauthorized, "unauthorised", "")
+		return
+	}
+	jobID := chi.URLParam(r, "id")
+	err := s.abandonJob(r.Context(), &user.ID, jobID, "abandoned by operator")
+	switch {
+	case errors.Is(err, store.ErrNotFound):
+		writeJSONError(w, stdhttp.StatusNotFound, "job_not_found", "")
+	case errors.Is(err, errJobTerminal):
+		writeJSONError(w, stdhttp.StatusConflict, "job_terminal", "job is already terminal")
+	case err != nil:
+		writeJSONError(w, stdhttp.StatusInternalServerError, "internal", "")
+	default:
+		writeJSON(w, stdhttp.StatusOK, map[string]string{"job_id": jobID, "status": string(api.JobCancelled)})
+	}
+}
+
+func (s *Server) abandonStuckAlerts(ctx context.Context, userID *string) (int, error) {
+	open, err := s.deps.Store.ListAlerts(ctx, store.AlertFilter{Status: "open"})
+	if err != nil {
+		return 0, err
+	}
+	acked, err := s.deps.Store.ListAlerts(ctx, store.AlertFilter{Status: "acknowledged"})
+	if err != nil {
+		return 0, err
+	}
+	count := 0
+	seen := map[string]bool{}
+	for _, a := range append(open, acked...) {
+		if a.Kind != alert.KindJobStuck || seen[a.DedupKey] {
+			continue
+		}
+		seen[a.DedupKey] = true
+		err := s.abandonJob(ctx, userID, a.DedupKey, "abandoned by operator after stuck-job alert")
+		if err != nil {
+			if errors.Is(err, errJobTerminal) || errors.Is(err, store.ErrNotFound) {
+				s.resolveAlert(ctx, a.ID)
+				continue
+			}
+			return count, err
+		}
+		count++
+		s.resolveAlert(ctx, a.ID)
+	}
+	return count, nil
+}
+
+func (s *Server) resolveAlert(ctx context.Context, alertID string) {
+	when := time.Now().UTC()
+	if s.deps.AlertEngine != nil {
+		_ = s.deps.AlertEngine.Resolve(ctx, alertID, when)
+		return
+	}
+	_ = s.deps.Store.Resolve(ctx, alertID, when)
+}
+
+func (s *Server) handleAbandonStuckJobs(w stdhttp.ResponseWriter, r *stdhttp.Request) {
+	user, ok := s.requireUser(r)
+	if !ok {
+		writeJSONError(w, stdhttp.StatusUnauthorized, "unauthorised", "")
+		return
+	}
+	count, err := s.abandonStuckAlerts(r.Context(), &user.ID)
+	if err != nil {
+		writeJSONError(w, stdhttp.StatusInternalServerError, "internal", "")
+		return
+	}
+	writeJSON(w, stdhttp.StatusOK, map[string]int{"abandoned": count})
+}
+
+func (s *Server) handleUIAbandonAlertJob(w stdhttp.ResponseWriter, r *stdhttp.Request) {
+	user := s.requireUIUser(w, r)
+	if user == nil {
+		return
+	}
+	alertID := chi.URLParam(r, "id")
+	a, err := s.deps.Store.GetAlert(r.Context(), alertID)
+	if err != nil || a == nil || a.Kind != alert.KindJobStuck {
+		stdhttp.Error(w, "stuck-job alert not found", stdhttp.StatusNotFound)
+		return
+	}
+	if err := s.abandonJob(r.Context(), &user.ID, a.DedupKey, "abandoned by operator after stuck-job alert"); err != nil && !errors.Is(err, errJobTerminal) {
+		stdhttp.Error(w, "unable to abandon job", stdhttp.StatusConflict)
+		return
+	}
+	s.resolveAlert(r.Context(), alertID)
+	w.Header().Set("HX-Redirect", "/alerts?"+r.URL.RawQuery)
+	w.WriteHeader(stdhttp.StatusNoContent)
+}
+
+func (s *Server) handleUIAbandonStuckJobs(w stdhttp.ResponseWriter, r *stdhttp.Request) {
+	user := s.requireUIUser(w, r)
+	if user == nil {
+		return
+	}
+	if _, err := s.abandonStuckAlerts(r.Context(), &user.ID); err != nil {
+		stdhttp.Error(w, "unable to abandon stuck jobs", stdhttp.StatusInternalServerError)
+		return
+	}
+	stdhttp.Redirect(w, r, "/alerts", stdhttp.StatusSeeOther)
+}
@@ -0,0 +1,93 @@
+package http
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	stdhttp "net/http"
+	"testing"
+	"time"
+
+	"github.com/oklog/ulid/v2"
+
+	"gitea.dcglab.co.uk/steve/restic-manager/internal/alert"
+	"gitea.dcglab.co.uk/steve/restic-manager/internal/store"
+)
+
+func seedRunningJob(t *testing.T, st *store.Store, hostID string) string {
+	t.Helper()
+	id := ulid.Make().String()
+	when := time.Now().UTC().Add(-24 * time.Hour)
+	if err := st.CreateJob(context.Background(), store.Job{
+		ID: id, HostID: hostID, Kind: "backup", ActorKind: "user", CreatedAt: when,
+	}); err != nil {
+		t.Fatalf("create job: %v", err)
+	}
+	if err := st.MarkJobStarted(context.Background(), id, when); err != nil {
+		t.Fatalf("start job: %v", err)
+	}
+	return id
+}
+
+func TestAbandonJobMakesOrphanTerminalAndAudits(t *testing.T) {
+	t.Parallel()
+	_, ts, st := rawTestServer(t)
+	hostID := makeHost(t, st, "abandon-host")
+	jobID := seedRunningJob(t, st, hostID)
+	req, _ := stdhttp.NewRequest(stdhttp.MethodPost, ts.URL+"/api/jobs/"+jobID+"/abandon", nil)
+	req.AddCookie(loginAsAdmin(t, st))
+	res, err := stdhttp.DefaultClient.Do(req)
+	if err != nil {
+		t.Fatalf("abandon: %v", err)
+	}
+	defer res.Body.Close()
+	if res.StatusCode != stdhttp.StatusOK {
+		t.Fatalf("status: got %d", res.StatusCode)
+	}
+	job, _ := st.GetJob(context.Background(), jobID)
+	if job.Status != "cancelled" || job.FinishedAt == nil || job.Error == nil {
+		t.Fatalf("job not terminalised: %+v", job)
+	}
+	var audits int
+	_ = st.DB().QueryRow(`SELECT COUNT(*) FROM audit_log WHERE action = 'job.abandon' AND target_id = ?`, jobID).Scan(&audits)
+	if audits != 1 {
+		t.Fatalf("audit rows: got %d", audits)
+	}
+}
+
+func TestBulkAbandonTargetsOnlyAlertedStuckJobs(t *testing.T) {
+	t.Parallel()
+	_, ts, st := rawTestServer(t)
+	hostID := makeHost(t, st, "bulk-abandon-host")
+	stuckID := seedRunningJob(t, st, hostID)
+	unalertedID := seedRunningJob(t, st, hostID)
+	_, _, err := st.RaiseOrTouch(context.Background(), hostID, alert.KindJobStuck, stuckID,
+		"warning", "stuck", time.Now().UTC())
+	if err != nil {
+		t.Fatalf("raise alert: %v", err)
+	}
+	req, _ := stdhttp.NewRequest(stdhttp.MethodPost, ts.URL+"/api/jobs/abandon-stuck", bytes.NewReader(nil))
+	req.AddCookie(loginAsAdmin(t, st))
+	res, err := stdhttp.DefaultClient.Do(req)
+	if err != nil {
+		t.Fatalf("bulk abandon: %v", err)
+	}
+	defer res.Body.Close()
+	if res.StatusCode != stdhttp.StatusOK {
+		t.Fatalf("status: got %d", res.StatusCode)
+	}
+	var out map[string]int
+	_ = json.NewDecoder(res.Body).Decode(&out)
+	if out["abandoned"] != 1 {
+		t.Fatalf("abandoned: %+v", out)
+	}
+	stuck, _ := st.GetJob(context.Background(), stuckID)
+	unalerted, _ := st.GetJob(context.Background(), unalertedID)
+	if stuck.Status != "cancelled" || unalerted.Status != "running" {
+		t.Fatalf("statuses: stuck=%s unalerted=%s", stuck.Status, unalerted.Status)
+	}
+	open, _ := st.ListAlerts(context.Background(), store.AlertFilter{Status: "open"})
+	if len(open) != 0 {
+		t.Fatalf("stuck alerts remained open: %+v", open)
+	}
+}
@@ -261,6 +261,8 @@ func (s *Server) routes(r chi.Router) {
 		r.Post("/api/hosts/{id}/repo/check", s.handleRunRepoCheck)
 		r.Post("/api/hosts/{id}/repo/unlock", s.handleRunRepoUnlock)
 		r.Post("/api/jobs/{id}/cancel", s.handleCancelJob)
+		r.Post("/api/jobs/{id}/abandon", s.handleAbandonJob)
+		r.Post("/api/jobs/abandon-stuck", s.handleAbandonStuckJobs)
 		r.Post("/api/hosts/{id}/snapshots/diff", s.handleSnapshotDiff)
 		r.Post("/api/hosts/{id}/snapshots/refresh", s.handleRefreshHostSnapshots)

@@ -298,6 +300,8 @@ func (s *Server) routes(r chi.Router) {
 			r.Post("/hosts/{id}/restore", s.handleUIRestorePost)
 			r.Post("/alerts/{id}/acknowledge", s.handleUIAlertAcknowledge)
 			r.Post("/alerts/{id}/resolve", s.handleUIAlertResolve)
+			r.Post("/alerts/{id}/abandon-job", s.handleUIAbandonAlertJob)
+			r.Post("/alerts/abandon-stuck", s.handleUIAbandonStuckJobs)
 		}
 	})