diff --git a/cmd/agent/cancel_test.go b/cmd/agent/cancel_test.go
new file mode 100644
index 0000000..63e7eaf
--- /dev/null
+++ b/cmd/agent/cancel_test.go
@@ -0,0 +1,68 @@
+package main
+
+import (
+ "context"
+ "testing"
+
+ "gitea.dcglab.co.uk/steve/restic-manager/internal/api"
+)
+
+type recordingSender struct {
+ envelopes []api.Envelope
+}
+
+func (s *recordingSender) Send(env api.Envelope) error {
+ s.envelopes = append(s.envelopes, env)
+ return nil
+}
+
+func TestCancelCommandReportsUnknownJob(t *testing.T) {
+ d := &dispatcher{}
+ tx := &recordingSender{}
+ env, err := api.Marshal(api.MsgCommandCancel, "request-1", api.CommandCancelPayload{JobID: "missing-job"})
+ if err != nil {
+ t.Fatalf("marshal command: %v", err)
+ }
+
+ if err := d.handle(context.Background(), env, tx); err != nil {
+ t.Fatalf("handle cancel: %v", err)
+ }
+ if len(tx.envelopes) != 1 {
+ t.Fatalf("sent %d envelopes, want 1", len(tx.envelopes))
+ }
+ if tx.envelopes[0].Type != api.MsgCommandResult || tx.envelopes[0].ID != env.ID {
+ t.Fatalf("unexpected result envelope: %+v", tx.envelopes[0])
+ }
+ var result api.CommandResultPayload
+ if err := tx.envelopes[0].UnmarshalPayload(&result); err != nil {
+ t.Fatalf("unmarshal result: %v", err)
+ }
+ if result.JobID != "missing-job" || result.Accepted || result.Error != "job_not_found" {
+ t.Fatalf("unexpected result: %+v", result)
+ }
+}
+
+func TestCancelCommandCancelsTrackedJob(t *testing.T) {
+ d := &dispatcher{}
+ cancelled := false
+ d.trackJob("running-job", func() { cancelled = true })
+ tx := &recordingSender{}
+ env, err := api.Marshal(api.MsgCommandCancel, "request-2", api.CommandCancelPayload{JobID: "running-job"})
+ if err != nil {
+ t.Fatalf("marshal command: %v", err)
+ }
+
+ if err := d.handle(context.Background(), env, tx); err != nil {
+ t.Fatalf("handle cancel: %v", err)
+ }
+ if !cancelled {
+ t.Fatal("tracked job was not cancelled")
+ }
+ var result api.CommandResultPayload
+ if err := tx.envelopes[0].UnmarshalPayload(&result); err != nil {
+ t.Fatalf("unmarshal result: %v", err)
+ }
+ if !result.Accepted || result.Error != "" {
+ t.Fatalf("unexpected result: %+v", result)
+ }
+}
diff --git a/cmd/agent/main.go b/cmd/agent/main.go
index 96aedff..f500746 100644
--- a/cmd/agent/main.go
+++ b/cmd/agent/main.go
@@ -276,7 +276,8 @@ func (d *dispatcher) handle(ctx context.Context, env api.Envelope, tx wsclient.S
if err := env.UnmarshalPayload(&p); err != nil {
return fmt.Errorf("command.cancel: %w", err)
}
- if d.cancelJob(p.JobID) {
+ known := d.cancelJob(p.JobID)
+ if known {
slog.Info("ws agent: command.cancel applied", "job_id", p.JobID)
} else {
// Job already finished or was never seen on this agent.
@@ -284,6 +285,17 @@ func (d *dispatcher) handle(ctx context.Context, env api.Envelope, tx wsclient.S
// natural completion. Server-side state is authoritative.
slog.Info("ws agent: command.cancel for unknown job (already finished?)", "job_id", p.JobID)
}
+ result := api.CommandResultPayload{JobID: p.JobID, Accepted: known}
+ if !known {
+ result.Error = "job_not_found"
+ }
+ ack, err := api.Marshal(api.MsgCommandResult, env.ID, result)
+ if err != nil {
+ return fmt.Errorf("command.cancel result: %w", err)
+ }
+ if err := tx.Send(ack); err != nil {
+ return fmt.Errorf("command.cancel result send: %w", err)
+ }
case api.MsgTreeList:
// Synchronous RPC for the restore wizard's tree browser. The
diff --git a/docs/book/src/operations/backups-and-restores.md b/docs/book/src/operations/backups-and-restores.md
index 31c1a29..f97ad91 100644
--- a/docs/book/src/operations/backups-and-restores.md
+++ b/docs/book/src/operations/backups-and-restores.md
@@ -6,11 +6,12 @@ Three ways to trigger one:
1. **Scheduled** — the agent's local cron fires at the time set
on the schedule.
-2. **Run-now** — operator clicks **Run now** on the host detail
- right rail. Posts to `/hosts/{id}/run-backup` (defaults to all
- source groups) or to a per-group form for finer control.
-3. **API** — `POST /api/hosts/{id}/jobs` with the appropriate
- payload. Same audit + dispatch path.
+2. **Run-now** — operator clicks **Run now** for a specific source
+ group. This uses `POST /hosts/{id}/source-groups/{gid}/run`.
+3. **API** — use `POST /api/hosts/{id}/source-groups/{gid}/run` for a
+ configured source group. The lower-level `POST /api/hosts/{id}/jobs`
+ backup form requires explicit paths in `args`; an empty backup is
+ rejected instead of dispatching a job that restic cannot run.
In every case the server creates a `jobs` row, broadcasts a
`command.run` to the host, and lands the operator on the live
diff --git a/internal/server/http/job_abandon.go b/internal/server/http/job_abandon.go
new file mode 100644
index 0000000..36d5bf5
--- /dev/null
+++ b/internal/server/http/job_abandon.go
@@ -0,0 +1,143 @@
+package http
+
+import (
+ "context"
+ "errors"
+ stdhttp "net/http"
+ "time"
+
+ "github.com/go-chi/chi/v5"
+ "github.com/oklog/ulid/v2"
+
+ "gitea.dcglab.co.uk/steve/restic-manager/internal/alert"
+ "gitea.dcglab.co.uk/steve/restic-manager/internal/api"
+ "gitea.dcglab.co.uk/steve/restic-manager/internal/store"
+)
+
+func (s *Server) abandonJob(ctx context.Context, userID *string, jobID, reason string) error {
+ job, err := s.deps.Store.GetJob(ctx, jobID)
+ if err != nil {
+ return err
+ }
+ switch api.JobStatus(job.Status) {
+ case api.JobSucceeded, api.JobFailed, api.JobCancelled:
+ return errJobTerminal
+ }
+ when := time.Now().UTC()
+ if err := s.deps.Store.MarkJobFinished(ctx, jobID, string(api.JobCancelled), -1, nil, reason, when); err != nil {
+ return err
+ }
+ _ = s.deps.Store.AppendAudit(ctx, store.AuditEntry{
+ ID: ulid.Make().String(), UserID: userID, Actor: "user",
+ Action: "job.abandon", TargetKind: ptr("job"), TargetID: &jobID,
+ TS: when,
+ })
+ return nil
+}
+
+var errJobTerminal = errors.New("job already terminal")
+
+func (s *Server) handleAbandonJob(w stdhttp.ResponseWriter, r *stdhttp.Request) {
+ user, ok := s.requireUser(r)
+ if !ok {
+ writeJSONError(w, stdhttp.StatusUnauthorized, "unauthorised", "")
+ return
+ }
+ jobID := chi.URLParam(r, "id")
+ err := s.abandonJob(r.Context(), &user.ID, jobID, "abandoned by operator")
+ switch {
+ case errors.Is(err, store.ErrNotFound):
+ writeJSONError(w, stdhttp.StatusNotFound, "job_not_found", "")
+ case errors.Is(err, errJobTerminal):
+ writeJSONError(w, stdhttp.StatusConflict, "job_terminal", "job is already terminal")
+ case err != nil:
+ writeJSONError(w, stdhttp.StatusInternalServerError, "internal", "")
+ default:
+ writeJSON(w, stdhttp.StatusOK, map[string]string{"job_id": jobID, "status": string(api.JobCancelled)})
+ }
+}
+
+func (s *Server) abandonStuckAlerts(ctx context.Context, userID *string) (int, error) {
+ open, err := s.deps.Store.ListAlerts(ctx, store.AlertFilter{Status: "open"})
+ if err != nil {
+ return 0, err
+ }
+ acked, err := s.deps.Store.ListAlerts(ctx, store.AlertFilter{Status: "acknowledged"})
+ if err != nil {
+ return 0, err
+ }
+ count := 0
+ seen := map[string]bool{}
+ for _, a := range append(open, acked...) {
+ if a.Kind != alert.KindJobStuck || seen[a.DedupKey] {
+ continue
+ }
+ seen[a.DedupKey] = true
+ err := s.abandonJob(ctx, userID, a.DedupKey, "abandoned by operator after stuck-job alert")
+ if err != nil {
+ if errors.Is(err, errJobTerminal) || errors.Is(err, store.ErrNotFound) {
+ s.resolveAlert(ctx, a.ID)
+ continue
+ }
+ return count, err
+ }
+ count++
+ s.resolveAlert(ctx, a.ID)
+ }
+ return count, nil
+}
+
+func (s *Server) resolveAlert(ctx context.Context, alertID string) {
+ when := time.Now().UTC()
+ if s.deps.AlertEngine != nil {
+ _ = s.deps.AlertEngine.Resolve(ctx, alertID, when)
+ return
+ }
+ _ = s.deps.Store.Resolve(ctx, alertID, when)
+}
+
+func (s *Server) handleAbandonStuckJobs(w stdhttp.ResponseWriter, r *stdhttp.Request) {
+ user, ok := s.requireUser(r)
+ if !ok {
+ writeJSONError(w, stdhttp.StatusUnauthorized, "unauthorised", "")
+ return
+ }
+ count, err := s.abandonStuckAlerts(r.Context(), &user.ID)
+ if err != nil {
+ writeJSONError(w, stdhttp.StatusInternalServerError, "internal", "")
+ return
+ }
+ writeJSON(w, stdhttp.StatusOK, map[string]int{"abandoned": count})
+}
+
+func (s *Server) handleUIAbandonAlertJob(w stdhttp.ResponseWriter, r *stdhttp.Request) {
+ user := s.requireUIUser(w, r)
+ if user == nil {
+ return
+ }
+ alertID := chi.URLParam(r, "id")
+ a, err := s.deps.Store.GetAlert(r.Context(), alertID)
+ if err != nil || a == nil || a.Kind != alert.KindJobStuck {
+ stdhttp.Error(w, "stuck-job alert not found", stdhttp.StatusNotFound)
+ return
+ }
+ if err := s.abandonJob(r.Context(), &user.ID, a.DedupKey, "abandoned by operator after stuck-job alert"); err != nil && !errors.Is(err, errJobTerminal) {
+ stdhttp.Error(w, "unable to abandon job", stdhttp.StatusConflict)
+ return
+ }
+ s.resolveAlert(r.Context(), alertID)
+ w.Header().Set("HX-Redirect", "/alerts?"+r.URL.RawQuery)
+ w.WriteHeader(stdhttp.StatusNoContent)
+}
+
+func (s *Server) handleUIAbandonStuckJobs(w stdhttp.ResponseWriter, r *stdhttp.Request) {
+ user := s.requireUIUser(w, r)
+ if user == nil {
+ return
+ }
+ if _, err := s.abandonStuckAlerts(r.Context(), &user.ID); err != nil {
+ stdhttp.Error(w, "unable to abandon stuck jobs", stdhttp.StatusInternalServerError)
+ return
+ }
+ stdhttp.Redirect(w, r, "/alerts", stdhttp.StatusSeeOther)
+}
diff --git a/internal/server/http/job_abandon_test.go b/internal/server/http/job_abandon_test.go
new file mode 100644
index 0000000..1b156b7
--- /dev/null
+++ b/internal/server/http/job_abandon_test.go
@@ -0,0 +1,93 @@
+package http
+
+import (
+ "bytes"
+ "context"
+ "encoding/json"
+ stdhttp "net/http"
+ "testing"
+ "time"
+
+ "github.com/oklog/ulid/v2"
+
+ "gitea.dcglab.co.uk/steve/restic-manager/internal/alert"
+ "gitea.dcglab.co.uk/steve/restic-manager/internal/store"
+)
+
+func seedRunningJob(t *testing.T, st *store.Store, hostID string) string {
+ t.Helper()
+ id := ulid.Make().String()
+ when := time.Now().UTC().Add(-24 * time.Hour)
+ if err := st.CreateJob(context.Background(), store.Job{
+ ID: id, HostID: hostID, Kind: "backup", ActorKind: "user", CreatedAt: when,
+ }); err != nil {
+ t.Fatalf("create job: %v", err)
+ }
+ if err := st.MarkJobStarted(context.Background(), id, when); err != nil {
+ t.Fatalf("start job: %v", err)
+ }
+ return id
+}
+
+func TestAbandonJobMakesOrphanTerminalAndAudits(t *testing.T) {
+ t.Parallel()
+ _, ts, st := rawTestServer(t)
+ hostID := makeHost(t, st, "abandon-host")
+ jobID := seedRunningJob(t, st, hostID)
+ req, _ := stdhttp.NewRequest(stdhttp.MethodPost, ts.URL+"/api/jobs/"+jobID+"/abandon", nil)
+ req.AddCookie(loginAsAdmin(t, st))
+ res, err := stdhttp.DefaultClient.Do(req)
+ if err != nil {
+ t.Fatalf("abandon: %v", err)
+ }
+ defer res.Body.Close()
+ if res.StatusCode != stdhttp.StatusOK {
+ t.Fatalf("status: got %d", res.StatusCode)
+ }
+ job, _ := st.GetJob(context.Background(), jobID)
+ if job.Status != "cancelled" || job.FinishedAt == nil || job.Error == nil {
+ t.Fatalf("job not terminalised: %+v", job)
+ }
+ var audits int
+ _ = st.DB().QueryRow(`SELECT COUNT(*) FROM audit_log WHERE action = 'job.abandon' AND target_id = ?`, jobID).Scan(&audits)
+ if audits != 1 {
+ t.Fatalf("audit rows: got %d", audits)
+ }
+}
+
+func TestBulkAbandonTargetsOnlyAlertedStuckJobs(t *testing.T) {
+ t.Parallel()
+ _, ts, st := rawTestServer(t)
+ hostID := makeHost(t, st, "bulk-abandon-host")
+ stuckID := seedRunningJob(t, st, hostID)
+ unalertedID := seedRunningJob(t, st, hostID)
+ _, _, err := st.RaiseOrTouch(context.Background(), hostID, alert.KindJobStuck, stuckID,
+ "warning", "stuck", time.Now().UTC())
+ if err != nil {
+ t.Fatalf("raise alert: %v", err)
+ }
+ req, _ := stdhttp.NewRequest(stdhttp.MethodPost, ts.URL+"/api/jobs/abandon-stuck", bytes.NewReader(nil))
+ req.AddCookie(loginAsAdmin(t, st))
+ res, err := stdhttp.DefaultClient.Do(req)
+ if err != nil {
+ t.Fatalf("bulk abandon: %v", err)
+ }
+ defer res.Body.Close()
+ if res.StatusCode != stdhttp.StatusOK {
+ t.Fatalf("status: got %d", res.StatusCode)
+ }
+ var out map[string]int
+ _ = json.NewDecoder(res.Body).Decode(&out)
+ if out["abandoned"] != 1 {
+ t.Fatalf("abandoned: %+v", out)
+ }
+ stuck, _ := st.GetJob(context.Background(), stuckID)
+ unalerted, _ := st.GetJob(context.Background(), unalertedID)
+ if stuck.Status != "cancelled" || unalerted.Status != "running" {
+ t.Fatalf("statuses: stuck=%s unalerted=%s", stuck.Status, unalerted.Status)
+ }
+ open, _ := st.ListAlerts(context.Background(), store.AlertFilter{Status: "open"})
+ if len(open) != 0 {
+ t.Fatalf("stuck alerts remained open: %+v", open)
+ }
+}
diff --git a/internal/server/http/jobs.go b/internal/server/http/jobs.go
index a71c679..c3e97b9 100644
--- a/internal/server/http/jobs.go
+++ b/internal/server/http/jobs.go
@@ -69,6 +69,10 @@ func (s *Server) dispatchJob(ctx context.Context, user *store.User,
Kind: kind,
Args: args,
}
+ if kind == api.JobBackup && len(args) == 0 {
+ return res, stdhttp.StatusUnprocessableEntity, "backup_paths_required",
+ "backup requires paths in args; configured backups must use POST /api/hosts/{id}/source-groups/{gid}/run"
+ }
if kind == api.JobForget {
if !validForgetArgs(args) {
return res, stdhttp.StatusBadRequest, "invalid_args",
diff --git a/internal/server/http/jobs_test.go b/internal/server/http/jobs_test.go
index b526591..e53572f 100644
--- a/internal/server/http/jobs_test.go
+++ b/internal/server/http/jobs_test.go
@@ -14,6 +14,34 @@ import (
"gitea.dcglab.co.uk/steve/restic-manager/internal/store"
)
+func TestRunNowBackupRejectsEmptyPaths(t *testing.T) {
+ t.Parallel()
+ _, ts, st := rawTestServer(t)
+ cookie := loginAsAdmin(t, st)
+ hostID := makeHost(t, st, "empty-backup-host")
+ req, _ := stdhttp.NewRequest(stdhttp.MethodPost, ts.URL+"/api/hosts/"+hostID+"/jobs",
+ bytes.NewReader([]byte(`{"kind":"backup"}`)))
+ req.Header.Set("Content-Type", "application/json")
+ req.AddCookie(cookie)
+ res, err := stdhttp.DefaultClient.Do(req)
+ if err != nil {
+ t.Fatalf("post run-now: %v", err)
+ }
+ defer res.Body.Close()
+ if res.StatusCode != stdhttp.StatusUnprocessableEntity {
+ t.Fatalf("status: got %d, want 422", res.StatusCode)
+ }
+ body := readJSONError(t, res.Body)
+ if body.Code != "backup_paths_required" {
+ t.Fatalf("code: got %q", body.Code)
+ }
+ var jobs int
+ _ = st.DB().QueryRow(`SELECT COUNT(*) FROM jobs WHERE host_id = ?`, hostID).Scan(&jobs)
+ if jobs != 0 {
+ t.Fatalf("invalid backup created %d jobs", jobs)
+ }
+}
+
func TestRunNowForgetShipsRetentionGroupsAndDryRun(t *testing.T) {
t.Parallel()
srv, ts, st := rawTestServer(t)
diff --git a/internal/server/http/p2r01_test.go b/internal/server/http/p2r01_test.go
index 07451db..51a42ff 100644
--- a/internal/server/http/p2r01_test.go
+++ b/internal/server/http/p2r01_test.go
@@ -153,6 +153,8 @@ func TestSourceGroupsCRUD(t *testing.T) {
},
"retry_max": 3,
"retry_backoff_seconds": 60,
+ "pre_hook": "prepare-db",
+ "post_hook": "resume-db",
}, cookie)
if status != 201 {
t.Fatalf("create status: %d, body: %+v", status, body)
@@ -161,6 +163,16 @@ func TestSourceGroupsCRUD(t *testing.T) {
if gid == "" {
t.Fatalf("create: no id returned: %+v", body)
}
+ if body["has_pre_hook"] != true || body["has_post_hook"] != true {
+ t.Fatalf("create hook indicators: %+v", body)
+ }
+ stored, err := st.GetSourceGroup(context.Background(), hostID, gid)
+ if err != nil {
+ t.Fatalf("get stored group: %v", err)
+ }
+ if stored.PreHook == "prepare-db" || stored.PostHook == "resume-db" || stored.PreHook == "" || stored.PostHook == "" {
+ t.Fatalf("hooks not encrypted at rest: pre=%q post=%q", stored.PreHook, stored.PostHook)
+ }
// Duplicate name → 409.
status, _ = doJSON(t, url, "POST", "/api/hosts/"+hostID+"/source-groups",
@@ -185,6 +197,20 @@ func TestSourceGroupsCRUD(t *testing.T) {
if got := body["name"]; got != "system" {
t.Errorf("rename: got %v want system", got)
}
+ stored, _ = st.GetSourceGroup(context.Background(), hostID, gid)
+ if stored.PreHook == "" || stored.PostHook == "" {
+ t.Fatal("PUT without hook fields cleared existing hooks")
+ }
+
+ // Explicit empty hook clears only that hook; plaintext is never returned.
+ status, body = doJSON(t, url, "PUT", "/api/hosts/"+hostID+"/source-groups/"+gid,
+ map[string]any{"name": "system", "includes": []string{"/etc"}, "pre_hook": ""}, cookie)
+ if status != 200 || body["has_pre_hook"] != false || body["has_post_hook"] != true {
+ t.Fatalf("clear hook status=%d body=%+v", status, body)
+ }
+ if _, exposed := body["post_hook"]; exposed {
+ t.Fatal("source-group response exposed hook plaintext field")
+ }
// Delete.
status, _ = doJSON(t, url, "DELETE", "/api/hosts/"+hostID+"/source-groups/"+gid, nil, cookie)
diff --git a/internal/server/http/server.go b/internal/server/http/server.go
index af63427..24641e8 100644
--- a/internal/server/http/server.go
+++ b/internal/server/http/server.go
@@ -261,6 +261,8 @@ func (s *Server) routes(r chi.Router) {
r.Post("/api/hosts/{id}/repo/check", s.handleRunRepoCheck)
r.Post("/api/hosts/{id}/repo/unlock", s.handleRunRepoUnlock)
r.Post("/api/jobs/{id}/cancel", s.handleCancelJob)
+ r.Post("/api/jobs/{id}/abandon", s.handleAbandonJob)
+ r.Post("/api/jobs/abandon-stuck", s.handleAbandonStuckJobs)
r.Post("/api/hosts/{id}/snapshots/diff", s.handleSnapshotDiff)
r.Post("/api/hosts/{id}/snapshots/refresh", s.handleRefreshHostSnapshots)
@@ -298,6 +300,8 @@ func (s *Server) routes(r chi.Router) {
r.Post("/hosts/{id}/restore", s.handleUIRestorePost)
r.Post("/alerts/{id}/acknowledge", s.handleUIAlertAcknowledge)
r.Post("/alerts/{id}/resolve", s.handleUIAlertResolve)
+ r.Post("/alerts/{id}/abandon-job", s.handleUIAbandonAlertJob)
+ r.Post("/alerts/abandon-stuck", s.handleUIAbandonStuckJobs)
}
})
diff --git a/internal/server/http/source_groups.go b/internal/server/http/source_groups.go
index 88c2688..00cfb72 100644
--- a/internal/server/http/source_groups.go
+++ b/internal/server/http/source_groups.go
@@ -29,6 +29,8 @@ type sourceGroupView struct {
RetryMax int `json:"retry_max"`
RetryBackoffSeconds int `json:"retry_backoff_seconds"`
ConflictDimension string `json:"conflict_dimension,omitempty"`
+ HasPreHook bool `json:"has_pre_hook"`
+ HasPostHook bool `json:"has_post_hook"`
CreatedAt time.Time `json:"created_at"`
UpdatedAt time.Time `json:"updated_at"`
}
@@ -49,6 +51,8 @@ func toSourceGroupView(g store.SourceGroup) sourceGroupView {
RetryMax: g.RetryMax,
RetryBackoffSeconds: g.RetryBackoffSeconds,
ConflictDimension: g.ConflictDimension,
+ HasPreHook: g.PreHook != "",
+ HasPostHook: g.PostHook != "",
CreatedAt: g.CreatedAt,
UpdatedAt: g.UpdatedAt,
}
@@ -62,6 +66,29 @@ type sourceGroupWriteRequest struct {
RetentionPolicy store.RetentionPolicy `json:"retention_policy"`
RetryMax int `json:"retry_max"`
RetryBackoffSeconds int `json:"retry_backoff_seconds"`
+ // Pointer fields distinguish omission (preserve on PUT) from an explicit
+ // empty string (clear). Hook plaintext is accepted on write but never
+ // returned by the JSON API.
+ PreHook *string `json:"pre_hook,omitempty"`
+ PostHook *string `json:"post_hook,omitempty"`
+}
+
+func (s *Server) applySourceGroupHooks(hostID string, req sourceGroupWriteRequest, g *store.SourceGroup) error {
+ if req.PreHook != nil {
+ enc, err := s.EncryptHookForGroup(hostID, "pre", *req.PreHook)
+ if err != nil {
+ return err
+ }
+ g.PreHook = enc
+ }
+ if req.PostHook != nil {
+ enc, err := s.EncryptHookForGroup(hostID, "post", *req.PostHook)
+ if err != nil {
+ return err
+ }
+ g.PostHook = enc
+ }
+ return nil
}
func (s *Server) handleListSourceGroups(w stdhttp.ResponseWriter, r *stdhttp.Request) {
@@ -142,6 +169,10 @@ func (s *Server) handleCreateSourceGroup(w stdhttp.ResponseWriter, r *stdhttp.Re
RetryMax: req.RetryMax,
RetryBackoffSeconds: req.RetryBackoffSeconds,
}
+ if err := s.applySourceGroupHooks(hostID, req, &g); err != nil {
+ writeJSONError(w, stdhttp.StatusInternalServerError, "hook_encryption_failed", "")
+ return
+ }
if err := s.deps.Store.CreateSourceGroup(r.Context(), &g); err != nil {
writeJSONError(w, stdhttp.StatusInternalServerError, "internal", err.Error())
return
@@ -157,7 +188,8 @@ func (s *Server) handleUpdateSourceGroup(w stdhttp.ResponseWriter, r *stdhttp.Re
}
hostID := chi.URLParam(r, "id")
groupID := chi.URLParam(r, "gid")
- if _, err := s.deps.Store.GetSourceGroup(r.Context(), hostID, groupID); err != nil {
+ existingGroup, err := s.deps.Store.GetSourceGroup(r.Context(), hostID, groupID)
+ if err != nil {
if errors.Is(err, store.ErrNotFound) {
writeJSONError(w, stdhttp.StatusNotFound, "group_not_found", "")
return
@@ -188,6 +220,12 @@ func (s *Server) handleUpdateSourceGroup(w stdhttp.ResponseWriter, r *stdhttp.Re
RetentionPolicy: req.RetentionPolicy,
RetryMax: req.RetryMax,
RetryBackoffSeconds: req.RetryBackoffSeconds,
+ PreHook: existingGroup.PreHook,
+ PostHook: existingGroup.PostHook,
+ }
+ if err := s.applySourceGroupHooks(hostID, req, &g); err != nil {
+ writeJSONError(w, stdhttp.StatusInternalServerError, "hook_encryption_failed", "")
+ return
}
if err := s.deps.Store.UpdateSourceGroup(r.Context(), &g); err != nil {
writeJSONError(w, stdhttp.StatusInternalServerError, "internal", err.Error())
diff --git a/internal/server/ws/handler.go b/internal/server/ws/handler.go
index 88b9ec9..d2a4a87 100644
--- a/internal/server/ws/handler.go
+++ b/internal/server/ws/handler.go
@@ -357,10 +357,26 @@ func dispatchAgentMessage(ctx context.Context, c *Conn, hostID string, env api.E
}
case api.MsgCommandResult:
- // TODO(P2): persist command.result acks for "did the agent
- // accept the dispatch?" forensics. Currently the job lifecycle
- // (job.started → job.finished) is sufficient signal.
- slog.Debug("ws msg not yet handled", "type", env.Type, "host_id", hostID)
+ var p api.CommandResultPayload
+ if err := env.UnmarshalPayload(&p); err != nil {
+ slog.Warn("ws: decode command result", "host_id", hostID, "err", err)
+ break
+ }
+ // A cancel for a job unknown to the current agent process can never
+ // produce job.finished. Terminalise the orphan server-side.
+ if !p.Accepted && p.Error == "job_not_found" && p.JobID != "" {
+ job, err := deps.Store.GetJob(ctx, p.JobID)
+ if err != nil || job.HostID != hostID {
+ slog.Warn("ws: reject orphan result for foreign or missing job", "job_id", p.JobID, "host_id", hostID)
+ break
+ }
+ if job.Status == string(api.JobQueued) || job.Status == string(api.JobRunning) {
+ if err := deps.Store.MarkJobFinished(ctx, p.JobID, string(api.JobCancelled), -1, nil,
+ "agent reported job not found during cancellation", time.Now().UTC()); err != nil {
+ slog.Warn("ws: terminalise orphaned job", "job_id", p.JobID, "err", err)
+ }
+ }
+ }
case api.MsgTreeListResult:
// Reply to a synchronous tree.list RPC. Route to the waiter
diff --git a/internal/server/ws/handler_test.go b/internal/server/ws/handler_test.go
index 1bd2088..7f68da8 100644
--- a/internal/server/ws/handler_test.go
+++ b/internal/server/ws/handler_test.go
@@ -36,6 +36,54 @@ func seedHostWS(t *testing.T, s *store.Store, hostID string) {
func int64ptrWS(v int64) *int64 { return &v }
func boolptrWS(v bool) *bool { return &v }
+func TestUnknownCancelResultTerminalisesOrphanedJob(t *testing.T) {
+ t.Parallel()
+ s := openWSTestStore(t)
+ ctx := context.Background()
+ const hostID = "h-orphan-ws"
+ seedHostWS(t, s, hostID)
+ jobID := "j-orphan-ws"
+ started := time.Now().UTC().Add(-time.Hour)
+ if err := s.CreateJob(ctx, store.Job{ID: jobID, HostID: hostID, Kind: "backup", ActorKind: "user", CreatedAt: started}); err != nil {
+ t.Fatalf("create job: %v", err)
+ }
+ if err := s.MarkJobStarted(ctx, jobID, started); err != nil {
+ t.Fatalf("start job: %v", err)
+ }
+ env, _ := api.Marshal(api.MsgCommandResult, jobID, api.CommandResultPayload{
+ JobID: jobID, Accepted: false, Error: "job_not_found",
+ })
+ dispatchAgentMessage(ctx, nil, hostID, env, HandlerDeps{Store: s})
+ job, _ := s.GetJob(ctx, jobID)
+ if job.Status != "cancelled" || job.FinishedAt == nil {
+ t.Fatalf("orphan not terminalised: %+v", job)
+ }
+}
+
+func TestUnknownCancelResultCannotTerminaliseAnotherHostsJob(t *testing.T) {
+ t.Parallel()
+ s := openWSTestStore(t)
+ ctx := context.Background()
+ seedHostWS(t, s, "reporting-host")
+ seedHostWS(t, s, "job-host")
+ const jobID = "j-foreign-ws"
+ started := time.Now().UTC().Add(-time.Hour)
+ if err := s.CreateJob(ctx, store.Job{ID: jobID, HostID: "job-host", Kind: "backup", ActorKind: "user", CreatedAt: started}); err != nil {
+ t.Fatalf("create job: %v", err)
+ }
+ if err := s.MarkJobStarted(ctx, jobID, started); err != nil {
+ t.Fatalf("start job: %v", err)
+ }
+ env, _ := api.Marshal(api.MsgCommandResult, jobID, api.CommandResultPayload{
+ JobID: jobID, Accepted: false, Error: "job_not_found",
+ })
+ dispatchAgentMessage(ctx, nil, "reporting-host", env, HandlerDeps{Store: s})
+ job, _ := s.GetJob(ctx, jobID)
+ if job.Status != "running" || job.FinishedAt != nil {
+ t.Fatalf("foreign job was modified: %+v", job)
+ }
+}
+
func TestRepoStatsReportPersisted(t *testing.T) {
t.Parallel()
s := openWSTestStore(t)
diff --git a/web/templates/pages/alerts.html b/web/templates/pages/alerts.html
index 10a8dbc..aa37a5c 100644
--- a/web/templates/pages/alerts.html
+++ b/web/templates/pages/alerts.html
@@ -24,6 +24,10 @@
diff --git a/web/templates/partials/alert_row.html b/web/templates/partials/alert_row.html
index 0ac05db..dc12272 100644
--- a/web/templates/partials/alert_row.html
+++ b/web/templates/partials/alert_row.html
@@ -66,6 +66,14 @@
{{/* actions */}}
{{if eq $status "open"}}
+ {{if eq $a.Kind "job_stuck"}}
+
+ {{end}}