package http import ( "context" "errors" stdhttp "net/http" "time" "github.com/go-chi/chi/v5" "github.com/oklog/ulid/v2" "gitea.dcglab.co.uk/steve/restic-manager/internal/alert" "gitea.dcglab.co.uk/steve/restic-manager/internal/api" "gitea.dcglab.co.uk/steve/restic-manager/internal/store" ) func (s *Server) abandonJob(ctx context.Context, userID *string, jobID, reason string) error { job, err := s.deps.Store.GetJob(ctx, jobID) if err != nil { return err } switch api.JobStatus(job.Status) { case api.JobSucceeded, api.JobFailed, api.JobCancelled: return errJobTerminal } when := time.Now().UTC() if err := s.deps.Store.MarkJobFinished(ctx, jobID, string(api.JobCancelled), -1, nil, reason, when); err != nil { return err } _ = s.deps.Store.AppendAudit(ctx, store.AuditEntry{ ID: ulid.Make().String(), UserID: userID, Actor: "user", Action: "job.abandon", TargetKind: ptr("job"), TargetID: &jobID, TS: when, }) return nil } var errJobTerminal = errors.New("job already terminal") func (s *Server) handleAbandonJob(w stdhttp.ResponseWriter, r *stdhttp.Request) { user, ok := s.requireUser(r) if !ok { writeJSONError(w, stdhttp.StatusUnauthorized, "unauthorised", "") return } jobID := chi.URLParam(r, "id") err := s.abandonJob(r.Context(), &user.ID, jobID, "abandoned by operator") switch { case errors.Is(err, store.ErrNotFound): writeJSONError(w, stdhttp.StatusNotFound, "job_not_found", "") case errors.Is(err, errJobTerminal): writeJSONError(w, stdhttp.StatusConflict, "job_terminal", "job is already terminal") case err != nil: writeJSONError(w, stdhttp.StatusInternalServerError, "internal", "") default: writeJSON(w, stdhttp.StatusOK, map[string]string{"job_id": jobID, "status": string(api.JobCancelled)}) } } func (s *Server) abandonStuckAlerts(ctx context.Context, userID *string) (int, error) { open, err := s.deps.Store.ListAlerts(ctx, store.AlertFilter{Status: "open"}) if err != nil { return 0, err } acked, err := s.deps.Store.ListAlerts(ctx, store.AlertFilter{Status: "acknowledged"}) if err != nil { return 0, err } count := 0 seen := map[string]bool{} for _, a := range append(open, acked...) { if a.Kind != alert.KindJobStuck || seen[a.DedupKey] { continue } seen[a.DedupKey] = true err := s.abandonJob(ctx, userID, a.DedupKey, "abandoned by operator after stuck-job alert") if err != nil { if errors.Is(err, errJobTerminal) || errors.Is(err, store.ErrNotFound) { s.resolveAlert(ctx, a.ID) continue } return count, err } count++ s.resolveAlert(ctx, a.ID) } return count, nil } func (s *Server) resolveAlert(ctx context.Context, alertID string) { when := time.Now().UTC() if s.deps.AlertEngine != nil { _ = s.deps.AlertEngine.Resolve(ctx, alertID, when) return } _ = s.deps.Store.Resolve(ctx, alertID, when) } func (s *Server) handleAbandonStuckJobs(w stdhttp.ResponseWriter, r *stdhttp.Request) { user, ok := s.requireUser(r) if !ok { writeJSONError(w, stdhttp.StatusUnauthorized, "unauthorised", "") return } count, err := s.abandonStuckAlerts(r.Context(), &user.ID) if err != nil { writeJSONError(w, stdhttp.StatusInternalServerError, "internal", "") return } writeJSON(w, stdhttp.StatusOK, map[string]int{"abandoned": count}) } func (s *Server) handleUIAbandonAlertJob(w stdhttp.ResponseWriter, r *stdhttp.Request) { user := s.requireUIUser(w, r) if user == nil { return } alertID := chi.URLParam(r, "id") a, err := s.deps.Store.GetAlert(r.Context(), alertID) if err != nil || a == nil || a.Kind != alert.KindJobStuck { stdhttp.Error(w, "stuck-job alert not found", stdhttp.StatusNotFound) return } if err := s.abandonJob(r.Context(), &user.ID, a.DedupKey, "abandoned by operator after stuck-job alert"); err != nil && !errors.Is(err, errJobTerminal) { stdhttp.Error(w, "unable to abandon job", stdhttp.StatusConflict) return } s.resolveAlert(r.Context(), alertID) w.Header().Set("HX-Redirect", "/alerts?"+r.URL.RawQuery) w.WriteHeader(stdhttp.StatusNoContent) } func (s *Server) handleUIAbandonStuckJobs(w stdhttp.ResponseWriter, r *stdhttp.Request) { user := s.requireUIUser(w, r) if user == nil { return } if _, err := s.abandonStuckAlerts(r.Context(), &user.ID); err != nil { stdhttp.Error(w, "unable to abandon stuck jobs", stdhttp.StatusInternalServerError) return } stdhttp.Redirect(w, r, "/alerts", stdhttp.StatusSeeOther) }