restic-manager/internal/server/http/job_abandon.go

package http

import (
	"context"
	"errors"
	stdhttp "net/http"
	"time"

	"github.com/go-chi/chi/v5"
	"github.com/oklog/ulid/v2"

	"gitea.dcglab.co.uk/steve/restic-manager/internal/alert"
	"gitea.dcglab.co.uk/steve/restic-manager/internal/api"
	"gitea.dcglab.co.uk/steve/restic-manager/internal/store"
)

func (s *Server) abandonJob(ctx context.Context, userID *string, jobID, reason string) error {
	job, err := s.deps.Store.GetJob(ctx, jobID)
	if err != nil {
		return err
	}
	switch api.JobStatus(job.Status) {
	case api.JobSucceeded, api.JobFailed, api.JobCancelled:
		return errJobTerminal
	}
	when := time.Now().UTC()
	if err := s.deps.Store.MarkJobFinished(ctx, jobID, string(api.JobCancelled), -1, nil, reason, when); err != nil {
		return err
	}
	_ = s.deps.Store.AppendAudit(ctx, store.AuditEntry{
		ID: ulid.Make().String(), UserID: userID, Actor: "user",
		Action: "job.abandon", TargetKind: ptr("job"), TargetID: &jobID,
		TS: when,
	})
	return nil
}

var errJobTerminal = errors.New("job already terminal")

func (s *Server) handleAbandonJob(w stdhttp.ResponseWriter, r *stdhttp.Request) {
	user, ok := s.requireUser(r)
	if !ok {
		writeJSONError(w, stdhttp.StatusUnauthorized, "unauthorised", "")
		return
	}
	jobID := chi.URLParam(r, "id")
	err := s.abandonJob(r.Context(), &user.ID, jobID, "abandoned by operator")
	switch {
	case errors.Is(err, store.ErrNotFound):
		writeJSONError(w, stdhttp.StatusNotFound, "job_not_found", "")
	case errors.Is(err, errJobTerminal):
		writeJSONError(w, stdhttp.StatusConflict, "job_terminal", "job is already terminal")
	case err != nil:
		writeJSONError(w, stdhttp.StatusInternalServerError, "internal", "")
	default:
		writeJSON(w, stdhttp.StatusOK, map[string]string{"job_id": jobID, "status": string(api.JobCancelled)})
	}
}

func (s *Server) abandonStuckAlerts(ctx context.Context, userID *string) (int, error) {
	open, err := s.deps.Store.ListAlerts(ctx, store.AlertFilter{Status: "open"})
	if err != nil {
		return 0, err
	}
	acked, err := s.deps.Store.ListAlerts(ctx, store.AlertFilter{Status: "acknowledged"})
	if err != nil {
		return 0, err
	}
	count := 0
	seen := map[string]bool{}
	for _, a := range append(open, acked...) {
		if a.Kind != alert.KindJobStuck || seen[a.DedupKey] {
			continue
		}
		seen[a.DedupKey] = true
		err := s.abandonJob(ctx, userID, a.DedupKey, "abandoned by operator after stuck-job alert")
		if err != nil {
			if errors.Is(err, errJobTerminal) || errors.Is(err, store.ErrNotFound) {
				s.resolveAlert(ctx, a.ID)
				continue
			}
			return count, err
		}
		count++
		s.resolveAlert(ctx, a.ID)
	}
	return count, nil
}

func (s *Server) resolveAlert(ctx context.Context, alertID string) {
	when := time.Now().UTC()
	if s.deps.AlertEngine != nil {
		_ = s.deps.AlertEngine.Resolve(ctx, alertID, when)
		return
	}
	_ = s.deps.Store.Resolve(ctx, alertID, when)
}

func (s *Server) handleAbandonStuckJobs(w stdhttp.ResponseWriter, r *stdhttp.Request) {
	user, ok := s.requireUser(r)
	if !ok {
		writeJSONError(w, stdhttp.StatusUnauthorized, "unauthorised", "")
		return
	}
	count, err := s.abandonStuckAlerts(r.Context(), &user.ID)
	if err != nil {
		writeJSONError(w, stdhttp.StatusInternalServerError, "internal", "")
		return
	}
	writeJSON(w, stdhttp.StatusOK, map[string]int{"abandoned": count})
}

func (s *Server) handleUIAbandonAlertJob(w stdhttp.ResponseWriter, r *stdhttp.Request) {
	user := s.requireUIUser(w, r)
	if user == nil {
		return
	}
	alertID := chi.URLParam(r, "id")
	a, err := s.deps.Store.GetAlert(r.Context(), alertID)
	if err != nil || a == nil || a.Kind != alert.KindJobStuck {
		stdhttp.Error(w, "stuck-job alert not found", stdhttp.StatusNotFound)
		return
	}
	if err := s.abandonJob(r.Context(), &user.ID, a.DedupKey, "abandoned by operator after stuck-job alert"); err != nil && !errors.Is(err, errJobTerminal) {
		stdhttp.Error(w, "unable to abandon job", stdhttp.StatusConflict)
		return
	}
	s.resolveAlert(r.Context(), alertID)
	w.Header().Set("HX-Redirect", "/alerts?"+r.URL.RawQuery)
	w.WriteHeader(stdhttp.StatusNoContent)
}

func (s *Server) handleUIAbandonStuckJobs(w stdhttp.ResponseWriter, r *stdhttp.Request) {
	user := s.requireUIUser(w, r)
	if user == nil {
		return
	}
	if _, err := s.abandonStuckAlerts(r.Context(), &user.ID); err != nil {
		stdhttp.Error(w, "unable to abandon stuck jobs", stdhttp.StatusInternalServerError)
		return
	}
	stdhttp.Redirect(w, r, "/alerts", stdhttp.StatusSeeOther)
}