From 727c610765a9afbd68c56f9027aa49c623f70c6e Mon Sep 17 00:00:00 2001 From: Steve Cliff Date: Mon, 4 May 2026 17:12:45 +0100 Subject: [PATCH] P3 follow-up: log download (txt + ndjson) on the live job page MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The diff job's full output streams to the standard live job log page, which can be a lot of text the operator wants to grep through or paste into a ticket. Add a Download button. Source of truth is the persisted job_logs table — works any time (running or finished) and doesn't need to pause the live WS stream. The download is 'everything the server has up to right now'; if the operator wants a fuller snapshot of a still-running job, they hit Download again. - New endpoint GET /api/jobs/{id}/log.{txt,ndjson} (chi {format} matcher constrained to the two known suffixes). Auth via session cookie. 404 on unknown job. - internal/server/http/job_download.go writeLogsText emits a small header + 'HH:MM:SS.mmm TAG payload' rows mirroring what the live page shows. writeLogsNDJSON emits one self-contained {seq,ts,stream, payload} JSON object per line — appending stays valid (each line stands alone), and the whole file pipes cleanly into jq. NDJSON is newline-delimited JSON; not the same as a JSON array. - web/templates/pages/job_detail.html grows two header buttons: 'Download log' (txt) + '.ndjson' ghost variant for tooling. Tests cover the txt format (header + per-row shape), the ndjson format (each line round-trips through json.Unmarshal), unknown job 404, unauthenticated 401. --- internal/server/http/job_download.go | 135 ++++++++++++++++ internal/server/http/job_download_test.go | 181 ++++++++++++++++++++++ internal/server/http/server.go | 5 + web/templates/pages/job_detail.html | 5 + 4 files changed, 326 insertions(+) create mode 100644 internal/server/http/job_download.go create mode 100644 internal/server/http/job_download_test.go diff --git a/internal/server/http/job_download.go b/internal/server/http/job_download.go new file mode 100644 index 0000000..d268cec --- /dev/null +++ b/internal/server/http/job_download.go @@ -0,0 +1,135 @@ +package http + +import ( + "bufio" + "encoding/json" + "fmt" + stdhttp "net/http" + "strings" + + "github.com/go-chi/chi/v5" + + "gitea.dcglab.co.uk/steve/restic-manager/internal/store" +) + +// handleJobLogDownload is GET /api/jobs/{id}/log{.txt,.ndjson}. +// +// Source of truth is the persisted job_logs table — works any time, +// regardless of whether the job is running or already finished. The +// download is "everything the server has up to right now"; the live +// stream is unaffected (no pause needed). If the operator wants a +// fuller snapshot of a still-running job, they hit Download again. +// +// Format is picked from the URL suffix (.txt | .ndjson) for a +// sensible filename in the browser, or the ?format= query param for +// REST callers. Default is txt. +func (s *Server) handleJobLogDownload(w stdhttp.ResponseWriter, r *stdhttp.Request) { + if _, ok := s.requireUser(r); !ok { + writeJSONError(w, stdhttp.StatusUnauthorized, "unauthorized", "") + return + } + jobID := chi.URLParam(r, "id") + if jobID == "" { + writeJSONError(w, stdhttp.StatusBadRequest, "missing_job_id", "") + return + } + job, err := s.deps.Store.GetJob(r.Context(), jobID) + if err != nil { + writeJSONError(w, stdhttp.StatusNotFound, "job_not_found", "") + return + } + + format := r.URL.Query().Get("format") + if format == "" { + // Sniff the URL — chi routes both /log.txt and /log.ndjson here + // (or .log if a future route adds it) via the {format} matcher. + fmtParam := chi.URLParam(r, "format") + switch fmtParam { + case "ndjson": + format = "ndjson" + default: + format = "txt" + } + } + + logs, err := s.deps.Store.ListJobLogs(r.Context(), jobID, 0, 0) + if err != nil { + writeJSONError(w, stdhttp.StatusInternalServerError, "internal", err.Error()) + return + } + + short := jobID + if len(short) > 8 { + short = short[:8] + } + filename := "job-" + job.Kind + "-" + short + switch format { + case "ndjson": + w.Header().Set("Content-Type", "application/x-ndjson; charset=utf-8") + w.Header().Set("Content-Disposition", + `attachment; filename="`+filename+`.ndjson"`) + writeLogsNDJSON(w, logs) + default: + w.Header().Set("Content-Type", "text/plain; charset=utf-8") + w.Header().Set("Content-Disposition", + `attachment; filename="`+filename+`.txt"`) + writeLogsText(w, job, logs) + } +} + +// writeLogsText renders the logs in the same shape the live page shows: +// "HH:MM:SS.mmm TAG payload". Adds a small header so the file is +// useful as a standalone artifact (operator pastes it into a ticket). +func writeLogsText(w stdhttp.ResponseWriter, job *store.Job, logs []store.JobLogLine) { + bw := bufio.NewWriter(w) + defer func() { _ = bw.Flush() }() + _, _ = fmt.Fprintf(bw, "# job %s · kind %s · status %s\n", + job.ID, job.Kind, job.Status) + if job.StartedAt != nil { + _, _ = fmt.Fprintf(bw, "# started %s\n", job.StartedAt.UTC().Format("2006-01-02T15:04:05.000Z")) + } + if job.FinishedAt != nil { + _, _ = fmt.Fprintf(bw, "# finished %s\n", job.FinishedAt.UTC().Format("2006-01-02T15:04:05.000Z")) + } + _, _ = fmt.Fprintf(bw, "# %d log lines\n\n", len(logs)) + for _, l := range logs { + tag := streamTag(l.Stream) + ts := l.TS.UTC().Format("15:04:05.000") + // Strip embedded newlines from payload — log lines should be + // single-line, but defensive: a stray '\n' in stderr would + // break grep -n. + payload := strings.ReplaceAll(l.Payload, "\n", " ") + _, _ = fmt.Fprintf(bw, "%s %s %s\n", ts, tag, payload) + } +} + +// writeLogsNDJSON emits one JSON object per line. Each object stands +// alone — appending to the file remains valid NDJSON. +func writeLogsNDJSON(w stdhttp.ResponseWriter, logs []store.JobLogLine) { + enc := json.NewEncoder(w) + for _, l := range logs { + _ = enc.Encode(struct { + Seq int64 `json:"seq"` + TS string `json:"ts"` + Stream string `json:"stream"` + Payload string `json:"payload"` + }{ + Seq: l.Seq, + TS: l.TS.UTC().Format("2006-01-02T15:04:05.000Z"), + Stream: l.Stream, + Payload: l.Payload, + }) + } +} + +func streamTag(s string) string { + switch s { + case "stdout": + return "OUT" + case "stderr": + return "ERR" + case "event": + return "EVENT" + } + return strings.ToUpper(s) +} diff --git a/internal/server/http/job_download_test.go b/internal/server/http/job_download_test.go new file mode 100644 index 0000000..429f76c --- /dev/null +++ b/internal/server/http/job_download_test.go @@ -0,0 +1,181 @@ +// job_download_test.go — covers GET /api/jobs/{id}/log.{txt,ndjson}. +package http + +import ( + "context" + "encoding/json" + stdhttp "net/http" + "strings" + "testing" + "time" + + "github.com/oklog/ulid/v2" + + "gitea.dcglab.co.uk/steve/restic-manager/internal/store" +) + +// seedJobWithLogs creates a job + a few log lines for it. Returns the +// job ID. Caller is responsible for the test server + auth. +func seedJobWithLogs(t *testing.T, st *store.Store, hostID string, lineCount int) string { + t.Helper() + jobID := ulid.Make().String() + now := time.Now().UTC() + if err := st.CreateJob(context.Background(), store.Job{ + ID: jobID, HostID: hostID, Kind: "diff", + ActorKind: "user", CreatedAt: now, + }); err != nil { + t.Fatalf("create job: %v", err) + } + if err := st.MarkJobStarted(context.Background(), jobID, now); err != nil { + t.Fatalf("mark started: %v", err) + } + for i := 0; i < lineCount; i++ { + stream := "stdout" + if i%5 == 0 { + stream = "stderr" + } + payload := `{"message_type":"change","path":"/etc/file` + + ulid.Make().String()[:6] + `","modifier":"M"}` + if err := st.AppendJobLog(context.Background(), jobID, int64(i+1), + now.Add(time.Duration(i)*time.Millisecond), + stream, payload); err != nil { + t.Fatalf("append log: %v", err) + } + } + if err := st.MarkJobFinished(context.Background(), jobID, "succeeded", 0, nil, "", now); err != nil { + t.Fatalf("mark finished: %v", err) + } + return jobID +} + +// TestJobLogDownloadTxt: plain-text format includes a header + one +// line per log row in the expected shape. +func TestJobLogDownloadTxt(t *testing.T) { + t.Parallel() + srv, ts, st := rawTestServer(t) + hostID, _ := enrolHostForWS(t, srv, st, "dl-txt-host") + jobID := seedJobWithLogs(t, st, hostID, 12) + cookie := loginAsAdmin(t, st) + + req, _ := stdhttp.NewRequest("GET", + ts.URL+"/api/jobs/"+jobID+"/log.txt", nil) + req.AddCookie(cookie) + res, err := stdhttp.DefaultClient.Do(req) + if err != nil { + t.Fatalf("do: %v", err) + } + defer res.Body.Close() + if res.StatusCode != stdhttp.StatusOK { + t.Fatalf("status: got %d, want 200", res.StatusCode) + } + if ct := res.Header.Get("Content-Type"); !strings.HasPrefix(ct, "text/plain") { + t.Errorf("content-type: got %q", ct) + } + if cd := res.Header.Get("Content-Disposition"); !strings.Contains(cd, ".txt") { + t.Errorf("content-disposition: got %q", cd) + } + body := readBody(t, res.Body) + // Header lines. + if !strings.HasPrefix(body, "# job ") { + t.Errorf("expected '# job ...' header line; got %q", short(body)) + } + if !strings.Contains(body, "12 log lines") { + t.Errorf("expected '12 log lines'; got %q", short(body)) + } + // One body line per log row — count non-comment, non-empty lines. + var rows int + for _, line := range strings.Split(body, "\n") { + l := strings.TrimSpace(line) + if l == "" || strings.HasPrefix(l, "#") { + continue + } + rows++ + } + if rows != 12 { + t.Errorf("expected 12 body rows, got %d", rows) + } + // Tag check: at least one ERR row (every 5th was stderr). + if !strings.Contains(body, " ERR ") { + t.Errorf("expected at least one ERR row") + } +} + +// TestJobLogDownloadNDJSON: each line is a self-contained JSON object. +func TestJobLogDownloadNDJSON(t *testing.T) { + t.Parallel() + srv, ts, st := rawTestServer(t) + hostID, _ := enrolHostForWS(t, srv, st, "dl-ndjson-host") + jobID := seedJobWithLogs(t, st, hostID, 5) + cookie := loginAsAdmin(t, st) + + req, _ := stdhttp.NewRequest("GET", + ts.URL+"/api/jobs/"+jobID+"/log.ndjson", nil) + req.AddCookie(cookie) + res, err := stdhttp.DefaultClient.Do(req) + if err != nil { + t.Fatalf("do: %v", err) + } + defer res.Body.Close() + if res.StatusCode != stdhttp.StatusOK { + t.Fatalf("status: got %d, want 200", res.StatusCode) + } + if ct := res.Header.Get("Content-Type"); !strings.HasPrefix(ct, "application/x-ndjson") { + t.Errorf("content-type: got %q", ct) + } + body := readBody(t, res.Body) + // Each non-empty line should parse as an object with seq/ts/stream/payload. + var seen int + for _, line := range strings.Split(body, "\n") { + if strings.TrimSpace(line) == "" { + continue + } + var obj struct { + Seq int64 `json:"seq"` + TS string `json:"ts"` + Stream string `json:"stream"` + Payload string `json:"payload"` + } + if err := json.Unmarshal([]byte(line), &obj); err != nil { + t.Fatalf("parse line %q: %v", line, err) + } + if obj.Seq == 0 || obj.TS == "" || obj.Stream == "" || obj.Payload == "" { + t.Errorf("incomplete object: %+v", obj) + } + seen++ + } + if seen != 5 { + t.Errorf("parsed %d objects, want 5", seen) + } +} + +// TestJobLogDownloadNotFound: 404 for an unknown job id. +func TestJobLogDownloadNotFound(t *testing.T) { + t.Parallel() + _, ts, st := rawTestServer(t) + cookie := loginAsAdmin(t, st) + req, _ := stdhttp.NewRequest("GET", + ts.URL+"/api/jobs/"+ulid.Make().String()+"/log.txt", nil) + req.AddCookie(cookie) + res, err := stdhttp.DefaultClient.Do(req) + if err != nil { + t.Fatalf("do: %v", err) + } + defer res.Body.Close() + if res.StatusCode != stdhttp.StatusNotFound { + t.Fatalf("status: got %d, want 404", res.StatusCode) + } +} + +// TestJobLogDownloadUnauthenticated: without a session cookie, 401. +func TestJobLogDownloadUnauthenticated(t *testing.T) { + t.Parallel() + _, ts, _ := rawTestServer(t) + res, err := stdhttp.Get(ts.URL + "/api/jobs/x/log.txt") + if err != nil { + t.Fatalf("do: %v", err) + } + defer res.Body.Close() + if res.StatusCode != stdhttp.StatusUnauthorized { + t.Fatalf("status: got %d, want 401", res.StatusCode) + } +} diff --git a/internal/server/http/server.go b/internal/server/http/server.go index 0f14f4b..aa1313f 100644 --- a/internal/server/http/server.go +++ b/internal/server/http/server.go @@ -304,6 +304,11 @@ func (s *Server) routes(r chi.Router) { if s.deps.JobHub != nil { r.Get("/api/jobs/{id}/stream", s.handleJobStream) } + + // Job log download (txt + ndjson). Source of truth is the + // persisted job_logs table; safe to call any time, no pause + // needed against the live stream. + r.Get("/api/jobs/{id}/log.{format:txt|ndjson}", s.handleJobLogDownload) } // Start begins listening. Blocks until ListenAndServe returns diff --git a/web/templates/pages/job_detail.html b/web/templates/pages/job_detail.html index 47e2445..2d238be 100644 --- a/web/templates/pages/job_detail.html +++ b/web/templates/pages/job_detail.html @@ -63,6 +63,11 @@
+ + Download log + + .ndjson {{if $page.IsActive}}