ec0bf0f6c3
Schedules CRUD now takes {cron, enabled, source_group_ids[]} with cron
parsed via robfig/cron/v3 and group membership scoped to the host.
New source-groups CRUD lives at /api/hosts/{id}/source-groups; delete
refuses with 409 if any schedule still references the group, returning
the schedule list so the UI can prompt 'remove from these schedules
first.' Repo-maintenance GET/PUT manages forget/prune/check cadences
on host_repo_maintenance — no version bump, the server-side ticker
(P2R-06) drives execution.
Per-source-group Run-now (POST /hosts/{id}/source-groups/{gid}/run)
resolves the group's includes/excludes/retention/tag and dispatches a
backup command.run with the new structured CommandRunPayload fields
(Includes/Excludes/Tag). Old per-host /hosts/{id}/run-backup and
/hosts/{id}/init-repo return 410 Gone with a redirect message.
schedule_push.go is rebuilt: buildScheduleSetPayload assembles the
slim wire shape, pushScheduleSetOnConn ships it during the on-hello
window, pushScheduleSetAsync fires after every CRUD mutation, and
dispatchScheduledJob handles agent schedule.fire by iterating the
schedule's source groups and dispatching one backup per group with
actor_kind=schedule and scheduled_id pointing at the schedule.
Auto-init at first WS connect: when the host has repo creds bound and
no init job in its history, server dispatches restic init. Restic's
'config file already exists' soft-success means re-runs against an
existing repo no-op; we don't auto-retry on failure (operator triggers
re-init manually via the danger zone in P2R-09).
api.Schedule drops Kind/Paths/Excludes/Tags/RetentionPolicy/Manual etc.
in favour of {id, cron, enabled, source_groups: [...]}. The agent
scheduler stops checking sch.Manual; cmd/agent's backup dispatch reads
Includes/Excludes/Tag instead of Args.
Tests cover the new HTTP surface end-to-end: source-groups CRUD with
in-use refusal, schedule validation (bad cron / missing groups /
foreign group), repo-maintenance auto-seed and validation, the 410
route, and buildScheduleSetPayload's wire-shape correctness. Full
suite passes; smoke env exercises auto-init dispatch on hello,
async push after schedule create, and per-source-group Run-now
landing the right paths/excludes/tag at the agent.
219 lines
7.6 KiB
Go
219 lines
7.6 KiB
Go
// schedule_push.go — server → agent reconciliation push and the
|
|
// inbound schedule.fire dispatch.
|
|
//
|
|
// The slim-schedule wire shape is built here from the (Schedule,
|
|
// SourceGroup) pair. Each schedule is sent with its resolved source
|
|
// groups inlined so the agent doesn't have to keep its own copy of
|
|
// the group catalogue. Cron + enabled drive the agent's local timer;
|
|
// when an entry fires the agent ships back a schedule.fire and
|
|
// dispatchScheduledJob below resolves the schedule's groups and
|
|
// dispatches one backup command.run per group.
|
|
package http
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"log/slog"
|
|
"time"
|
|
|
|
"github.com/oklog/ulid/v2"
|
|
|
|
"gitea.dcglab.co.uk/steve/restic-manager/internal/api"
|
|
"gitea.dcglab.co.uk/steve/restic-manager/internal/server/ws"
|
|
"gitea.dcglab.co.uk/steve/restic-manager/internal/store"
|
|
)
|
|
|
|
// pushScheduleSetOnConn ships the canonical schedule set straight down
|
|
// the freshly-accepted hello connection. Callers are inside the
|
|
// hello window — using the conn directly avoids racing the hub's
|
|
// register-then-supersede sequence.
|
|
func (s *Server) pushScheduleSetOnConn(ctx context.Context, hostID string, conn *ws.Conn) {
|
|
payload, err := s.buildScheduleSetPayload(ctx, hostID)
|
|
if err != nil {
|
|
slog.Warn("schedule push: build payload", "host_id", hostID, "err", err)
|
|
return
|
|
}
|
|
env, err := api.Marshal(api.MsgScheduleSet, "", payload)
|
|
if err != nil {
|
|
slog.Warn("schedule push: marshal payload", "host_id", hostID, "err", err)
|
|
return
|
|
}
|
|
sendCtx, cancel := context.WithTimeout(ctx, 5*time.Second)
|
|
defer cancel()
|
|
if err := conn.Send(sendCtx, env); err != nil {
|
|
slog.Warn("schedule push on-hello: send", "host_id", hostID, "err", err)
|
|
}
|
|
}
|
|
|
|
// pushScheduleSetAsync pushes the latest schedule set to a connected
|
|
// agent (via the hub) on a best-effort basis. Mutations call this
|
|
// after a successful CRUD; offline agents pick the new version up on
|
|
// next reconnect via pushScheduleSetOnConn.
|
|
func (s *Server) pushScheduleSetAsync(hostID string) {
|
|
if s.deps.Hub == nil || !s.deps.Hub.Connected(hostID) {
|
|
return
|
|
}
|
|
go func() {
|
|
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
|
defer cancel()
|
|
payload, err := s.buildScheduleSetPayload(ctx, hostID)
|
|
if err != nil {
|
|
slog.Warn("schedule push async: build payload", "host_id", hostID, "err", err)
|
|
return
|
|
}
|
|
env, err := api.Marshal(api.MsgScheduleSet, "", payload)
|
|
if err != nil {
|
|
slog.Warn("schedule push async: marshal", "host_id", hostID, "err", err)
|
|
return
|
|
}
|
|
if err := s.deps.Hub.Send(ctx, hostID, env); err != nil {
|
|
slog.Debug("schedule push async: send", "host_id", hostID, "err", err)
|
|
}
|
|
}()
|
|
}
|
|
|
|
// buildScheduleSetPayload assembles the canonical wire shape: every
|
|
// schedule for the host with its source groups resolved inline.
|
|
func (s *Server) buildScheduleSetPayload(ctx context.Context, hostID string) (api.ScheduleSetPayload, error) {
|
|
version, err := s.deps.Store.GetHostScheduleVersion(ctx, hostID)
|
|
if err != nil {
|
|
return api.ScheduleSetPayload{}, err
|
|
}
|
|
schedules, err := s.deps.Store.ListSchedulesByHost(ctx, hostID)
|
|
if err != nil {
|
|
return api.ScheduleSetPayload{}, err
|
|
}
|
|
groups, err := s.deps.Store.ListSourceGroupsByHost(ctx, hostID)
|
|
if err != nil {
|
|
return api.ScheduleSetPayload{}, err
|
|
}
|
|
groupByID := make(map[string]store.SourceGroup, len(groups))
|
|
for _, g := range groups {
|
|
groupByID[g.ID] = g
|
|
}
|
|
|
|
out := api.ScheduleSetPayload{Version: version, Schedules: make([]api.Schedule, 0, len(schedules))}
|
|
for _, sc := range schedules {
|
|
entry := api.Schedule{
|
|
ID: sc.ID,
|
|
CronExpr: sc.CronExpr,
|
|
Enabled: sc.Enabled,
|
|
SourceGroups: make([]api.ScheduleSourceGroup, 0, len(sc.SourceGroupIDs)),
|
|
}
|
|
for _, gid := range sc.SourceGroupIDs {
|
|
g, ok := groupByID[gid]
|
|
if !ok {
|
|
continue
|
|
}
|
|
retention, _ := json.Marshal(g.RetentionPolicy)
|
|
entry.SourceGroups = append(entry.SourceGroups, api.ScheduleSourceGroup{
|
|
Name: g.Name,
|
|
Includes: g.Includes,
|
|
Excludes: g.Excludes,
|
|
RetentionPolicy: retention,
|
|
RetryMax: g.RetryMax,
|
|
RetryBackoffSeconds: g.RetryBackoffSeconds,
|
|
})
|
|
}
|
|
out.Schedules = append(out.Schedules, entry)
|
|
}
|
|
return out, nil
|
|
}
|
|
|
|
// applyScheduleAck persists the version the agent has confirmed.
|
|
func (s *Server) applyScheduleAck(ctx context.Context, hostID string, version int64, appliedAt time.Time) {
|
|
if err := s.deps.Store.SetHostAppliedScheduleVersion(ctx, hostID, version); err != nil {
|
|
slog.Warn("schedule.ack: persist applied version", "host_id", hostID, "err", err)
|
|
}
|
|
}
|
|
|
|
// dispatchScheduledJob handles an agent's schedule.fire. Resolves the
|
|
// schedule's source groups and dispatches one backup command.run per
|
|
// group, persisting each as a job row with actor_kind=schedule and
|
|
// scheduled_id pointing at the schedule.
|
|
func (s *Server) dispatchScheduledJob(ctx context.Context, hostID string, conn *ws.Conn, scheduleID string, scheduledAt time.Time) {
|
|
sc, err := s.deps.Store.GetSchedule(ctx, hostID, scheduleID)
|
|
if err != nil {
|
|
if errors.Is(err, store.ErrNotFound) {
|
|
slog.Info("schedule.fire: schedule unknown, ignoring",
|
|
"host_id", hostID, "schedule_id", scheduleID)
|
|
return
|
|
}
|
|
slog.Warn("schedule.fire: load schedule", "host_id", hostID, "err", err)
|
|
return
|
|
}
|
|
if !sc.Enabled {
|
|
slog.Info("schedule.fire: schedule disabled, ignoring",
|
|
"host_id", hostID, "schedule_id", scheduleID)
|
|
return
|
|
}
|
|
if len(sc.SourceGroupIDs) == 0 {
|
|
slog.Warn("schedule.fire: schedule has no source groups",
|
|
"host_id", hostID, "schedule_id", scheduleID)
|
|
return
|
|
}
|
|
for _, gid := range sc.SourceGroupIDs {
|
|
g, err := s.deps.Store.GetSourceGroup(ctx, hostID, gid)
|
|
if err != nil {
|
|
slog.Warn("schedule.fire: load source group",
|
|
"host_id", hostID, "schedule_id", scheduleID, "group_id", gid, "err", err)
|
|
continue
|
|
}
|
|
s.dispatchBackupForGroup(ctx, conn, hostID, scheduleID, g, scheduledAt)
|
|
}
|
|
}
|
|
|
|
// dispatchBackupForGroup builds and sends a single backup command.run
|
|
// envelope on conn for the given group. Persists the job row first so
|
|
// the live log viewer can subscribe to it.
|
|
func (s *Server) dispatchBackupForGroup(ctx context.Context, conn *ws.Conn, hostID, scheduleID string, g *store.SourceGroup, scheduledAt time.Time) {
|
|
jobID := ulid.Make().String()
|
|
now := time.Now().UTC()
|
|
scheduleRef := scheduleID
|
|
if err := s.deps.Store.CreateJob(ctx, store.Job{
|
|
ID: jobID,
|
|
HostID: hostID,
|
|
Kind: string(api.JobBackup),
|
|
ScheduledID: &scheduleRef,
|
|
ActorKind: "schedule",
|
|
CreatedAt: now,
|
|
}); err != nil {
|
|
slog.Warn("schedule.fire: persist job", "host_id", hostID,
|
|
"schedule_id", scheduleID, "group", g.Name, "err", err)
|
|
return
|
|
}
|
|
retention, _ := json.Marshal(g.RetentionPolicy)
|
|
env, err := api.Marshal(api.MsgCommandRun, jobID, api.CommandRunPayload{
|
|
JobID: jobID,
|
|
Kind: api.JobBackup,
|
|
Includes: g.Includes,
|
|
Excludes: g.Excludes,
|
|
Tag: g.Name,
|
|
RetentionPolicy: retention,
|
|
})
|
|
if err != nil {
|
|
slog.Warn("schedule.fire: marshal command.run",
|
|
"host_id", hostID, "schedule_id", scheduleID, "err", err)
|
|
return
|
|
}
|
|
sendCtx, cancel := context.WithTimeout(ctx, 5*time.Second)
|
|
defer cancel()
|
|
if err := conn.Send(sendCtx, env); err != nil {
|
|
slog.Warn("schedule.fire: send command.run",
|
|
"host_id", hostID, "schedule_id", scheduleID, "err", err)
|
|
return
|
|
}
|
|
_ = s.deps.Store.AppendAudit(ctx, store.AuditEntry{
|
|
ID: ulid.Make().String(),
|
|
Actor: "schedule",
|
|
Action: "job.run_now",
|
|
TargetKind: ptr("job"),
|
|
TargetID: &jobID,
|
|
TS: now,
|
|
})
|
|
slog.Info("schedule.fire: dispatched backup",
|
|
"host_id", hostID, "schedule_id", scheduleID,
|
|
"group", g.Name, "job_id", jobID, "scheduled_at", scheduledAt)
|
|
}
|