// catchup_scheduler_test.go — integration tests for the catch-up scheduler. package http import ( "context" "testing" "time" "github.com/oklog/ulid/v2" "gitea.dcglab.co.uk/steve/restic-manager/internal/api" "gitea.dcglab.co.uk/steve/restic-manager/internal/store" ) // TestRunCatchupDispatchesOverdue verifies four properties of the // catch-up scheduler in separate sub-tests sharing no state. func TestRunCatchupDispatchesOverdue(t *testing.T) { t.Parallel() // --- 1. Overdue host with connected agent → backup dispatched ------- t.Run("overdue_dispatch", func(t *testing.T) { t.Parallel() srv, ts, st := rawTestServer(t) hostID, token := enrolHostForWS(t, srv, st, "catchup-overdue") if err := st.SetHostAlwaysOn(context.Background(), hostID, false); err != nil { t.Fatalf("set always_on: %v", err) } // Last backup ~8 days ago → schedule overdue. eightDaysAgo := time.Now().UTC().Add(-8 * 24 * time.Hour) if err := st.SetHostLastBackup(context.Background(), hostID, "succeeded", eightDaysAgo); err != nil { t.Fatalf("set last backup: %v", err) } if err := st.CreateJob(context.Background(), store.Job{ ID: ulid.Make().String(), HostID: hostID, Kind: "init", ActorKind: "system", CreatedAt: time.Now().UTC(), }); err != nil { t.Fatalf("seed init: %v", err) } gid := ulid.Make().String() if err := st.CreateSourceGroup(context.Background(), &store.SourceGroup{ ID: gid, HostID: hostID, Name: "home", Includes: []string{"/home"}, }); err != nil { t.Fatalf("source group: %v", err) } sid := ulid.Make().String() if err := st.CreateSchedule(context.Background(), &store.Schedule{ ID: sid, HostID: hostID, CronExpr: "0 2 * * *", Enabled: true, SourceGroupIDs: []string{gid}, }); err != nil { t.Fatalf("schedule: %v", err) } c := agentDial(t, srv, ts, hostID, token) sendHello(t, c, "catchup-overdue") _ = drainUntil(t, c, api.MsgScheduleSet) // Arm with a past time so the settle window is already elapsed. srv.ArmCatchup(hostID, time.Now().UTC().Add(-2*time.Minute)) srv.RunCatchupsDue(context.Background()) // Give the dispatch goroutine a moment to write the job row. time.Sleep(100 * time.Millisecond) var n int if err := st.DB().QueryRow( `SELECT COUNT(*) FROM jobs WHERE host_id = ? AND kind = 'backup'`, hostID).Scan(&n); err != nil { t.Fatalf("count: %v", err) } if n < 1 { t.Errorf("overdue host: want ≥1 backup job, got %d", n) } }) // --- 2. Not overdue → no dispatch ----------------------------------- t.Run("not_overdue_no_dispatch", func(t *testing.T) { t.Parallel() srv, ts, st := rawTestServer(t) hostID, token := enrolHostForWS(t, srv, st, "catchup-notoverdue") if err := st.SetHostAlwaysOn(context.Background(), hostID, false); err != nil { t.Fatalf("set always_on: %v", err) } // Last backup just now → not overdue. now := time.Now().UTC() if err := st.SetHostLastBackup(context.Background(), hostID, "succeeded", now); err != nil { t.Fatalf("set last backup: %v", err) } if err := st.CreateJob(context.Background(), store.Job{ ID: ulid.Make().String(), HostID: hostID, Kind: "init", ActorKind: "system", CreatedAt: now, }); err != nil { t.Fatalf("seed init: %v", err) } gid := ulid.Make().String() if err := st.CreateSourceGroup(context.Background(), &store.SourceGroup{ ID: gid, HostID: hostID, Name: "home", Includes: []string{"/home"}, }); err != nil { t.Fatalf("source group: %v", err) } sid := ulid.Make().String() if err := st.CreateSchedule(context.Background(), &store.Schedule{ ID: sid, HostID: hostID, CronExpr: "0 2 * * *", Enabled: true, SourceGroupIDs: []string{gid}, }); err != nil { t.Fatalf("schedule: %v", err) } c := agentDial(t, srv, ts, hostID, token) sendHello(t, c, "catchup-notoverdue") _ = drainUntil(t, c, api.MsgScheduleSet) srv.ArmCatchup(hostID, time.Now().UTC().Add(-2*time.Minute)) srv.RunCatchupsDue(context.Background()) time.Sleep(100 * time.Millisecond) var n int if err := st.DB().QueryRow( `SELECT COUNT(*) FROM jobs WHERE host_id = ? AND kind = 'backup'`, hostID).Scan(&n); err != nil { t.Fatalf("count: %v", err) } if n != 0 { t.Errorf("not-overdue host: want 0 backup jobs, got %d", n) } }) // --- 3. Active backup in flight → no new dispatch ------------------- t.Run("active_backup_blocks_dispatch", func(t *testing.T) { t.Parallel() srv, ts, st := rawTestServer(t) hostID, token := enrolHostForWS(t, srv, st, "catchup-active") if err := st.SetHostAlwaysOn(context.Background(), hostID, false); err != nil { t.Fatalf("set always_on: %v", err) } eightDaysAgo := time.Now().UTC().Add(-8 * 24 * time.Hour) if err := st.SetHostLastBackup(context.Background(), hostID, "succeeded", eightDaysAgo); err != nil { t.Fatalf("set last backup: %v", err) } if err := st.CreateJob(context.Background(), store.Job{ ID: ulid.Make().String(), HostID: hostID, Kind: "init", ActorKind: "system", CreatedAt: time.Now().UTC(), }); err != nil { t.Fatalf("seed init: %v", err) } gid := ulid.Make().String() if err := st.CreateSourceGroup(context.Background(), &store.SourceGroup{ ID: gid, HostID: hostID, Name: "home", Includes: []string{"/home"}, }); err != nil { t.Fatalf("source group: %v", err) } sid := ulid.Make().String() if err := st.CreateSchedule(context.Background(), &store.Schedule{ ID: sid, HostID: hostID, CronExpr: "0 2 * * *", Enabled: true, SourceGroupIDs: []string{gid}, }); err != nil { t.Fatalf("schedule: %v", err) } // Seed a queued backup job — this is "already in flight". if err := st.CreateJob(context.Background(), store.Job{ ID: ulid.Make().String(), HostID: hostID, Kind: "backup", ActorKind: "schedule", CreatedAt: time.Now().UTC(), }); err != nil { t.Fatalf("seed queued backup: %v", err) } c := agentDial(t, srv, ts, hostID, token) sendHello(t, c, "catchup-active") _ = drainUntil(t, c, api.MsgScheduleSet) srv.ArmCatchup(hostID, time.Now().UTC().Add(-2*time.Minute)) srv.RunCatchupsDue(context.Background()) time.Sleep(100 * time.Millisecond) var n int if err := st.DB().QueryRow( `SELECT COUNT(*) FROM jobs WHERE host_id = ? AND kind = 'backup'`, hostID).Scan(&n); err != nil { t.Fatalf("count: %v", err) } // Count must still be exactly 1 — no second job added. if n != 1 { t.Errorf("active backup guard: want 1 job (the seeded one), got %d", n) } }) // --- 4. Disconnected host → no dispatch ----------------------------- t.Run("disconnected_no_dispatch", func(t *testing.T) { t.Parallel() srv, _, st := rawTestServer(t) hostID, _ := enrolHostForWS(t, srv, st, "catchup-disconnected") if err := st.SetHostAlwaysOn(context.Background(), hostID, false); err != nil { t.Fatalf("set always_on: %v", err) } eightDaysAgo := time.Now().UTC().Add(-8 * 24 * time.Hour) if err := st.SetHostLastBackup(context.Background(), hostID, "succeeded", eightDaysAgo); err != nil { t.Fatalf("set last backup: %v", err) } if err := st.CreateJob(context.Background(), store.Job{ ID: ulid.Make().String(), HostID: hostID, Kind: "init", ActorKind: "system", CreatedAt: time.Now().UTC(), }); err != nil { t.Fatalf("seed init: %v", err) } gid := ulid.Make().String() if err := st.CreateSourceGroup(context.Background(), &store.SourceGroup{ ID: gid, HostID: hostID, Name: "home", Includes: []string{"/home"}, }); err != nil { t.Fatalf("source group: %v", err) } sid := ulid.Make().String() if err := st.CreateSchedule(context.Background(), &store.Schedule{ ID: sid, HostID: hostID, CronExpr: "0 2 * * *", Enabled: true, SourceGroupIDs: []string{gid}, }); err != nil { t.Fatalf("schedule: %v", err) } // Host is NOT connected — no agentDial. srv.ArmCatchup(hostID, time.Now().UTC().Add(-2*time.Minute)) srv.RunCatchupsDue(context.Background()) time.Sleep(100 * time.Millisecond) var n int if err := st.DB().QueryRow( `SELECT COUNT(*) FROM jobs WHERE host_id = ? AND kind = 'backup'`, hostID).Scan(&n); err != nil { t.Fatalf("count: %v", err) } if n != 0 { t.Errorf("disconnected host: want 0 backup jobs, got %d", n) } }) }