194e6c9719
When dispatchBackupForGroup's conn.Send errors, queue a pending_runs row (attempt=1, next_attempt_at = now + group.RetryBackoffSeconds) instead of silently dropping the fire. The orphaned queued job row is left behind for forensic visibility — the drainer will create a fresh job row on its retry. Also adds Store.ListPendingRunsForHost — the on-reconnect drain walks every row for the host, regardless of due-ness, since the host being back makes 'due' irrelevant.
297 lines
8.4 KiB
Go
297 lines
8.4 KiB
Go
package store
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"testing"
|
|
"time"
|
|
)
|
|
|
|
func TestSourceGroupCRUDAndVersionBump(t *testing.T) {
|
|
t.Parallel()
|
|
s := openTestStore(t)
|
|
ctx := context.Background()
|
|
hostID := makeSchedHost(t, s)
|
|
|
|
keepLast := 7
|
|
g := SourceGroup{
|
|
ID: "01HSGRP00000000000000001", HostID: hostID, Name: "default",
|
|
Includes: []string{"/etc", "/home"},
|
|
Excludes: []string{"*.tmp"},
|
|
RetentionPolicy: RetentionPolicy{KeepLast: &keepLast},
|
|
}
|
|
if err := s.CreateSourceGroup(ctx, &g); err != nil {
|
|
t.Fatalf("create: %v", err)
|
|
}
|
|
if g.RetryMax != 3 || g.RetryBackoffSeconds != 60 {
|
|
t.Fatalf("retry defaults not applied: %+v", g)
|
|
}
|
|
v, _ := s.GetHostScheduleVersion(ctx, hostID)
|
|
if v != 1 {
|
|
t.Fatalf("version after create: got %d, want 1", v)
|
|
}
|
|
|
|
// Round-trip.
|
|
got, err := s.GetSourceGroup(ctx, hostID, g.ID)
|
|
if err != nil {
|
|
t.Fatalf("get: %v", err)
|
|
}
|
|
if got.Name != "default" || len(got.Includes) != 2 || len(got.Excludes) != 1 {
|
|
t.Fatalf("round-trip: %+v", got)
|
|
}
|
|
if got.RetentionPolicy.KeepLast == nil || *got.RetentionPolicy.KeepLast != 7 {
|
|
t.Fatalf("retention round-trip: %+v", got.RetentionPolicy)
|
|
}
|
|
|
|
// By name.
|
|
byName, err := s.GetSourceGroupByName(ctx, hostID, "default")
|
|
if err != nil || byName.ID != g.ID {
|
|
t.Fatalf("get by name: err=%v got=%v", err, byName)
|
|
}
|
|
|
|
// Update — rename + new retention. Version bumps.
|
|
keepDaily := 14
|
|
g.Name = "system"
|
|
g.RetentionPolicy = RetentionPolicy{KeepDaily: &keepDaily}
|
|
if err := s.UpdateSourceGroup(ctx, &g); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
v, _ = s.GetHostScheduleVersion(ctx, hostID)
|
|
if v != 2 {
|
|
t.Fatalf("version after update: got %d, want 2", v)
|
|
}
|
|
got, _ = s.GetSourceGroup(ctx, hostID, g.ID)
|
|
if got.Name != "system" || got.RetentionPolicy.KeepLast != nil || got.RetentionPolicy.KeepDaily == nil {
|
|
t.Fatalf("update did not persist: %+v", got)
|
|
}
|
|
|
|
// Conflict cache (no version bump).
|
|
if err := s.SetSourceGroupConflict(ctx, g.ID, "hourly"); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
got, _ = s.GetSourceGroup(ctx, hostID, g.ID)
|
|
if got.ConflictDimension != "hourly" {
|
|
t.Fatalf("conflict not cached: %q", got.ConflictDimension)
|
|
}
|
|
v2, _ := s.GetHostScheduleVersion(ctx, hostID)
|
|
if v2 != v {
|
|
t.Fatalf("conflict cache should not bump version: %d → %d", v, v2)
|
|
}
|
|
|
|
// List.
|
|
list, _ := s.ListSourceGroupsByHost(ctx, hostID)
|
|
if len(list) != 1 || list[0].ID != g.ID {
|
|
t.Fatalf("list: %v", list)
|
|
}
|
|
|
|
// Delete bumps version.
|
|
if err := s.DeleteSourceGroup(ctx, hostID, g.ID); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
v3, _ := s.GetHostScheduleVersion(ctx, hostID)
|
|
if v3 != 3 {
|
|
t.Fatalf("version after delete: got %d, want 3", v3)
|
|
}
|
|
if err := s.DeleteSourceGroup(ctx, hostID, g.ID); !errors.Is(err, ErrNotFound) {
|
|
t.Fatalf("delete after delete: want ErrNotFound, got %v", err)
|
|
}
|
|
}
|
|
|
|
func TestSourceGroupNameUniquePerHost(t *testing.T) {
|
|
t.Parallel()
|
|
s := openTestStore(t)
|
|
ctx := context.Background()
|
|
hostID := makeSchedHost(t, s)
|
|
|
|
if err := s.CreateSourceGroup(ctx, &SourceGroup{
|
|
ID: "01HUNIQGRP00000000000001", HostID: hostID, Name: "shared",
|
|
}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
err := s.CreateSourceGroup(ctx, &SourceGroup{
|
|
ID: "01HUNIQGRP00000000000002", HostID: hostID, Name: "shared",
|
|
})
|
|
if err == nil {
|
|
t.Fatal("expected unique-constraint error on duplicate name within host")
|
|
}
|
|
}
|
|
|
|
func TestRepoMaintenanceDefaultsAndUpdate(t *testing.T) {
|
|
t.Parallel()
|
|
s := openTestStore(t)
|
|
ctx := context.Background()
|
|
hostID := makeSchedHost(t, s)
|
|
|
|
if _, err := s.GetRepoMaintenance(ctx, hostID); !errors.Is(err, ErrNotFound) {
|
|
t.Fatalf("expected ErrNotFound before seed, got %v", err)
|
|
}
|
|
|
|
if err := s.CreateDefaultRepoMaintenance(ctx, hostID); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
m, err := s.GetRepoMaintenance(ctx, hostID)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if m.ForgetCron != "0 3 * * *" || !m.ForgetEnabled {
|
|
t.Fatalf("forget defaults: %+v", m)
|
|
}
|
|
if m.PruneCron != "0 4 * * 0" || m.CheckSubsetPct != 5 {
|
|
t.Fatalf("other defaults: %+v", m)
|
|
}
|
|
|
|
m.ForgetCron = "0 4 * * *"
|
|
m.PruneEnabled = false
|
|
m.CheckSubsetPct = 10
|
|
if err := s.UpdateRepoMaintenance(ctx, m); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
m2, _ := s.GetRepoMaintenance(ctx, hostID)
|
|
if m2.ForgetCron != "0 4 * * *" || m2.PruneEnabled || m2.CheckSubsetPct != 10 {
|
|
t.Fatalf("update did not persist: %+v", m2)
|
|
}
|
|
|
|
// CreateDefaultRepoMaintenance is idempotent (INSERT OR IGNORE).
|
|
if err := s.CreateDefaultRepoMaintenance(ctx, hostID); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
m3, _ := s.GetRepoMaintenance(ctx, hostID)
|
|
if m3.ForgetCron != "0 4 * * *" {
|
|
t.Fatalf("INSERT OR IGNORE clobbered existing row: %+v", m3)
|
|
}
|
|
}
|
|
|
|
func TestPendingRunQueue(t *testing.T) {
|
|
t.Parallel()
|
|
s := openTestStore(t)
|
|
ctx := context.Background()
|
|
hostID := makeSchedHost(t, s)
|
|
gid := makeGroup(t, s, hostID, "default", "01HPENDGRP00000000000001")
|
|
schedID := "01HPENDSCHED0000000000001"
|
|
if err := s.CreateSchedule(ctx, &Schedule{
|
|
ID: schedID, HostID: hostID, CronExpr: "@hourly", Enabled: true,
|
|
SourceGroupIDs: []string{gid},
|
|
}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
now := time.Now().UTC()
|
|
if err := s.EnqueuePendingRun(ctx, &PendingRun{
|
|
ID: "01HPEND00000000000000001",
|
|
ScheduleID: schedID, SourceGroupID: gid, HostID: hostID,
|
|
NextAttemptAt: now.Add(-time.Second), // already due
|
|
ScheduledAt: now.Add(-time.Minute),
|
|
}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
due, err := s.DuePendingRuns(ctx, now, 10)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if len(due) != 1 {
|
|
t.Fatalf("due: got %d, want 1", len(due))
|
|
}
|
|
if due[0].Attempt != 1 {
|
|
t.Fatalf("attempt: %d", due[0].Attempt)
|
|
}
|
|
|
|
// Bump.
|
|
next := now.Add(2 * time.Minute)
|
|
if err := s.BumpPendingRunAttempt(ctx, due[0].ID, next, "agent offline"); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
// No longer due at `now`.
|
|
due, _ = s.DuePendingRuns(ctx, now, 10)
|
|
if len(due) != 0 {
|
|
t.Fatalf("should not be due yet: %v", due)
|
|
}
|
|
// Due at `next`.
|
|
due, _ = s.DuePendingRuns(ctx, next, 10)
|
|
if len(due) != 1 || due[0].Attempt != 2 || due[0].LastError != "agent offline" {
|
|
t.Fatalf("after bump: %+v", due)
|
|
}
|
|
|
|
if err := s.DeletePendingRun(ctx, due[0].ID); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
due, _ = s.DuePendingRuns(ctx, next, 10)
|
|
if len(due) != 0 {
|
|
t.Fatalf("after delete: %v", due)
|
|
}
|
|
}
|
|
|
|
func TestListPendingRunsForHost(t *testing.T) {
|
|
t.Parallel()
|
|
s := openTestStore(t)
|
|
ctx := context.Background()
|
|
hostA := makeSchedHost(t, s)
|
|
hostB := "01HPENDLISTHOSTB00000001"
|
|
if err := s.CreateHost(ctx, Host{
|
|
ID: hostB, Name: "pending-list-host-b", OS: "linux", Arch: "amd64",
|
|
AgentVersion: "dev", ResticVersion: "0.16.0", ProtocolVersion: 1,
|
|
EnrolledAt: time.Now().UTC(),
|
|
}, "tokenhashB", ""); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
gA := makeGroup(t, s, hostA, "default", "01HPENDLISTGRPA000000001")
|
|
gB := makeGroup(t, s, hostB, "default", "01HPENDLISTGRPB000000001")
|
|
schedA := "01HPENDLISTSCHEDA0000001"
|
|
schedB := "01HPENDLISTSCHEDB0000001"
|
|
if err := s.CreateSchedule(ctx, &Schedule{
|
|
ID: schedA, HostID: hostA, CronExpr: "@hourly", Enabled: true,
|
|
SourceGroupIDs: []string{gA},
|
|
}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := s.CreateSchedule(ctx, &Schedule{
|
|
ID: schedB, HostID: hostB, CronExpr: "@hourly", Enabled: true,
|
|
SourceGroupIDs: []string{gB},
|
|
}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
now := time.Now().UTC()
|
|
// Two rows for hostA — one not-yet-due, one already-due — and one
|
|
// for hostB. ListPendingRunsForHost(A) must return both A rows
|
|
// (regardless of due-ness) ordered by next_attempt_at ascending.
|
|
rows := []*PendingRun{
|
|
{
|
|
ID: "01HPENDLISTROW0000000A02", ScheduleID: schedA, SourceGroupID: gA, HostID: hostA,
|
|
NextAttemptAt: now.Add(time.Hour), ScheduledAt: now,
|
|
},
|
|
{
|
|
ID: "01HPENDLISTROW0000000A01", ScheduleID: schedA, SourceGroupID: gA, HostID: hostA,
|
|
NextAttemptAt: now.Add(-time.Minute), ScheduledAt: now.Add(-time.Hour),
|
|
},
|
|
{
|
|
ID: "01HPENDLISTROW0000000B01", ScheduleID: schedB, SourceGroupID: gB, HostID: hostB,
|
|
NextAttemptAt: now, ScheduledAt: now,
|
|
},
|
|
}
|
|
for _, r := range rows {
|
|
if err := s.EnqueuePendingRun(ctx, r); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
|
|
out, err := s.ListPendingRunsForHost(ctx, hostA)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if len(out) != 2 {
|
|
t.Fatalf("len=%d, want 2: %+v", len(out), out)
|
|
}
|
|
// Ordered ascending by next_attempt_at: the -1m row first, then +1h.
|
|
if out[0].ID != "01HPENDLISTROW0000000A01" || out[1].ID != "01HPENDLISTROW0000000A02" {
|
|
t.Fatalf("order: got %s,%s", out[0].ID, out[1].ID)
|
|
}
|
|
|
|
out, err = s.ListPendingRunsForHost(ctx, "non-existent-host")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if len(out) != 0 {
|
|
t.Fatalf("non-existent host: got %d rows", len(out))
|
|
}
|
|
}
|