restic-manager/internal/alert/rules_test.go

package alert

import (
	"context"
	"path/filepath"
	"strings"
	"testing"
	"time"

	"github.com/oklog/ulid/v2"

	"gitea.dcglab.co.uk/steve/restic-manager/internal/crypto"
	"gitea.dcglab.co.uk/steve/restic-manager/internal/notification"
	"gitea.dcglab.co.uk/steve/restic-manager/internal/store"
)

func setupEngine(t *testing.T) (*Engine, *store.Store, string) {
	t.Helper()
	dir := t.TempDir()
	st, _ := store.Open(context.Background(), filepath.Join(dir, "rm.db"))
	t.Cleanup(func() { _ = st.Close() })
	keyPath := filepath.Join(dir, "secret.key")
	_ = crypto.GenerateKeyFile(keyPath)
	key, _ := crypto.LoadKeyFromFile(keyPath)
	aead, _ := crypto.NewAEAD(key)
	hub := notification.NewHub(st, aead, "https://rm.example")
	eng := NewEngine(st, hub)
	t.Cleanup(eng.waitNotifications)
	hostID := ulid.Make().String()
	if err := st.CreateHost(context.Background(), store.Host{
		ID: hostID, Name: "alfa-01", OS: "linux", Arch: "amd64",
		EnrolledAt: time.Now().UTC(),
	}, "deadbeef", ""); err != nil {
		t.Fatalf("create host: %v", err)
	}
	return eng, st, hostID
}

func TestEngineBackupFailedRaisesThenResolves(t *testing.T) {
	t.Parallel()
	eng, st, hostID := setupEngine(t)
	ctx := context.Background()

	eng.handleJobFinished(ctx, JobFinishedEvent{
		HostID: hostID, JobID: "j1", Kind: "backup", Status: "failed",
		When: time.Now().UTC(),
	})
	open, _ := st.ListAlerts(ctx, store.AlertFilter{Status: "open", HostID: hostID})
	if len(open) != 1 || open[0].Kind != KindBackupFailed {
		t.Fatalf("expected one backup_failed open; got %+v", open)
	}

	// Second failed job should TOUCH (not raise a fresh row).
	eng.handleJobFinished(ctx, JobFinishedEvent{
		HostID: hostID, JobID: "j2", Kind: "backup", Status: "failed",
		When: time.Now().UTC().Add(time.Minute),
	})
	open, _ = st.ListAlerts(ctx, store.AlertFilter{Status: "open", HostID: hostID})
	if len(open) != 1 {
		t.Fatalf("expected dedup to stay at 1 open; got %d", len(open))
	}

	// Success auto-resolves.
	eng.handleJobFinished(ctx, JobFinishedEvent{
		HostID: hostID, JobID: "j3", Kind: "backup", Status: "succeeded",
		When: time.Now().UTC().Add(2 * time.Minute),
	})
	open, _ = st.ListAlerts(ctx, store.AlertFilter{Status: "open", HostID: hostID})
	if len(open) != 0 {
		t.Fatalf("expected zero open after success; got %d", len(open))
	}
}

func TestEngineStuckJobRaisesDeduplicatesAndResolves(t *testing.T) {
	t.Parallel()
	eng, st, hostID := setupEngine(t)
	ctx := context.Background()
	now := time.Now().UTC().Truncate(time.Second)
	started := now.Add(-7 * time.Hour)
	if err := st.CreateJob(ctx, store.Job{ID: "stuck-job", HostID: hostID, Kind: "prune", ActorKind: "user", CreatedAt: started}); err != nil {
		t.Fatalf("create job: %v", err)
	}
	if err := st.MarkJobStarted(ctx, "stuck-job", started); err != nil {
		t.Fatalf("start job: %v", err)
	}

	eng.evaluateStuckJobs(ctx, now)
	eng.evaluateStuckJobs(ctx, now.Add(time.Minute))
	open, err := st.ListAlerts(ctx, store.AlertFilter{Status: "open", HostID: hostID})
	if err != nil {
		t.Fatalf("list alerts: %v", err)
	}
	if len(open) != 1 || open[0].Kind != KindJobStuck || open[0].DedupKey != "stuck-job" {
		t.Fatalf("expected one deduplicated stuck alert, got %+v", open)
	}
	if !strings.Contains(open[0].Message, started.Format(time.RFC3339)) {
		t.Errorf("alert does not identify start/activity time: %q", open[0].Message)
	}

	if err := st.MarkJobFinished(ctx, "stuck-job", "succeeded", 0, nil, "", now); err != nil {
		t.Fatalf("finish job: %v", err)
	}
	eng.evaluateStuckJobs(ctx, now.Add(2*time.Minute))
	open, _ = st.ListAlerts(ctx, store.AlertFilter{Status: "open", HostID: hostID})
	if len(open) != 0 {
		t.Fatalf("expected terminal job alert resolved, got %+v", open)
	}
}

func TestEngineStuckJobUsesLatestLogActivity(t *testing.T) {
	t.Parallel()
	eng, st, hostID := setupEngine(t)
	ctx := context.Background()
	now := time.Now().UTC().Truncate(time.Second)
	started := now.Add(-7 * time.Hour)
	if err := st.CreateJob(ctx, store.Job{ID: "active-job", HostID: hostID, Kind: "forget", ActorKind: "user", CreatedAt: started}); err != nil {
		t.Fatalf("create job: %v", err)
	}
	if err := st.MarkJobStarted(ctx, "active-job", started); err != nil {
		t.Fatalf("start job: %v", err)
	}
	if err := st.AppendJobLog(ctx, "active-job", 1, now.Add(-time.Hour), "stdout", "active"); err != nil {
		t.Fatalf("append log: %v", err)
	}

	eng.evaluateStuckJobs(ctx, now)
	open, _ := st.ListAlerts(ctx, store.AlertFilter{Status: "open", HostID: hostID})
	if len(open) != 0 {
		t.Fatalf("recent activity should suppress stuck alert, got %+v", open)
	}
}

func TestEngineCheckFailedSeverityCritical(t *testing.T) {
	t.Parallel()
	eng, st, hostID := setupEngine(t)
	eng.handleJobFinished(context.Background(), JobFinishedEvent{
		HostID: hostID, Kind: "check", Status: "failed", When: time.Now().UTC(),
	})
	open, _ := st.ListAlerts(context.Background(),
		store.AlertFilter{Status: "open", HostID: hostID})
	if len(open) != 1 || open[0].Severity != "critical" {
		t.Fatalf("got %+v", open)
	}
}

func TestEngineAgentOfflineRespects15MinFloor(t *testing.T) {
	t.Parallel()
	eng, st, hostID := setupEngine(t)
	// Host's last_seen_at defaulted to NULL via CreateHost (enrolled but never
	// seen). Force a stale value for the test by direct DB update.
	if _, err := st.DB().Exec(
		`UPDATE hosts SET last_seen_at = ? WHERE id = ?`,
		time.Now().UTC().Add(-20*time.Minute).Format(time.RFC3339Nano), hostID,
	); err != nil {
		t.Fatalf("update last_seen_at: %v", err)
	}
	eng.handleHostOffline(context.Background(), hostID)
	open, _ := st.ListAlerts(context.Background(),
		store.AlertFilter{Status: "open", HostID: hostID})
	if len(open) != 1 {
		t.Fatalf("expected agent_offline raised; got %d", len(open))
	}

	// Bring back online — should auto-resolve.
	eng.handleHostOnline(context.Background(), hostID)
	open, _ = st.ListAlerts(context.Background(),
		store.AlertFilter{Status: "open", HostID: hostID})
	if len(open) != 0 {
		t.Fatalf("expected agent_offline resolved; got %d", len(open))
	}
}

func TestEngineAgentOfflineUnderFloorNoRaise(t *testing.T) {
	t.Parallel()
	eng, st, hostID := setupEngine(t)
	// last_seen_at is NULL from CreateHost (never touched). A nil
	// last_seen_at means the host was enrolled but never connected —
	// treat that as "now" for the floor check so we don't raise
	// immediately. handleHostOffline must skip the raise.
	eng.handleHostOffline(context.Background(), hostID)
	open, _ := st.ListAlerts(context.Background(),
		store.AlertFilter{Status: "open", HostID: hostID})
	if len(open) != 0 {
		t.Fatalf("expected no raise within 15-min floor; got %d", len(open))
	}
}