internal/alert/engine.go

@@ -61,9 +61,20 @@ type Engine struct {
 	stuckThresholds   map[string]time.Duration
 
 	closeOnce sync.Once
+	notifyWG  sync.WaitGroup
 	done      chan struct{}
 }
 
+func (e *Engine) dispatchNotification(ctx context.Context, payload notification.Payload) {
+	e.notifyWG.Add(1)
+	go func() {
+		defer e.notifyWG.Done()
+		e.hub.Dispatch(ctx, payload)
+	}()
+}
+
+func (e *Engine) waitNotifications() { e.notifyWG.Wait() }
+
 // NewEngine builds the engine. agentOfflineFloor + tickPeriod default
 // to 15min and 60s respectively when zero.
 func NewEngine(st *store.Store, hub *notification.Hub) *Engine {

internal/alert/rules.go

@@ -60,7 +60,7 @@ func (e *Engine) raiseAndNotify(ctx context.Context, hostID, kind, dedupKey, sev
 	if err == nil {
 		hostName = host.Name
 	}
-	go e.hub.Dispatch(ctx, notification.Payload{
+	e.dispatchNotification(ctx, notification.Payload{
 		Event:    notification.EventRaised,
 		AlertID:  id,
 		Severity: severity,
@@ -85,7 +85,7 @@ func (e *Engine) Acknowledge(ctx context.Context, alertID, userID string, when t
 		return nil //nolint:nilerr
 	}
 	p := alertPayload(ctx, e.store, notification.EventAcknowledged, a)
-	go e.hub.Dispatch(context.WithoutCancel(ctx), p)
+	e.dispatchNotification(context.WithoutCancel(ctx), p)
 	return nil
 }
 
@@ -99,7 +99,7 @@ func (e *Engine) Resolve(ctx context.Context, alertID string, when time.Time) er
 		return nil
 	}
 	p := alertPayload(ctx, e.store, notification.EventResolved, a)
-	go e.hub.Dispatch(context.WithoutCancel(ctx), p)
+	e.dispatchNotification(context.WithoutCancel(ctx), p)
 	return nil
 }
 
@@ -164,7 +164,7 @@ func (e *Engine) resolveAndNotify(ctx context.Context, hostID, kind, dedupKey st
 		if a.Kind != kind || a.DedupKey != dedupKey {
 			continue
 		}
-		go e.hub.Dispatch(ctx, notification.Payload{
+		e.dispatchNotification(ctx, notification.Payload{
 			Event:    notification.EventResolved,
 			AlertID:  a.ID,
 			Severity: a.Severity,

internal/alert/rules_test.go

@@ -25,6 +25,7 @@ func setupEngine(t *testing.T) (*Engine, *store.Store, string) {
 	aead, _ := crypto.NewAEAD(key)
 	hub := notification.NewHub(st, aead, "https://rm.example")
 	eng := NewEngine(st, hub)
+	t.Cleanup(eng.waitNotifications)
 	hostID := ulid.Make().String()
 	if err := st.CreateHost(context.Background(), store.Host{
 		ID: hostID, Name: "alfa-01", OS: "linux", Arch: "amd64",