fix(ingestion): tripwire alerts survive outages; drop only permanent rejections

PR #160 round 2: dropping a message after 5 failures could not tell a
poison message from an ntfy outage. A 15-minute outage dropped the only
incident with nothing reported, and drops were not counted (74 of 300
removals were neither delivered nor counted).

- audit.NtfyNotifier returns *RejectedError for non-2xx; Permanent() is
  true for 4xx other than 429.
- The tripwire queue drops a message only on a permanent rejection and
  counts it; every other failure is an outage and is retried, however
  long. Each flush tries every queued message, so an undeliverable one
  cannot block the rest.
- Overflow and drops go into one counter, sent as a summary retried
  every tick and never dropped. Every removal is delivered or counted.

Tests (red first): an outage of ~100 ticks still delivers the only
incident; delivered + counted equals the number of removals; a
permanently rejected message is dropped and counted. The short-outage
test is renamed to say it is short. The Watch doc no longer claims
'loses none' unconditionally.

Refs #157

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

ingestion/internal/symlinkguard/symlinkguard_test.go

@@ -3,6 +3,7 @@ package symlinkguard
 import (
 	"context"
 	"errors"
+	"fmt"
 	"os"
 	"path/filepath"
 	"strconv"
@@ -305,12 +306,18 @@ type rejectNotifier struct {
 	reject string
 }
 
+// permanentErr is a rejection the notifier says can never succeed (like ntfy 413).
+type permanentErr struct{}
+
+func (permanentErr) Error() string   { return "413 payload too large" }
+func (permanentErr) Permanent() bool { return true }
+
 func (r *rejectNotifier) Notify(ctx context.Context, m string) error {
 	if strings.Contains(m, r.reject) {
 		r.mu.Lock()
 		r.msgs = append(r.msgs, "REJECTED")
 		r.mu.Unlock()
-		return errors.New("413")
+		return permanentErr{}
 	}
 	return r.recNotifier.Notify(ctx, m)
 }
@@ -383,7 +390,7 @@ func TestWatchQueueIsBoundedDuringAnOutage(t *testing.T) {
 // Within the bound, every alert queued during an outage is delivered after
 // it, each naming its own removal (PR #160 review: truncating the queue after
 // the first send stayed green).
-func TestWatchDeliversEveryQueuedAlertAfterAnOutage(t *testing.T) {
+func TestWatchDeliversEveryQueuedAlertAfterAShortOutage(t *testing.T) {
 	brain := t.TempDir()
 	n := &toggleNotifier{}
 	n.down.Store(true)
@@ -403,3 +410,89 @@ func TestWatchDeliversEveryQueuedAlertAfterAnOutage(t *testing.T) {
 		assert.Contains(t, n.msgs[i], strconv.Quote(name))
 	}
 }
+
+// PR #160 round 2: dropping after N failures could not tell a poison message
+// from an outage, so a 15-minute outage silently lost the only incident. A
+// message is only dropped when it keeps failing while others send; through
+// an outage nothing is dropped, however long.
+func TestWatchLosesNoAlertThroughALongOutage(t *testing.T) {
+	brain := t.TempDir()
+	n := &toggleNotifier{}
+	n.down.Store(true)
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+	go Watch(ctx, brain, 2*time.Millisecond, n)
+	link := filepath.Join(brain, "only-incident")
+	require.NoError(t, os.Symlink(t.TempDir(), link))
+	require.Eventually(t, func() bool { _, err := os.Lstat(link); return os.IsNotExist(err) }, 2*time.Second, 2*time.Millisecond)
+	time.Sleep(200 * time.Millisecond) // ~100 ticks of outage, far past any attempt limit
+	n.down.Store(false)
+	require.Eventually(t, func() bool { return n.count() >= 1 }, 2*time.Second, 5*time.Millisecond)
+	n.mu.Lock()
+	defer n.mu.Unlock()
+	assert.Contains(t, n.msgs[0], strconv.Quote("only-incident"))
+}
+
+// Every removal is either delivered or counted in the suppression summary:
+// none disappears from the record (round 2: 74 of 300 were in neither).
+func TestWatchAccountsForEveryRemovalDuringAnOutage(t *testing.T) {
+	brain := t.TempDir()
+	n := &toggleNotifier{}
+	n.down.Store(true)
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+	go Watch(ctx, brain, 2*time.Millisecond, n)
+	const plants = 25
+	for i := 0; i < plants; i++ {
+		link := filepath.Join(brain, "acct"+strconv.Itoa(i))
+		require.NoError(t, os.Symlink(t.TempDir(), link))
+		require.Eventually(t, func() bool { _, err := os.Lstat(link); return os.IsNotExist(err) }, 2*time.Second, 1*time.Millisecond)
+	}
+	n.down.Store(false)
+	require.Eventually(t, func() bool {
+		n.mu.Lock()
+		defer n.mu.Unlock()
+		for _, m := range n.msgs {
+			if strings.Contains(m, "suppressed") {
+				return true
+			}
+		}
+		return false
+	}, 2*time.Second, 5*time.Millisecond)
+	time.Sleep(50 * time.Millisecond)
+	n.mu.Lock()
+	defer n.mu.Unlock()
+	delivered, counted := 0, 0
+	for _, m := range n.msgs {
+		if strings.Contains(m, "removed 1 symlink(s)") {
+			delivered++
+		}
+		if i := strings.Index(m, "): "); i >= 0 && strings.Contains(m, "suppressed") {
+			_, _ = fmt.Sscanf(m[i+3:], "%d", &counted)
+		}
+	}
+	assert.Equal(t, plants, delivered+counted, "delivered %d + counted %d must equal %d removals", delivered, counted, plants)
+}
+
+// A poison message is still dropped (others send while it fails), and the
+// drop is counted in the summary.
+func TestWatchDropsAPoisonMessageAndCountsIt(t *testing.T) {
+	brain := t.TempDir()
+	n := &rejectNotifier{reject: "POISON"}
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+	go Watch(ctx, brain, 5*time.Millisecond, n)
+	require.NoError(t, os.Symlink(t.TempDir(), filepath.Join(brain, "POISON")))
+	time.Sleep(20 * time.Millisecond)
+	require.NoError(t, os.Symlink(t.TempDir(), filepath.Join(brain, "fine")))
+	require.Eventually(t, func() bool {
+		n.mu.Lock()
+		defer n.mu.Unlock()
+		for _, m := range n.msgs {
+			if strings.Contains(m, "1 further alert(s)") {
+				return true
+			}
+		}
+		return false
+	}, 3*time.Second, 5*time.Millisecond, "the dropped poison alert is counted in a summary")
+}