mathiasandClaude Opus 5
125c99bb46
Four call sites did `s[:limit] + "…"`. Both halves are byte operations: len() counts bytes and a slice cuts bytes. On Swedish text — å ä ö are two bytes, an em dash three — the cut lands mid-character and leaves an orphaned lead byte, producing invalid UTF-8. It failed loudly in exactly one place: wiki/index.md is written raw to disk, where every strict decoder then rejects the whole file. The other three (brain_context excerpts x2, brain_query excerpts) feed MCP responses, where encoding/json silently substitutes U+FFFD — so those had been quietly eating the last character of truncated excerpts, with no error anywhere and nothing to notice. textutil.Truncate counts runes and appends the marker only when content was actually dropped; an ellipsis on complete text is a lie about the content. The test is a property, not an example: it walks the limit across every offset of a multi-byte string and asserts utf8.Valid at each one, so a future off-by-one cannot pass by picking a lucky boundary. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01BeAp5LTscnz2W7Ubt4eJ6m
71 lines
1.8 KiB
Go
71 lines
1.8 KiB
Go
// ingestion/internal/wiki/index.go
|
|
package wiki
|
|
|
|
import (
|
|
"fmt"
|
|
"os"
|
|
"path/filepath"
|
|
"strings"
|
|
|
|
"github.com/mathiasbq/hyperguild/ingestion/internal/textutil"
|
|
)
|
|
|
|
// RebuildIndex writes brain/wiki/index.md from the current wiki contents.
|
|
func RebuildIndex(brainDir, date string) error {
|
|
inv, err := LoadInventory(brainDir)
|
|
if err != nil {
|
|
return fmt.Errorf("load inventory: %w", err)
|
|
}
|
|
|
|
total := len(inv[PageTypeConcept]) + len(inv[PageTypeEntity]) + len(inv[PageTypeSource])
|
|
var sb strings.Builder
|
|
fmt.Fprintf(&sb, "# Wiki Index\n\n")
|
|
fmt.Fprintf(&sb, "_Updated: %s — %d pages (%d concepts, %d entities, %d sources)_\n\n",
|
|
date, total,
|
|
len(inv[PageTypeConcept]),
|
|
len(inv[PageTypeEntity]),
|
|
len(inv[PageTypeSource]))
|
|
|
|
for _, pt := range []PageType{PageTypeConcept, PageTypeEntity, PageTypeSource} {
|
|
entries := inv[pt]
|
|
if len(entries) == 0 {
|
|
continue
|
|
}
|
|
label := strings.ToUpper(string(pt)[:1]) + string(pt)[1:]
|
|
fmt.Fprintf(&sb, "## %s\n\n", label)
|
|
for _, e := range entries {
|
|
summary := pageFirstSentence(brainDir, e)
|
|
if summary != "" {
|
|
fmt.Fprintf(&sb, "- [[%s|%s]] — %s\n", e.Slug, e.Title, summary)
|
|
} else {
|
|
fmt.Fprintf(&sb, "- [[%s|%s]]\n", e.Slug, e.Title)
|
|
}
|
|
}
|
|
sb.WriteString("\n")
|
|
}
|
|
|
|
dest := filepath.Join(brainDir, "wiki", "index.md")
|
|
return os.WriteFile(dest, []byte(sb.String()), 0o644)
|
|
}
|
|
|
|
func pageFirstSentence(brainDir string, e Entry) string {
|
|
path := filepath.Join(brainDir, "wiki", string(e.Type), e.Slug+".md")
|
|
content, err := os.ReadFile(path)
|
|
if err != nil {
|
|
return ""
|
|
}
|
|
parts := strings.SplitN(string(content), "---", 3)
|
|
body := string(content)
|
|
if len(parts) == 3 {
|
|
body = parts[2]
|
|
}
|
|
for _, line := range strings.Split(body, "\n") {
|
|
line = strings.TrimSpace(line)
|
|
if line == "" || strings.HasPrefix(line, "#") {
|
|
continue
|
|
}
|
|
return textutil.Truncate(line, 100)
|
|
}
|
|
return ""
|
|
}
|