mathiasandClaude Opus 5.5
2f3422adf2
The remaining writers created or rewrote brain files by path: - pipeline buildAndWrite: os.MkdirAll + os.WriteFile of every page. - BackfillRefs: os.WriteFile of each concept page; a page swapped for a symlink after the read was written through (the race test overwrote an in-brain target before this change). - watcher: the raw/processed and raw/failed archive copies (os.Create), the .processed/.failed markers and the log.md entry (os.OpenFile). - cmd/server: the claude-session archive. All now go through brainfs PutRegular/AppendRegular (new package-level PutFile/AppendFile for one-off writers: the watcher's call-site lint bans .Open calls in that file so raw/ reads stay on OpenRegular). Stacked on #189 (needs PutRegular/AppendRegular). Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
98 lines
2.6 KiB
Go
98 lines
2.6 KiB
Go
// ingestion/internal/pipeline/backfill.go
|
|
package pipeline
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"fmt"
|
|
"github.com/mathiasbq/hyperguild/ingestion/internal/brainfs"
|
|
"io/fs"
|
|
"strings"
|
|
|
|
"github.com/mathiasbq/hyperguild/ingestion/internal/wiki"
|
|
)
|
|
|
|
// BackfillRefs walks wiki/sources/ and injects source back-references into every
|
|
// concept and entity page that each source links to.
|
|
// Changes for all sources are accumulated in memory before writing, so multiple
|
|
// sources referencing the same concept are merged in one pass.
|
|
// Deduplication is handled by wiki.Merge — running this multiple times is safe.
|
|
// Returns the number of concept/entity pages written.
|
|
func BackfillRefs(ctx context.Context, brainDir string) (int, error) {
|
|
inventory, err := wiki.LoadInventory(brainDir)
|
|
if err != nil {
|
|
return 0, fmt.Errorf("load inventory: %w", err)
|
|
}
|
|
|
|
// Listed and read through the brain root, regular files only
|
|
// (hyperguild#157).
|
|
bfs, err := brainfs.Open(brainDir)
|
|
if err != nil {
|
|
return 0, fmt.Errorf("open brain: %w", err)
|
|
}
|
|
defer func() { _ = bfs.Close() }()
|
|
names, err := bfs.ReadDirRegular("wiki/sources")
|
|
if err != nil {
|
|
if errors.Is(err, fs.ErrNotExist) {
|
|
return 0, nil
|
|
}
|
|
return 0, fmt.Errorf("read sources dir: %w", err)
|
|
}
|
|
|
|
// Accumulate all changes before writing: relPath → updated Page.
|
|
// Collecting first means two sources that both link the same concept
|
|
// get both refs merged before a single write.
|
|
pending := make(map[string]wiki.Page)
|
|
|
|
for _, name := range names {
|
|
if ctx.Err() != nil {
|
|
return 0, ctx.Err()
|
|
}
|
|
if !strings.HasSuffix(name, ".md") {
|
|
continue
|
|
}
|
|
|
|
b, err := bfs.ReadRegular("wiki/sources/" + name)
|
|
if err != nil {
|
|
continue
|
|
}
|
|
sourceContent := string(b)
|
|
sourceSlug := strings.TrimSuffix(name, ".md")
|
|
sourceTitle := extractTitle(sourceContent)
|
|
if sourceTitle == "" {
|
|
sourceTitle = sourceSlug
|
|
}
|
|
sourceRef := "- [[" + sourceSlug + "|" + sourceTitle + "]]"
|
|
|
|
for slug := range extractWikilinks(sourceContent) {
|
|
if slug == sourceSlug {
|
|
continue
|
|
}
|
|
pt, ok := findInInventory(slug, inventory)
|
|
if !ok {
|
|
continue
|
|
}
|
|
relPath := "wiki/" + string(pt) + "/" + slug + ".md"
|
|
|
|
// Start from already-accumulated version if we've seen this page.
|
|
page, seen := pending[relPath]
|
|
if !seen {
|
|
raw, err := bfs.ReadRegular(relPath)
|
|
if err != nil {
|
|
continue
|
|
}
|
|
page = wiki.Page{Path: relPath, Content: string(raw)}
|
|
}
|
|
pending[relPath] = addSourceRef(page, sourceRef)
|
|
}
|
|
}
|
|
|
|
for relPath, page := range pending {
|
|
if err := bfs.PutRegular(relPath, []byte(page.Content)); err != nil {
|
|
return 0, fmt.Errorf("write %s: %w", relPath, err)
|
|
}
|
|
}
|
|
|
|
return len(pending), nil
|
|
}
|