feat(ingestion): clips land in web-clip by rule; shared concepts/entities; approve parked notes; remove the classifier (#208, #209) #211

Merged
mathias merged 16 commits from wt/web-clip into main 2026-10-20 19:36:51 +00:00
4 changed files with 65 additions and 39 deletions
Showing only changes of commit 8e488b6e4b - Show all commits
+11 -4
View File
@@ -53,11 +53,18 @@ func Run(ctx context.Context, cfg Config, brainDir, content, source string, dryR
var err error var err error
if cfg.Wing != "" { if cfg.Wing != "" {
// A source slug is a page's graph identity: refuse to take over a slug // A source slug is a page's graph identity: refuse to take over a slug
// that lives anywhere but this wing's own sources/. // that lives anywhere but this wing's own sources/ — including when
if existing, found, ferr := wiki.FindSlug(brainDir, wiki.Slug(source)); ferr != nil { // this wing's own page of that name also exists.
slug := wiki.Slug(source)
own := "wiki/" + cfg.Wing + "/sources/" + slug + ".md"
existing, ferr := wiki.FindSlugs(brainDir, slug)
if ferr != nil {
return Result{}, fmt.Errorf("check source slug: %w", ferr) return Result{}, fmt.Errorf("check source slug: %w", ferr)
} else if own := "wiki/" + cfg.Wing + "/sources/" + wiki.Slug(source) + ".md"; found && existing != own { }
return Result{}, fmt.Errorf("source slug %q is already used by %s", wiki.Slug(source), existing) for _, p := range existing {
if p != own {
return Result{}, fmt.Errorf("source slug %q is already used by %s", slug, p)
}
} }
inventory, err = wiki.LoadWingInventory(brainDir, cfg.Wing) inventory, err = wiki.LoadWingInventory(brainDir, cfg.Wing)
} else { } else {
@@ -140,3 +140,17 @@ func listAll(t *testing.T, dir string) []string {
_ = filepath.WalkDir(dir, func(p string, _ os.DirEntry, _ error) error { out = append(out, p); return nil }) _ = filepath.WalkDir(dir, func(p string, _ os.DirEntry, _ error) error { out = append(out, p); return nil })
return out return out
} }
// The first match must not hide a second one: this wing's own page of the name
// exists AND a page of the same name lives elsewhere.
func TestRun_ASlugUsedInTwoPlacesIsRefusedEvenWhenTheOwnPageIsOneOfThem(t *testing.T) {
dir := t.TempDir()
require.NoError(t, os.MkdirAll(filepath.Join(dir, "wiki/web-clip/sources"), 0o755))
require.NoError(t, os.WriteFile(filepath.Join(dir, "wiki/web-clip/sources/talk.md"), []byte("---\ntitle: Talk\n---\nown"), 0o644))
require.NoError(t, os.MkdirAll(filepath.Join(dir, "wiki/entities"), 0o755))
require.NoError(t, os.WriteFile(filepath.Join(dir, "wiki/entities/talk.md"), []byte("---\ntitle: Talk\n---\nan entity"), 0o644))
_, err := Run(context.Background(), wingCfg("web-clip", sourceEntityConcept()...), dir, "body", "talk", false)
require.Error(t, err)
assert.Contains(t, err.Error(), "already used")
}
+17 -13
View File
@@ -16,37 +16,41 @@ func put(t *testing.T, dir, rel string) {
require.NoError(t, os.WriteFile(full, []byte("---\ntitle: x\n---\n"), 0o644)) require.NoError(t, os.WriteFile(full, []byte("---\ntitle: x\n---\n"), 0o644))
} }
func TestFindSlug(t *testing.T) { func TestFindSlugs(t *testing.T) {
dir := t.TempDir() dir := t.TempDir()
put(t, dir, "wiki/sources/legacy-src.md") put(t, dir, "wiki/sources/legacy-src.md")
put(t, dir, "wiki/concepts/a-concept.md") put(t, dir, "wiki/concepts/a-concept.md")
put(t, dir, "wiki/entities/an-entity.md") put(t, dir, "wiki/entities/an-entity.md")
put(t, dir, "wiki/homelab/failures/in-a-hall.md") put(t, dir, "wiki/homelab/failures/in-a-hall.md")
put(t, dir, "wiki/web-clip/sources/a-clip.md") put(t, dir, "wiki/web-clip/sources/a-clip.md")
// One slug in two places: the first hit must not hide the second.
put(t, dir, "wiki/sources/shared-slug.md")
put(t, dir, "wiki/entities/shared-slug.md")
put(t, dir, "wiki/aaa/facts/shared-slug.md")
tests := []struct { tests := []struct {
slug string slug string
want string want []string
}{ }{
{"legacy-src", "wiki/sources/legacy-src.md"}, {"legacy-src", []string{"wiki/sources/legacy-src.md"}},
{"a-concept", "wiki/concepts/a-concept.md"}, {"a-concept", []string{"wiki/concepts/a-concept.md"}},
{"an-entity", "wiki/entities/an-entity.md"}, {"an-entity", []string{"wiki/entities/an-entity.md"}},
{"in-a-hall", "wiki/homelab/failures/in-a-hall.md"}, {"in-a-hall", []string{"wiki/homelab/failures/in-a-hall.md"}},
{"a-clip", "wiki/web-clip/sources/a-clip.md"}, {"a-clip", []string{"wiki/web-clip/sources/a-clip.md"}},
{"nothing-here", ""}, {"shared-slug", []string{"wiki/aaa/facts/shared-slug.md", "wiki/entities/shared-slug.md", "wiki/sources/shared-slug.md"}},
{"nothing-here", nil},
} }
for _, tt := range tests { for _, tt := range tests {
t.Run(tt.slug, func(t *testing.T) { t.Run(tt.slug, func(t *testing.T) {
got, found, err := FindSlug(dir, tt.slug) got, err := FindSlugs(dir, tt.slug)
require.NoError(t, err) require.NoError(t, err)
assert.Equal(t, tt.want != "", found)
assert.Equal(t, tt.want, got) assert.Equal(t, tt.want, got)
}) })
} }
} }
func TestFindSlug_NoWikiDirIsNotAFinding(t *testing.T) { func TestFindSlugs_NoWikiDirFindsNothing(t *testing.T) {
_, found, err := FindSlug(t.TempDir(), "x") got, err := FindSlugs(t.TempDir(), "x")
require.NoError(t, err) require.NoError(t, err)
assert.False(t, found) assert.Empty(t, got)
} }
+23 -22
View File
@@ -7,6 +7,7 @@ import (
"errors" "errors"
"fmt" "fmt"
"io/fs" "io/fs"
"sort"
"strings" "strings"
"github.com/mathiasbq/hyperguild/ingestion/internal/brainfs" "github.com/mathiasbq/hyperguild/ingestion/internal/brainfs"
@@ -133,45 +134,46 @@ func LoadWingInventory(brainDir, wing string) (map[PageType][]Entry, error) {
return result, nil return result, nil
} }
// FindSlug returns the brain-relative path of an existing page whose basename is // FindSlugs returns the brain-relative path of EVERY existing page whose
// slug, searching the retired dirs (wiki/{sources,concepts,entities}/) and every // basename is slug, searching the retired dirs (wiki/{sources,concepts,
// wing hall (wiki/<wing>/<hall>/). The slug is a page's graph identity, so two // entities}/) and every wing hall (wiki/<wing>/<hall>/), sorted. The slug is a
// pages must not share one. // page's graph identity, so two pages must not share one; returning all of
func FindSlug(brainDir, slug string) (string, bool, error) { // them (not the first) is what lets a caller tell "only my own page" from "my
// own page and somebody else's".
func FindSlugs(brainDir, slug string) ([]string, error) {
bfs, err := brainfs.Open(brainDir) bfs, err := brainfs.Open(brainDir)
if err != nil { if err != nil {
return "", false, fmt.Errorf("open brain: %w", err) return nil, fmt.Errorf("open brain: %w", err)
} }
defer func() { _ = bfs.Close() }() defer func() { _ = bfs.Close() }()
file := slug + ".md" file := slug + ".md"
has := func(dir string) (bool, error) { var found []string
check := func(dir string) error {
names, err := bfs.ReadDirRegular(dir) names, err := bfs.ReadDirRegular(dir)
if errors.Is(err, fs.ErrNotExist) { if errors.Is(err, fs.ErrNotExist) {
return false, nil return nil
} }
if err != nil { if err != nil {
return false, err return err
} }
for _, n := range names { for _, n := range names {
if n == file { if n == file {
return true, nil found = append(found, dir+"/"+file)
} }
} }
return false, nil return nil
} }
for _, d := range []string{"wiki/sources", "wiki/concepts", "wiki/entities"} { for _, d := range []string{"wiki/sources", "wiki/concepts", "wiki/entities"} {
if ok, err := has(d); err != nil { if err := check(d); err != nil {
return "", false, err return nil, err
} else if ok {
return d + "/" + file, true, nil
} }
} }
wings, err := bfs.ReadDirDirs("wiki") wings, err := bfs.ReadDirDirs("wiki")
if errors.Is(err, fs.ErrNotExist) { if errors.Is(err, fs.ErrNotExist) {
return "", false, nil return found, nil
} }
if err != nil { if err != nil {
return "", false, err return nil, err
} }
for _, w := range wings { for _, w := range wings {
if w == "sources" || w == "concepts" || w == "entities" { if w == "sources" || w == "concepts" || w == "entities" {
@@ -182,12 +184,11 @@ func FindSlug(brainDir, slug string) (string, bool, error) {
continue continue
} }
for _, h := range halls { for _, h := range halls {
if ok, err := has("wiki/" + w + "/" + h); err != nil { if err := check("wiki/" + w + "/" + h); err != nil {
return "", false, err return nil, err
} else if ok {
return "wiki/" + w + "/" + h + "/" + file, true, nil
} }
} }
} }
return "", false, nil sort.Strings(found)
return found, nil
} }