feat(ingest): route DOCX/XLSX/PPTX/images through docmark (ADR-0013)
Wires the sovereign docmark MCP server (git.d-ma.be/mathias/docmark) as the converter for formats ingestion previously couldn't handle at all -- DOCX, XLSX, PPTX, PNG/JPG. A single self-contained tools/call POST (docmark runs stateless_http, no session handshake needed) does the conversion. - internal/extract/docmark.go: extractViaDocmark(path) -- reads DOCMARK_URL + DOCMARK_BEARER_TOKEN from env, base64-encodes the file, calls docmark's convert_to_markdown tool, surfaces tool/HTTP errors with context. - internal/extract/extract.go: routes .docx/.xlsx/.pptx/.png/.jpg/.jpeg to it. - internal/api/handler.go: isSupportedExtension() replaces the static supportedExtensions map -- the new extensions are gated on DOCMARK_URL being set, checked per-request. When DOCMARK_URL is unset (e.g. this repo's current deployed state), behavior is byte-for-byte unchanged from today: same 400/silent-skip as before. Zero risk until the env var is actually wired in infra. TDD throughout: docmark.go tested via httptest mock (success, not-configured, tool-error, HTTP-error, all 6 new extensions route correctly); handler-level test proves the DOCMARK_URL gate (docx 400 when unset, 200 when a working mock is configured). Full task check green (fmt/vet/govulncheck/all tests). Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -345,11 +345,19 @@ func (h *Handler) Ingest(w http.ResponseWriter, r *http.Request) {
|
||||
writeJSON(w, ingestResponse{Pages: pages, Warnings: warnings})
|
||||
}
|
||||
|
||||
// supportedExtensions lists file extensions that IngestPath will process.
|
||||
var supportedExtensions = map[string]bool{
|
||||
".md": true,
|
||||
".txt": true,
|
||||
".pdf": true,
|
||||
// isSupportedExtension reports whether IngestPath will process ext.
|
||||
// .docx/.xlsx/.pptx/.png/.jpg/.jpeg require docmark (ADR-0013) and are only
|
||||
// supported when DOCMARK_URL is configured — checked per-call (not cached at
|
||||
// package init) so it reflects the environment at request time.
|
||||
func isSupportedExtension(ext string) bool {
|
||||
switch ext {
|
||||
case ".md", ".txt", ".pdf":
|
||||
return true
|
||||
case ".docx", ".xlsx", ".pptx", ".png", ".jpg", ".jpeg":
|
||||
return os.Getenv("DOCMARK_URL") != ""
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
// IngestPath handles POST /ingest-path — ingest a file or directory.
|
||||
@@ -382,7 +390,7 @@ func (h *Handler) IngestPath(w http.ResponseWriter, r *http.Request) {
|
||||
return nil
|
||||
}
|
||||
ext := strings.ToLower(filepath.Ext(path))
|
||||
if !supportedExtensions[ext] {
|
||||
if !isSupportedExtension(ext) {
|
||||
return nil
|
||||
}
|
||||
content, readErr := extract.Text(path)
|
||||
@@ -410,7 +418,7 @@ func (h *Handler) IngestPath(w http.ResponseWriter, r *http.Request) {
|
||||
}
|
||||
} else {
|
||||
ext := strings.ToLower(filepath.Ext(req.Path))
|
||||
if !supportedExtensions[ext] {
|
||||
if !isSupportedExtension(ext) {
|
||||
writeError(w, http.StatusBadRequest, fmt.Sprintf("unsupported file extension: %s", ext))
|
||||
return
|
||||
}
|
||||
|
||||
@@ -0,0 +1,61 @@
|
||||
// ingestion/internal/api/ingestpath_docmark_test.go
|
||||
package api_test
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestIngestPath_DocxUnsupportedWhenDocmarkNotConfigured(t *testing.T) {
|
||||
t.Setenv("DOCMARK_URL", "")
|
||||
_, h := setup(t)
|
||||
|
||||
dir := t.TempDir()
|
||||
f := filepath.Join(dir, "doc.docx")
|
||||
require.NoError(t, os.WriteFile(f, []byte("fake docx"), 0o644))
|
||||
|
||||
body, _ := json.Marshal(map[string]any{"path": f, "source": "test-doc", "dry_run": true})
|
||||
req := httptest.NewRequest(http.MethodPost, "/ingest-path", bytes.NewReader(body))
|
||||
rec := httptest.NewRecorder()
|
||||
|
||||
h.IngestPath(rec, req)
|
||||
|
||||
assert.Equal(t, http.StatusBadRequest, rec.Code, rec.Body.String())
|
||||
}
|
||||
|
||||
func TestIngestPath_DocxSupportedWhenDocmarkConfigured(t *testing.T) {
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_, _ = w.Write([]byte(`{"jsonrpc":"2.0","id":1,"result":{"content":[{"type":"text","text":"# Converted Doc"}]}}`))
|
||||
}))
|
||||
defer srv.Close()
|
||||
t.Setenv("DOCMARK_URL", srv.URL+"/mcp")
|
||||
t.Setenv("DOCMARK_BEARER_TOKEN", "tok")
|
||||
|
||||
_, h := setup(t)
|
||||
|
||||
dir := t.TempDir()
|
||||
f := filepath.Join(dir, "doc.docx")
|
||||
require.NoError(t, os.WriteFile(f, []byte("fake docx"), 0o644))
|
||||
|
||||
body, _ := json.Marshal(map[string]any{"path": f, "source": "test-doc", "dry_run": true})
|
||||
req := httptest.NewRequest(http.MethodPost, "/ingest-path", bytes.NewReader(body))
|
||||
rec := httptest.NewRecorder()
|
||||
|
||||
h.IngestPath(rec, req)
|
||||
|
||||
require.Equal(t, http.StatusOK, rec.Code, rec.Body.String())
|
||||
var resp map[string]any
|
||||
require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &resp))
|
||||
pages, ok := resp["pages"].([]any)
|
||||
require.True(t, ok)
|
||||
assert.NotEmpty(t, pages)
|
||||
}
|
||||
Reference in New Issue
Block a user