feat(ingest): route DOCX/XLSX/PPTX/images through docmark (ADR-0013)
Wires the sovereign docmark MCP server (git.d-ma.be/mathias/docmark) as the converter for formats ingestion previously couldn't handle at all -- DOCX, XLSX, PPTX, PNG/JPG. A single self-contained tools/call POST (docmark runs stateless_http, no session handshake needed) does the conversion. - internal/extract/docmark.go: extractViaDocmark(path) -- reads DOCMARK_URL + DOCMARK_BEARER_TOKEN from env, base64-encodes the file, calls docmark's convert_to_markdown tool, surfaces tool/HTTP errors with context. - internal/extract/extract.go: routes .docx/.xlsx/.pptx/.png/.jpg/.jpeg to it. - internal/api/handler.go: isSupportedExtension() replaces the static supportedExtensions map -- the new extensions are gated on DOCMARK_URL being set, checked per-request. When DOCMARK_URL is unset (e.g. this repo's current deployed state), behavior is byte-for-byte unchanged from today: same 400/silent-skip as before. Zero risk until the env var is actually wired in infra. TDD throughout: docmark.go tested via httptest mock (success, not-configured, tool-error, HTTP-error, all 6 new extensions route correctly); handler-level test proves the DOCMARK_URL gate (docx 400 when unset, 200 when a working mock is configured). Full task check green (fmt/vet/govulncheck/all tests). Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
Claude Opus 4.8
@@ -0,0 +1,181 @@
|
||||
// ingestion/internal/extract/docmark_test.go
|
||||
package extract
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// mcpToolResult mirrors the shape of docmark's JSON-RPC tools/call response.
|
||||
type mcpToolResult struct {
|
||||
JSONRPC string `json:"jsonrpc"`
|
||||
ID int `json:"id"`
|
||||
Result *struct {
|
||||
Content []struct {
|
||||
Type string `json:"type"`
|
||||
Text string `json:"text"`
|
||||
} `json:"content"`
|
||||
IsError bool `json:"isError"`
|
||||
} `json:"result,omitempty"`
|
||||
Error *struct {
|
||||
Message string `json:"message"`
|
||||
} `json:"error,omitempty"`
|
||||
}
|
||||
|
||||
func writeMCPResponse(w http.ResponseWriter, body mcpToolResult) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_ = json.NewEncoder(w).Encode(body)
|
||||
}
|
||||
|
||||
func TestExtractViaDocmark_Success(t *testing.T) {
|
||||
var gotAuth, gotAccept string
|
||||
var gotBody map[string]any
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
gotAuth = r.Header.Get("Authorization")
|
||||
gotAccept = r.Header.Get("Accept")
|
||||
b, _ := io.ReadAll(r.Body)
|
||||
_ = json.Unmarshal(b, &gotBody)
|
||||
writeMCPResponse(w, mcpToolResult{
|
||||
JSONRPC: "2.0", ID: 1,
|
||||
Result: &struct {
|
||||
Content []struct {
|
||||
Type string `json:"type"`
|
||||
Text string `json:"text"`
|
||||
} `json:"content"`
|
||||
IsError bool `json:"isError"`
|
||||
}{
|
||||
Content: []struct {
|
||||
Type string `json:"type"`
|
||||
Text string `json:"text"`
|
||||
}{{Type: "text", Text: "# Converted\n\nhello"}},
|
||||
},
|
||||
})
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
t.Setenv("DOCMARK_URL", srv.URL+"/mcp")
|
||||
t.Setenv("DOCMARK_BEARER_TOKEN", "test-token-123")
|
||||
|
||||
dir := t.TempDir()
|
||||
path := filepath.Join(dir, "doc.docx")
|
||||
require.NoError(t, os.WriteFile(path, []byte("fake docx bytes"), 0o644))
|
||||
|
||||
got, err := extractViaDocmark(path)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "# Converted\n\nhello", got)
|
||||
assert.Equal(t, "Bearer test-token-123", gotAuth)
|
||||
assert.Contains(t, gotAccept, "application/json")
|
||||
params, _ := gotBody["params"].(map[string]any)
|
||||
require.NotNil(t, params)
|
||||
assert.Equal(t, "convert_to_markdown", params["name"])
|
||||
args, _ := params["arguments"].(map[string]any)
|
||||
require.NotNil(t, args)
|
||||
assert.Equal(t, "doc.docx", args["filename"])
|
||||
assert.NotEmpty(t, args["content_base64"])
|
||||
}
|
||||
|
||||
func TestExtractViaDocmark_NotConfigured(t *testing.T) {
|
||||
t.Setenv("DOCMARK_URL", "")
|
||||
|
||||
dir := t.TempDir()
|
||||
path := filepath.Join(dir, "doc.docx")
|
||||
require.NoError(t, os.WriteFile(path, []byte("x"), 0o644))
|
||||
|
||||
_, err := extractViaDocmark(path)
|
||||
require.Error(t, err)
|
||||
assert.Contains(t, err.Error(), "DOCMARK_URL")
|
||||
}
|
||||
|
||||
func TestExtractViaDocmark_ToolErrorSurfacesMessage(t *testing.T) {
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
writeMCPResponse(w, mcpToolResult{
|
||||
JSONRPC: "2.0", ID: 1,
|
||||
Result: &struct {
|
||||
Content []struct {
|
||||
Type string `json:"type"`
|
||||
Text string `json:"text"`
|
||||
} `json:"content"`
|
||||
IsError bool `json:"isError"`
|
||||
}{
|
||||
Content: []struct {
|
||||
Type string `json:"type"`
|
||||
Text string `json:"text"`
|
||||
}{{Type: "text", Text: "unsupported format for 'doc.docx'"}},
|
||||
IsError: true,
|
||||
},
|
||||
})
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
t.Setenv("DOCMARK_URL", srv.URL+"/mcp")
|
||||
t.Setenv("DOCMARK_BEARER_TOKEN", "tok")
|
||||
|
||||
dir := t.TempDir()
|
||||
path := filepath.Join(dir, "doc.docx")
|
||||
require.NoError(t, os.WriteFile(path, []byte("x"), 0o644))
|
||||
|
||||
_, err := extractViaDocmark(path)
|
||||
require.Error(t, err)
|
||||
assert.Contains(t, err.Error(), "unsupported format")
|
||||
}
|
||||
|
||||
func TestExtractViaDocmark_HTTPErrorSurfaces(t *testing.T) {
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.WriteHeader(http.StatusUnauthorized)
|
||||
_, _ = w.Write([]byte("unauthorized"))
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
t.Setenv("DOCMARK_URL", srv.URL+"/mcp")
|
||||
t.Setenv("DOCMARK_BEARER_TOKEN", "wrong")
|
||||
|
||||
dir := t.TempDir()
|
||||
path := filepath.Join(dir, "doc.docx")
|
||||
require.NoError(t, os.WriteFile(path, []byte("x"), 0o644))
|
||||
|
||||
_, err := extractViaDocmark(path)
|
||||
require.Error(t, err)
|
||||
assert.Contains(t, err.Error(), "401")
|
||||
}
|
||||
|
||||
func TestText_RoutesDocxXlsxPptxImagesToDocmark(t *testing.T) {
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
writeMCPResponse(w, mcpToolResult{
|
||||
JSONRPC: "2.0", ID: 1,
|
||||
Result: &struct {
|
||||
Content []struct {
|
||||
Type string `json:"type"`
|
||||
Text string `json:"text"`
|
||||
} `json:"content"`
|
||||
IsError bool `json:"isError"`
|
||||
}{
|
||||
Content: []struct {
|
||||
Type string `json:"type"`
|
||||
Text string `json:"text"`
|
||||
}{{Type: "text", Text: "converted"}},
|
||||
},
|
||||
})
|
||||
}))
|
||||
defer srv.Close()
|
||||
t.Setenv("DOCMARK_URL", srv.URL+"/mcp")
|
||||
t.Setenv("DOCMARK_BEARER_TOKEN", "tok")
|
||||
|
||||
for _, ext := range []string{".docx", ".xlsx", ".pptx", ".png", ".jpg", ".jpeg"} {
|
||||
t.Run(ext, func(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
path := filepath.Join(dir, "f"+ext)
|
||||
require.NoError(t, os.WriteFile(path, []byte("x"), 0o644))
|
||||
got, err := Text(path)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "converted", got)
|
||||
})
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user