feat(ingest): route DOCX/XLSX/PPTX/images through docmark (ADR-0013)
Wires the sovereign docmark MCP server (git.d-ma.be/mathias/docmark) as the converter for formats ingestion previously couldn't handle at all -- DOCX, XLSX, PPTX, PNG/JPG. A single self-contained tools/call POST (docmark runs stateless_http, no session handshake needed) does the conversion. - internal/extract/docmark.go: extractViaDocmark(path) -- reads DOCMARK_URL + DOCMARK_BEARER_TOKEN from env, base64-encodes the file, calls docmark's convert_to_markdown tool, surfaces tool/HTTP errors with context. - internal/extract/extract.go: routes .docx/.xlsx/.pptx/.png/.jpg/.jpeg to it. - internal/api/handler.go: isSupportedExtension() replaces the static supportedExtensions map -- the new extensions are gated on DOCMARK_URL being set, checked per-request. When DOCMARK_URL is unset (e.g. this repo's current deployed state), behavior is byte-for-byte unchanged from today: same 400/silent-skip as before. Zero risk until the env var is actually wired in infra. TDD throughout: docmark.go tested via httptest mock (success, not-configured, tool-error, HTTP-error, all 6 new extensions route correctly); handler-level test proves the DOCMARK_URL gate (docx 400 when unset, 200 when a working mock is configured). Full task check green (fmt/vet/govulncheck/all tests). Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,126 @@
|
||||
// ingestion/internal/extract/docmark.go
|
||||
package extract
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"os"
|
||||
"time"
|
||||
)
|
||||
|
||||
// docmarkRequest is a JSON-RPC 2.0 tools/call request for docmark's
|
||||
// convert_to_markdown tool.
|
||||
type docmarkRequest struct {
|
||||
JSONRPC string `json:"jsonrpc"`
|
||||
ID int `json:"id"`
|
||||
Method string `json:"method"`
|
||||
Params struct {
|
||||
Name string `json:"name"`
|
||||
Arguments struct {
|
||||
ContentBase64 string `json:"content_base64"`
|
||||
Filename string `json:"filename"`
|
||||
} `json:"arguments"`
|
||||
} `json:"params"`
|
||||
}
|
||||
|
||||
type docmarkResponse struct {
|
||||
Result *struct {
|
||||
Content []struct {
|
||||
Type string `json:"type"`
|
||||
Text string `json:"text"`
|
||||
} `json:"content"`
|
||||
IsError bool `json:"isError"`
|
||||
} `json:"result"`
|
||||
Error *struct {
|
||||
Message string `json:"message"`
|
||||
} `json:"error"`
|
||||
}
|
||||
|
||||
// extractViaDocmark converts path (PDF/DOCX/XLSX/PPTX/image) to Markdown by
|
||||
// calling the docmark MCP server with a single self-contained tools/call
|
||||
// request (docmark runs stateless_http -- no initialize handshake or session
|
||||
// ID needed). DOCMARK_URL must be set (e.g.
|
||||
// http://docmark.docmark.svc.cluster.local:3001/mcp); DOCMARK_BEARER_TOKEN
|
||||
// is docmark's static bearer (network is docmark's primary auth boundary,
|
||||
// this is defense-in-depth — ADR-0013).
|
||||
func extractViaDocmark(path string) (string, error) {
|
||||
url := os.Getenv("DOCMARK_URL")
|
||||
if url == "" {
|
||||
return "", fmt.Errorf("extractViaDocmark: DOCMARK_URL is not set")
|
||||
}
|
||||
|
||||
raw, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("read %s: %w", path, err)
|
||||
}
|
||||
|
||||
var reqBody docmarkRequest
|
||||
reqBody.JSONRPC = "2.0"
|
||||
reqBody.ID = 1
|
||||
reqBody.Method = "tools/call"
|
||||
reqBody.Params.Name = "convert_to_markdown"
|
||||
reqBody.Params.Arguments.ContentBase64 = base64.StdEncoding.EncodeToString(raw)
|
||||
reqBody.Params.Arguments.Filename = fileBase(path)
|
||||
|
||||
payload, err := json.Marshal(reqBody)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("marshal docmark request: %w", err)
|
||||
}
|
||||
|
||||
httpReq, err := http.NewRequest(http.MethodPost, url, bytes.NewReader(payload))
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("build docmark request: %w", err)
|
||||
}
|
||||
httpReq.Header.Set("Content-Type", "application/json")
|
||||
httpReq.Header.Set("Accept", "application/json, text/event-stream")
|
||||
if tok := os.Getenv("DOCMARK_BEARER_TOKEN"); tok != "" {
|
||||
httpReq.Header.Set("Authorization", "Bearer "+tok)
|
||||
}
|
||||
|
||||
client := &http.Client{Timeout: 60 * time.Second}
|
||||
resp, err := client.Do(httpReq)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("call docmark: %w", err)
|
||||
}
|
||||
defer func() { _ = resp.Body.Close() }()
|
||||
|
||||
body, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("read docmark response: %w", err)
|
||||
}
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return "", fmt.Errorf("docmark: HTTP %d: %s", resp.StatusCode, string(body))
|
||||
}
|
||||
|
||||
var out docmarkResponse
|
||||
if err := json.Unmarshal(body, &out); err != nil {
|
||||
return "", fmt.Errorf("decode docmark response: %w", err)
|
||||
}
|
||||
if out.Error != nil {
|
||||
return "", fmt.Errorf("docmark: %s", out.Error.Message)
|
||||
}
|
||||
if out.Result == nil || len(out.Result.Content) == 0 {
|
||||
return "", fmt.Errorf("docmark: empty response")
|
||||
}
|
||||
text := out.Result.Content[0].Text
|
||||
if out.Result.IsError {
|
||||
return "", fmt.Errorf("docmark: %s", text)
|
||||
}
|
||||
return text, nil
|
||||
}
|
||||
|
||||
// fileBase returns the final path segment (like filepath.Base, kept local to
|
||||
// avoid importing path/filepath just for this one call).
|
||||
func fileBase(path string) string {
|
||||
for i := len(path) - 1; i >= 0; i-- {
|
||||
if path[i] == '/' || path[i] == '\\' {
|
||||
return path[i+1:]
|
||||
}
|
||||
}
|
||||
return path
|
||||
}
|
||||
Reference in New Issue
Block a user