diff --git a/ingestion/internal/api/handler.go b/ingestion/internal/api/handler.go index 7ed0fed..7af073f 100644 --- a/ingestion/internal/api/handler.go +++ b/ingestion/internal/api/handler.go @@ -345,11 +345,19 @@ func (h *Handler) Ingest(w http.ResponseWriter, r *http.Request) { writeJSON(w, ingestResponse{Pages: pages, Warnings: warnings}) } -// supportedExtensions lists file extensions that IngestPath will process. -var supportedExtensions = map[string]bool{ - ".md": true, - ".txt": true, - ".pdf": true, +// isSupportedExtension reports whether IngestPath will process ext. +// .docx/.xlsx/.pptx/.png/.jpg/.jpeg require docmark (ADR-0013) and are only +// supported when DOCMARK_URL is configured — checked per-call (not cached at +// package init) so it reflects the environment at request time. +func isSupportedExtension(ext string) bool { + switch ext { + case ".md", ".txt", ".pdf": + return true + case ".docx", ".xlsx", ".pptx", ".png", ".jpg", ".jpeg": + return os.Getenv("DOCMARK_URL") != "" + default: + return false + } } // IngestPath handles POST /ingest-path — ingest a file or directory. @@ -382,7 +390,7 @@ func (h *Handler) IngestPath(w http.ResponseWriter, r *http.Request) { return nil } ext := strings.ToLower(filepath.Ext(path)) - if !supportedExtensions[ext] { + if !isSupportedExtension(ext) { return nil } content, readErr := extract.Text(path) @@ -410,7 +418,7 @@ func (h *Handler) IngestPath(w http.ResponseWriter, r *http.Request) { } } else { ext := strings.ToLower(filepath.Ext(req.Path)) - if !supportedExtensions[ext] { + if !isSupportedExtension(ext) { writeError(w, http.StatusBadRequest, fmt.Sprintf("unsupported file extension: %s", ext)) return } diff --git a/ingestion/internal/api/ingestpath_docmark_test.go b/ingestion/internal/api/ingestpath_docmark_test.go new file mode 100644 index 0000000..40da3ac --- /dev/null +++ b/ingestion/internal/api/ingestpath_docmark_test.go @@ -0,0 +1,61 @@ +// ingestion/internal/api/ingestpath_docmark_test.go +package api_test + +import ( + "bytes" + "encoding/json" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestIngestPath_DocxUnsupportedWhenDocmarkNotConfigured(t *testing.T) { + t.Setenv("DOCMARK_URL", "") + _, h := setup(t) + + dir := t.TempDir() + f := filepath.Join(dir, "doc.docx") + require.NoError(t, os.WriteFile(f, []byte("fake docx"), 0o644)) + + body, _ := json.Marshal(map[string]any{"path": f, "source": "test-doc", "dry_run": true}) + req := httptest.NewRequest(http.MethodPost, "/ingest-path", bytes.NewReader(body)) + rec := httptest.NewRecorder() + + h.IngestPath(rec, req) + + assert.Equal(t, http.StatusBadRequest, rec.Code, rec.Body.String()) +} + +func TestIngestPath_DocxSupportedWhenDocmarkConfigured(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"jsonrpc":"2.0","id":1,"result":{"content":[{"type":"text","text":"# Converted Doc"}]}}`)) + })) + defer srv.Close() + t.Setenv("DOCMARK_URL", srv.URL+"/mcp") + t.Setenv("DOCMARK_BEARER_TOKEN", "tok") + + _, h := setup(t) + + dir := t.TempDir() + f := filepath.Join(dir, "doc.docx") + require.NoError(t, os.WriteFile(f, []byte("fake docx"), 0o644)) + + body, _ := json.Marshal(map[string]any{"path": f, "source": "test-doc", "dry_run": true}) + req := httptest.NewRequest(http.MethodPost, "/ingest-path", bytes.NewReader(body)) + rec := httptest.NewRecorder() + + h.IngestPath(rec, req) + + require.Equal(t, http.StatusOK, rec.Code, rec.Body.String()) + var resp map[string]any + require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &resp)) + pages, ok := resp["pages"].([]any) + require.True(t, ok) + assert.NotEmpty(t, pages) +} diff --git a/ingestion/internal/extract/docmark.go b/ingestion/internal/extract/docmark.go new file mode 100644 index 0000000..4281742 --- /dev/null +++ b/ingestion/internal/extract/docmark.go @@ -0,0 +1,126 @@ +// ingestion/internal/extract/docmark.go +package extract + +import ( + "bytes" + "encoding/base64" + "encoding/json" + "fmt" + "io" + "net/http" + "os" + "time" +) + +// docmarkRequest is a JSON-RPC 2.0 tools/call request for docmark's +// convert_to_markdown tool. +type docmarkRequest struct { + JSONRPC string `json:"jsonrpc"` + ID int `json:"id"` + Method string `json:"method"` + Params struct { + Name string `json:"name"` + Arguments struct { + ContentBase64 string `json:"content_base64"` + Filename string `json:"filename"` + } `json:"arguments"` + } `json:"params"` +} + +type docmarkResponse struct { + Result *struct { + Content []struct { + Type string `json:"type"` + Text string `json:"text"` + } `json:"content"` + IsError bool `json:"isError"` + } `json:"result"` + Error *struct { + Message string `json:"message"` + } `json:"error"` +} + +// extractViaDocmark converts path (PDF/DOCX/XLSX/PPTX/image) to Markdown by +// calling the docmark MCP server with a single self-contained tools/call +// request (docmark runs stateless_http -- no initialize handshake or session +// ID needed). DOCMARK_URL must be set (e.g. +// http://docmark.docmark.svc.cluster.local:3001/mcp); DOCMARK_BEARER_TOKEN +// is docmark's static bearer (network is docmark's primary auth boundary, +// this is defense-in-depth — ADR-0013). +func extractViaDocmark(path string) (string, error) { + url := os.Getenv("DOCMARK_URL") + if url == "" { + return "", fmt.Errorf("extractViaDocmark: DOCMARK_URL is not set") + } + + raw, err := os.ReadFile(path) + if err != nil { + return "", fmt.Errorf("read %s: %w", path, err) + } + + var reqBody docmarkRequest + reqBody.JSONRPC = "2.0" + reqBody.ID = 1 + reqBody.Method = "tools/call" + reqBody.Params.Name = "convert_to_markdown" + reqBody.Params.Arguments.ContentBase64 = base64.StdEncoding.EncodeToString(raw) + reqBody.Params.Arguments.Filename = fileBase(path) + + payload, err := json.Marshal(reqBody) + if err != nil { + return "", fmt.Errorf("marshal docmark request: %w", err) + } + + httpReq, err := http.NewRequest(http.MethodPost, url, bytes.NewReader(payload)) + if err != nil { + return "", fmt.Errorf("build docmark request: %w", err) + } + httpReq.Header.Set("Content-Type", "application/json") + httpReq.Header.Set("Accept", "application/json, text/event-stream") + if tok := os.Getenv("DOCMARK_BEARER_TOKEN"); tok != "" { + httpReq.Header.Set("Authorization", "Bearer "+tok) + } + + client := &http.Client{Timeout: 60 * time.Second} + resp, err := client.Do(httpReq) + if err != nil { + return "", fmt.Errorf("call docmark: %w", err) + } + defer func() { _ = resp.Body.Close() }() + + body, err := io.ReadAll(resp.Body) + if err != nil { + return "", fmt.Errorf("read docmark response: %w", err) + } + + if resp.StatusCode != http.StatusOK { + return "", fmt.Errorf("docmark: HTTP %d: %s", resp.StatusCode, string(body)) + } + + var out docmarkResponse + if err := json.Unmarshal(body, &out); err != nil { + return "", fmt.Errorf("decode docmark response: %w", err) + } + if out.Error != nil { + return "", fmt.Errorf("docmark: %s", out.Error.Message) + } + if out.Result == nil || len(out.Result.Content) == 0 { + return "", fmt.Errorf("docmark: empty response") + } + text := out.Result.Content[0].Text + if out.Result.IsError { + return "", fmt.Errorf("docmark: %s", text) + } + return text, nil +} + +// fileBase returns the final path segment (like filepath.Base, kept local to +// avoid importing path/filepath just for this one call). +func fileBase(path string) string { + for i := len(path) - 1; i >= 0; i-- { + if path[i] == '/' || path[i] == '\\' { + return path[i+1:] + } + } + return path +} diff --git a/ingestion/internal/extract/docmark_test.go b/ingestion/internal/extract/docmark_test.go new file mode 100644 index 0000000..7ae6ae5 --- /dev/null +++ b/ingestion/internal/extract/docmark_test.go @@ -0,0 +1,181 @@ +// ingestion/internal/extract/docmark_test.go +package extract + +import ( + "encoding/json" + "io" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// mcpToolResult mirrors the shape of docmark's JSON-RPC tools/call response. +type mcpToolResult struct { + JSONRPC string `json:"jsonrpc"` + ID int `json:"id"` + Result *struct { + Content []struct { + Type string `json:"type"` + Text string `json:"text"` + } `json:"content"` + IsError bool `json:"isError"` + } `json:"result,omitempty"` + Error *struct { + Message string `json:"message"` + } `json:"error,omitempty"` +} + +func writeMCPResponse(w http.ResponseWriter, body mcpToolResult) { + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(body) +} + +func TestExtractViaDocmark_Success(t *testing.T) { + var gotAuth, gotAccept string + var gotBody map[string]any + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + gotAuth = r.Header.Get("Authorization") + gotAccept = r.Header.Get("Accept") + b, _ := io.ReadAll(r.Body) + _ = json.Unmarshal(b, &gotBody) + writeMCPResponse(w, mcpToolResult{ + JSONRPC: "2.0", ID: 1, + Result: &struct { + Content []struct { + Type string `json:"type"` + Text string `json:"text"` + } `json:"content"` + IsError bool `json:"isError"` + }{ + Content: []struct { + Type string `json:"type"` + Text string `json:"text"` + }{{Type: "text", Text: "# Converted\n\nhello"}}, + }, + }) + })) + defer srv.Close() + + t.Setenv("DOCMARK_URL", srv.URL+"/mcp") + t.Setenv("DOCMARK_BEARER_TOKEN", "test-token-123") + + dir := t.TempDir() + path := filepath.Join(dir, "doc.docx") + require.NoError(t, os.WriteFile(path, []byte("fake docx bytes"), 0o644)) + + got, err := extractViaDocmark(path) + require.NoError(t, err) + assert.Equal(t, "# Converted\n\nhello", got) + assert.Equal(t, "Bearer test-token-123", gotAuth) + assert.Contains(t, gotAccept, "application/json") + params, _ := gotBody["params"].(map[string]any) + require.NotNil(t, params) + assert.Equal(t, "convert_to_markdown", params["name"]) + args, _ := params["arguments"].(map[string]any) + require.NotNil(t, args) + assert.Equal(t, "doc.docx", args["filename"]) + assert.NotEmpty(t, args["content_base64"]) +} + +func TestExtractViaDocmark_NotConfigured(t *testing.T) { + t.Setenv("DOCMARK_URL", "") + + dir := t.TempDir() + path := filepath.Join(dir, "doc.docx") + require.NoError(t, os.WriteFile(path, []byte("x"), 0o644)) + + _, err := extractViaDocmark(path) + require.Error(t, err) + assert.Contains(t, err.Error(), "DOCMARK_URL") +} + +func TestExtractViaDocmark_ToolErrorSurfacesMessage(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + writeMCPResponse(w, mcpToolResult{ + JSONRPC: "2.0", ID: 1, + Result: &struct { + Content []struct { + Type string `json:"type"` + Text string `json:"text"` + } `json:"content"` + IsError bool `json:"isError"` + }{ + Content: []struct { + Type string `json:"type"` + Text string `json:"text"` + }{{Type: "text", Text: "unsupported format for 'doc.docx'"}}, + IsError: true, + }, + }) + })) + defer srv.Close() + + t.Setenv("DOCMARK_URL", srv.URL+"/mcp") + t.Setenv("DOCMARK_BEARER_TOKEN", "tok") + + dir := t.TempDir() + path := filepath.Join(dir, "doc.docx") + require.NoError(t, os.WriteFile(path, []byte("x"), 0o644)) + + _, err := extractViaDocmark(path) + require.Error(t, err) + assert.Contains(t, err.Error(), "unsupported format") +} + +func TestExtractViaDocmark_HTTPErrorSurfaces(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusUnauthorized) + _, _ = w.Write([]byte("unauthorized")) + })) + defer srv.Close() + + t.Setenv("DOCMARK_URL", srv.URL+"/mcp") + t.Setenv("DOCMARK_BEARER_TOKEN", "wrong") + + dir := t.TempDir() + path := filepath.Join(dir, "doc.docx") + require.NoError(t, os.WriteFile(path, []byte("x"), 0o644)) + + _, err := extractViaDocmark(path) + require.Error(t, err) + assert.Contains(t, err.Error(), "401") +} + +func TestText_RoutesDocxXlsxPptxImagesToDocmark(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + writeMCPResponse(w, mcpToolResult{ + JSONRPC: "2.0", ID: 1, + Result: &struct { + Content []struct { + Type string `json:"type"` + Text string `json:"text"` + } `json:"content"` + IsError bool `json:"isError"` + }{ + Content: []struct { + Type string `json:"type"` + Text string `json:"text"` + }{{Type: "text", Text: "converted"}}, + }, + }) + })) + defer srv.Close() + t.Setenv("DOCMARK_URL", srv.URL+"/mcp") + t.Setenv("DOCMARK_BEARER_TOKEN", "tok") + + for _, ext := range []string{".docx", ".xlsx", ".pptx", ".png", ".jpg", ".jpeg"} { + t.Run(ext, func(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "f"+ext) + require.NoError(t, os.WriteFile(path, []byte("x"), 0o644)) + got, err := Text(path) + require.NoError(t, err) + assert.Equal(t, "converted", got) + }) + } +} diff --git a/ingestion/internal/extract/extract.go b/ingestion/internal/extract/extract.go index 725c85f..f7512ec 100644 --- a/ingestion/internal/extract/extract.go +++ b/ingestion/internal/extract/extract.go @@ -8,7 +8,9 @@ import ( ) // Text reads the file at path and returns its plain-text content. -// Supported extensions: .md, .txt (passthrough), .pdf (via pdftotext). +// Supported extensions: .md, .txt (passthrough), .pdf (via pdftotext), +// .docx/.xlsx/.pptx/.png/.jpg/.jpeg (via docmark, ADR-0013 -- requires +// DOCMARK_URL to be set; see docmark.go). func Text(path string) (string, error) { ext := strings.ToLower(fileExt(path)) switch ext { @@ -20,6 +22,8 @@ func Text(path string) (string, error) { return string(b), nil case ".pdf": return extractPDF(path) + case ".docx", ".xlsx", ".pptx", ".png", ".jpg", ".jpeg": + return extractViaDocmark(path) default: return "", fmt.Errorf("unsupported file extension: %s", ext) }