fix(llm): send generous max_tokens on every request
The copied OpenAI-compatible client sent no max_tokens. Thinking models (qwen3, deepseek-r1) spend their budget on the reasoning trace and return EMPTY content when max_tokens is unset, which the summarizer treats as an error. ADR-004 says change Tapir's copy rather than the hyperguild upstream, so set a generous default (8192) leaving room for both reasoning and output. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -17,11 +17,20 @@ import (
|
|||||||
"time"
|
"time"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
// defaultMaxTokens is sent on every request. Tapir CHANGES this from the
|
||||||
|
// hyperguild copy (ADR-004 says change the copy, not the upstream): thinking
|
||||||
|
// models (qwen3, deepseek-r1) spend their budget on reasoning and return EMPTY
|
||||||
|
// content when max_tokens is unset or too low. A generous ceiling leaves room
|
||||||
|
// for both the reasoning trace and the actual summary. See
|
||||||
|
// docs/homelab-integration.md.
|
||||||
|
const defaultMaxTokens = 8192
|
||||||
|
|
||||||
// Client calls an OpenAI-compatible chat completions endpoint.
|
// Client calls an OpenAI-compatible chat completions endpoint.
|
||||||
type Client struct {
|
type Client struct {
|
||||||
baseURL string
|
baseURL string
|
||||||
apiKey string
|
apiKey string
|
||||||
model string
|
model string
|
||||||
|
maxTokens int
|
||||||
httpClient *http.Client
|
httpClient *http.Client
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -31,6 +40,7 @@ func New(baseURL, apiKey, model string, timeout time.Duration) *Client {
|
|||||||
baseURL: strings.TrimRight(baseURL, "/"),
|
baseURL: strings.TrimRight(baseURL, "/"),
|
||||||
apiKey: apiKey,
|
apiKey: apiKey,
|
||||||
model: model,
|
model: model,
|
||||||
|
maxTokens: defaultMaxTokens,
|
||||||
httpClient: &http.Client{Timeout: timeout},
|
httpClient: &http.Client{Timeout: timeout},
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -39,6 +49,7 @@ type chatRequest struct {
|
|||||||
Model string `json:"model"`
|
Model string `json:"model"`
|
||||||
Messages []message `json:"messages"`
|
Messages []message `json:"messages"`
|
||||||
Temperature float64 `json:"temperature"`
|
Temperature float64 `json:"temperature"`
|
||||||
|
MaxTokens int `json:"max_tokens,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
type message struct {
|
type message struct {
|
||||||
@@ -62,6 +73,7 @@ func (c *Client) Complete(ctx context.Context, system, user string) (string, err
|
|||||||
{Role: "user", Content: user},
|
{Role: "user", Content: user},
|
||||||
},
|
},
|
||||||
Temperature: 0.2,
|
Temperature: 0.2,
|
||||||
|
MaxTokens: c.maxTokens,
|
||||||
}
|
}
|
||||||
b, err := json.Marshal(body)
|
b, err := json.Marshal(body)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
|||||||
@@ -43,6 +43,27 @@ func TestClient_Complete(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// TestClient_SendsMaxTokens guards Tapir's ADR-004 change to the copied client:
|
||||||
|
// it MUST send a positive max_tokens, or thinking models return empty content.
|
||||||
|
func TestClient_SendsMaxTokens(t *testing.T) {
|
||||||
|
var body chatRequest
|
||||||
|
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
_ = json.NewDecoder(r.Body).Decode(&body)
|
||||||
|
_ = json.NewEncoder(w).Encode(map[string]any{
|
||||||
|
"choices": []map[string]any{{"message": map[string]any{"content": "ok"}}},
|
||||||
|
})
|
||||||
|
}))
|
||||||
|
defer srv.Close()
|
||||||
|
|
||||||
|
c := New(srv.URL, "", "test-model", 10*time.Second)
|
||||||
|
if _, err := c.Complete(context.Background(), "sys", "user"); err != nil {
|
||||||
|
t.Fatalf("Complete: %v", err)
|
||||||
|
}
|
||||||
|
if body.MaxTokens <= 0 {
|
||||||
|
t.Errorf("max_tokens = %d, want > 0 (thinking models return empty content without it)", body.MaxTokens)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestClient_ReturnsErrorOnNon200(t *testing.T) {
|
func TestClient_ReturnsErrorOnNon200(t *testing.T) {
|
||||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
http.Error(w, "overloaded", http.StatusServiceUnavailable)
|
http.Error(w, "overloaded", http.StatusServiceUnavailable)
|
||||||
|
|||||||
Reference in New Issue
Block a user