From 50ba146298538b4b683ef38f88548985d6871f47 Mon Sep 17 00:00:00 2001 From: wucm667 Date: Wed, 26 Aug 2026 15:04:17 +0800 Subject: [PATCH] fix(openai): preserve multimodal client tool outputs --- .../pkg/apicompat/responses_client_tools.go | 22 ++++++++ .../apicompat/responses_client_tools_test.go | 50 ++++++++++++++++++- ...nai_gateway_responses_client_tools_test.go | 28 +++++++++++ 3 files changed, 99 insertions(+), 1 deletion(-) diff --git a/backend/internal/pkg/apicompat/responses_client_tools.go b/backend/internal/pkg/apicompat/responses_client_tools.go index 5e831c1c52..c537761062 100644 --- a/backend/internal/pkg/apicompat/responses_client_tools.go +++ b/backend/internal/pkg/apicompat/responses_client_tools.go @@ -358,6 +358,9 @@ func normalizeClientToolOutput(item map[string]any) { if _, ok := output.(string); ok { return } + if isResponsesToolOutputContent(output) { + return + } if output == nil { item["output"] = "" return @@ -370,6 +373,25 @@ func normalizeClientToolOutput(item map[string]any) { item["output"] = string(encoded) } +func isResponsesToolOutputContent(output any) bool { + parts, ok := output.([]any) + if !ok || len(parts) == 0 { + return false + } + for _, part := range parts { + typed, ok := part.(map[string]any) + if !ok { + return false + } + switch stringValue(typed["type"]) { + case "input_text", "input_image", "input_file": + default: + return false + } + } + return true +} + // normalizeToolSearchOutput converts both tool_search output wire shapes into // the string output required by function_call_output. Older clients send an // output field directly; newer Codex clients return discovered definitions in diff --git a/backend/internal/pkg/apicompat/responses_client_tools_test.go b/backend/internal/pkg/apicompat/responses_client_tools_test.go index bfe3fd2eb7..dd8bb74fea 100644 --- a/backend/internal/pkg/apicompat/responses_client_tools_test.go +++ b/backend/internal/pkg/apicompat/responses_client_tools_test.go @@ -460,7 +460,55 @@ func TestAdaptResponsesClientToolsWithInheritedMapping_LowersFollowupHistoryWith output := requireResponsesClientToolValue[map[string]any](t, items[1]) require.Equal(t, "function_call_output", output["type"]) require.NotContains(t, output, "id") - require.JSONEq(t, `[{"text":"ok","type":"input_text"}]`, requireResponsesClientToolValue[string](t, output["output"])) + require.Equal(t, []any{map[string]any{"type": "input_text", "text": "ok"}}, output["output"]) +} + +func TestAdaptResponsesClientTools_NormalizesCustomToolOutput(t *testing.T) { + tests := []struct { + name string + output any + wantOutput any + }{ + { + name: "supported content parts remain an array", + output: []any{ + map[string]any{"type": "input_text", "text": "ok"}, + map[string]any{"type": "input_image", "image_url": "https://example.com/image.png"}, + map[string]any{"type": "input_file", "file_id": "file_123"}, + }, + wantOutput: []any{ + map[string]any{"type": "input_text", "text": "ok"}, + map[string]any{"type": "input_image", "image_url": "https://example.com/image.png"}, + map[string]any{"type": "input_file", "file_id": "file_123"}, + }, + }, + {name: "ordinary object is stringified", output: map[string]any{"ok": true}, wantOutput: `{"ok":true}`}, + {name: "arbitrary array is stringified", output: []any{"ok"}, wantOutput: `["ok"]`}, + {name: "empty array is stringified", output: []any{}, wantOutput: `[]`}, + {name: "mixed array is stringified", output: []any{map[string]any{"type": "input_text", "text": "ok"}, "bad"}, wantOutput: `[{"text":"ok","type":"input_text"},"bad"]`}, + {name: "unknown content type is stringified", output: []any{map[string]any{"type": "output_text", "text": "bad"}}, wantOutput: `[{"text":"bad","type":"output_text"}]`}, + {name: "whitespace-padded content type is stringified", output: []any{map[string]any{"type": " input_text ", "text": "bad"}}, wantOutput: `[{"text":"bad","type":" input_text "}]`}, + {name: "missing content type is stringified", output: []any{map[string]any{"text": "bad"}}, wantOutput: `[{"text":"bad"}]`}, + {name: "non-string content type is stringified", output: []any{map[string]any{"type": 1}}, wantOutput: `[{"type":1}]`}, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + req := map[string]any{ + "tools": []any{map[string]any{"type": "custom", "name": "exec"}}, + "input": []any{map[string]any{ + "type": "custom_tool_call_output", "call_id": "call_1", "output": tc.output, + }}, + } + + _, changed, err := AdaptResponsesClientTools(req) + require.NoError(t, err) + require.True(t, changed) + item := requireResponsesClientToolValue[map[string]any](t, requireResponsesClientToolValue[[]any](t, req["input"])[0]) + require.Equal(t, "function_call_output", item["type"]) + require.Equal(t, tc.wantOutput, item["output"]) + }) + } } func TestAdaptResponsesClientToolsWithInheritedMapping_PromotesOmittedToolsDiscoveryIntoEffectiveDeclarations(t *testing.T) { diff --git a/backend/internal/service/openai_gateway_responses_client_tools_test.go b/backend/internal/service/openai_gateway_responses_client_tools_test.go index fa4866decf..1c9bdd6bcd 100644 --- a/backend/internal/service/openai_gateway_responses_client_tools_test.go +++ b/backend/internal/service/openai_gateway_responses_client_tools_test.go @@ -250,6 +250,34 @@ func TestOpenAIPassthroughAPIKeyRestoresClientToolsNonStreaming(t *testing.T) { require.Equal(t, "*** Begin Patch", gjson.Get(recorder.Body.String(), "output.1.input").String()) } +func TestOpenAIPassthroughAPIKeyPreservesCustomToolOutputContentParts(t *testing.T) { + gin.SetMode(gin.TestMode) + body := []byte(`{"model":"gpt-5.4","stream":false,"tools":[{"type":"custom","name":"exec"}],"input":[{"type":"custom_tool_call_output","call_id":"call_1","output":[{"type":"input_text","text":"result"},{"type":"input_file","file_id":"file_123"}]}]}`) + recorder := httptest.NewRecorder() + c, _ := gin.CreateTestContext(recorder) + c.Request = httptest.NewRequest(http.MethodPost, "/v1/responses", bytes.NewReader(body)) + + upstream := &httpUpstreamRecorder{resp: &http.Response{ + StatusCode: http.StatusOK, + Header: http.Header{"Content-Type": []string{"application/json"}}, + Body: io.NopCloser(strings.NewReader(`{"id":"resp_tools","status":"completed","output":[],"usage":{}}`)), + }} + svc := openAIClientToolsTestService(upstream) + account := &Account{ID: 6240, Platform: PlatformOpenAI, Type: AccountTypeAPIKey, Credentials: map[string]any{"api_key": "test-key"}} + + result, err := svc.forwardOpenAIPassthrough(context.Background(), c, account, body, body, "gpt-5.4", false, nil, false, time.Now()) + + require.NoError(t, err) + require.NotNil(t, result) + require.Equal(t, "function_call_output", gjson.GetBytes(upstream.lastBody, "input.0.type").String()) + output := gjson.GetBytes(upstream.lastBody, "input.0.output") + require.True(t, output.IsArray(), "native Responses content parts must reach the upstream as an array") + require.Equal(t, "input_text", output.Get("0.type").String()) + require.Equal(t, "result", output.Get("0.text").String()) + require.Equal(t, "input_file", output.Get("1.type").String()) + require.Equal(t, "file_123", output.Get("1.file_id").String()) +} + func TestOpenAIPassthroughAPIKeyRestoresClientToolsStreaming(t *testing.T) { gin.SetMode(gin.TestMode) body := openAIClientToolsRequest(true)