Merge pull request #6246 from wucm667/fix/issue-6240-preserve-multimodal-tool-output

fix(openai): preserve multimodal client tool outputs

backend/internal/pkg/apicompat/responses_client_tools.go

@@ -358,6 +358,9 @@ func normalizeClientToolOutput(item map[string]any) {
 	if _, ok := output.(string); ok {
 		return
 	}
+	if isResponsesToolOutputContent(output) {
+		return
+	}
 	if output == nil {
 		item["output"] = ""
 		return
@@ -370,6 +373,25 @@ func normalizeClientToolOutput(item map[string]any) {
 	item["output"] = string(encoded)
 }
 
+func isResponsesToolOutputContent(output any) bool {
+	parts, ok := output.([]any)
+	if !ok || len(parts) == 0 {
+		return false
+	}
+	for _, part := range parts {
+		typed, ok := part.(map[string]any)
+		if !ok {
+			return false
+		}
+		switch stringValue(typed["type"]) {
+		case "input_text", "input_image", "input_file":
+		default:
+			return false
+		}
+	}
+	return true
+}
+
 // normalizeToolSearchOutput converts both tool_search output wire shapes into
 // the string output required by function_call_output. Older clients send an
 // output field directly; newer Codex clients return discovered definitions in

backend/internal/pkg/apicompat/responses_client_tools_test.go

@@ -460,7 +460,55 @@ func TestAdaptResponsesClientToolsWithInheritedMapping_LowersFollowupHistoryWith
 	output := requireResponsesClientToolValue[map[string]any](t, items[1])
 	require.Equal(t, "function_call_output", output["type"])
 	require.NotContains(t, output, "id")
-	require.JSONEq(t, `[{"text":"ok","type":"input_text"}]`, requireResponsesClientToolValue[string](t, output["output"]))
+	require.Equal(t, []any{map[string]any{"type": "input_text", "text": "ok"}}, output["output"])
+}
+
+func TestAdaptResponsesClientTools_NormalizesCustomToolOutput(t *testing.T) {
+	tests := []struct {
+		name       string
+		output     any
+		wantOutput any
+	}{
+		{
+			name: "supported content parts remain an array",
+			output: []any{
+				map[string]any{"type": "input_text", "text": "ok"},
+				map[string]any{"type": "input_image", "image_url": "https://example.com/image.png"},
+				map[string]any{"type": "input_file", "file_id": "file_123"},
+			},
+			wantOutput: []any{
+				map[string]any{"type": "input_text", "text": "ok"},
+				map[string]any{"type": "input_image", "image_url": "https://example.com/image.png"},
+				map[string]any{"type": "input_file", "file_id": "file_123"},
+			},
+		},
+		{name: "ordinary object is stringified", output: map[string]any{"ok": true}, wantOutput: `{"ok":true}`},
+		{name: "arbitrary array is stringified", output: []any{"ok"}, wantOutput: `["ok"]`},
+		{name: "empty array is stringified", output: []any{}, wantOutput: `[]`},
+		{name: "mixed array is stringified", output: []any{map[string]any{"type": "input_text", "text": "ok"}, "bad"}, wantOutput: `[{"text":"ok","type":"input_text"},"bad"]`},
+		{name: "unknown content type is stringified", output: []any{map[string]any{"type": "output_text", "text": "bad"}}, wantOutput: `[{"text":"bad","type":"output_text"}]`},
+		{name: "whitespace-padded content type is stringified", output: []any{map[string]any{"type": " input_text ", "text": "bad"}}, wantOutput: `[{"text":"bad","type":" input_text "}]`},
+		{name: "missing content type is stringified", output: []any{map[string]any{"text": "bad"}}, wantOutput: `[{"text":"bad"}]`},
+		{name: "non-string content type is stringified", output: []any{map[string]any{"type": 1}}, wantOutput: `[{"type":1}]`},
+	}
+
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			req := map[string]any{
+				"tools": []any{map[string]any{"type": "custom", "name": "exec"}},
+				"input": []any{map[string]any{
+					"type": "custom_tool_call_output", "call_id": "call_1", "output": tc.output,
+				}},
+			}
+
+			_, changed, err := AdaptResponsesClientTools(req)
+			require.NoError(t, err)
+			require.True(t, changed)
+			item := requireResponsesClientToolValue[map[string]any](t, requireResponsesClientToolValue[[]any](t, req["input"])[0])
+			require.Equal(t, "function_call_output", item["type"])
+			require.Equal(t, tc.wantOutput, item["output"])
+		})
+	}
 }
 
 func TestAdaptResponsesClientToolsWithInheritedMapping_PromotesOmittedToolsDiscoveryIntoEffectiveDeclarations(t *testing.T) {

backend/internal/service/openai_gateway_responses_client_tools_test.go

@@ -250,6 +250,34 @@ func TestOpenAIPassthroughAPIKeyRestoresClientToolsNonStreaming(t *testing.T) {
 	require.Equal(t, "*** Begin Patch", gjson.Get(recorder.Body.String(), "output.1.input").String())
 }
 
+func TestOpenAIPassthroughAPIKeyPreservesCustomToolOutputContentParts(t *testing.T) {
+	gin.SetMode(gin.TestMode)
+	body := []byte(`{"model":"gpt-5.4","stream":false,"tools":[{"type":"custom","name":"exec"}],"input":[{"type":"custom_tool_call_output","call_id":"call_1","output":[{"type":"input_text","text":"result"},{"type":"input_file","file_id":"file_123"}]}]}`)
+	recorder := httptest.NewRecorder()
+	c, _ := gin.CreateTestContext(recorder)
+	c.Request = httptest.NewRequest(http.MethodPost, "/v1/responses", bytes.NewReader(body))
+
+	upstream := &httpUpstreamRecorder{resp: &http.Response{
+		StatusCode: http.StatusOK,
+		Header:     http.Header{"Content-Type": []string{"application/json"}},
+		Body:       io.NopCloser(strings.NewReader(`{"id":"resp_tools","status":"completed","output":[],"usage":{}}`)),
+	}}
+	svc := openAIClientToolsTestService(upstream)
+	account := &Account{ID: 6240, Platform: PlatformOpenAI, Type: AccountTypeAPIKey, Credentials: map[string]any{"api_key": "test-key"}}
+
+	result, err := svc.forwardOpenAIPassthrough(context.Background(), c, account, body, body, "gpt-5.4", false, nil, false, time.Now())
+
+	require.NoError(t, err)
+	require.NotNil(t, result)
+	require.Equal(t, "function_call_output", gjson.GetBytes(upstream.lastBody, "input.0.type").String())
+	output := gjson.GetBytes(upstream.lastBody, "input.0.output")
+	require.True(t, output.IsArray(), "native Responses content parts must reach the upstream as an array")
+	require.Equal(t, "input_text", output.Get("0.type").String())
+	require.Equal(t, "result", output.Get("0.text").String())
+	require.Equal(t, "input_file", output.Get("1.type").String())
+	require.Equal(t, "file_123", output.Get("1.file_id").String())
+}
+
 func TestOpenAIPassthroughAPIKeyRestoresClientToolsStreaming(t *testing.T) {
 	gin.SetMode(gin.TestMode)
 	body := openAIClientToolsRequest(true)