Files
sub2api/backend/internal/service/gemini_error_policy_test.go
T
feeeei ab0fcd1a0e fix(gemini): Skipped 错误策略对齐 OpenAI,上游 4xx 不再硬改 500
ErrorPolicySkipped(池模式、或自定义错误码未命中)原来在响应写出上
自成一派:v1beta 原生把上游 4xx 硬改 500 后原文透传,/v1/messages
硬传 500 进映射(客户端拿到 502)。下游网关据此把请求级错误当可重
试的服务端故障反复换号,耗尽后改写成 All available accounts
exhausted(2026-08-17 gemini 生产事故链)。现对齐 OpenAI 路径语义:

- Skipped 只豁免账号状态标记,不豁免换号:可 failover 状态码一律
  返回 UpstreamFailoverError(poolModeSkippedFailoverError 泛化为
  skippedErrorPolicyFailoverError,同账号重试标记仍仅池模式携带)
- 池模式的不可 failover 4xx 保真:v1beta 原码+原文透传(新
  writeGeminiNativeUpstreamError 与 ErrorPolicyNone 共用同一写出,
  并补记 ops 事件),/v1/messages 与 chat completions 按真实状态
  码映射
- 自定义错误码未命中且不可 failover:三路径统一 500 + "Upstream
  gateway error" 固定文案,上游细节仅记 ops 错误日志
- 400 属确定性请求错误:mapped 写出回传脱敏后的上游 message,客户
  端可据此定位非法字段
2026-08-18 11:24:39 +08:00

638 lines
20 KiB
Go
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
//go:build unit
package service
import (
"context"
"net/http"
"net/http/httptest"
"testing"
"time"
"github.com/Wei-Shaw/sub2api/internal/config"
"github.com/gin-gonic/gin"
"github.com/stretchr/testify/require"
)
// ---------------------------------------------------------------------------
// TestShouldFailoverGeminiUpstreamError — verifies the failover decision
// for the ErrorPolicyNone path (original logic preserved).
// ---------------------------------------------------------------------------
func TestShouldFailoverGeminiUpstreamError(t *testing.T) {
svc := &GeminiMessagesCompatService{}
tests := []struct {
name string
statusCode int
expected bool
}{
{"401_failover", 401, true},
{"403_failover", 403, true},
{"429_failover", 429, true},
{"529_failover", 529, true},
{"500_failover", 500, true},
{"502_failover", 502, true},
{"503_failover", 503, true},
{"400_no_failover", 400, false},
{"404_no_failover", 404, false},
{"422_no_failover", 422, false},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got := svc.shouldFailoverGeminiUpstreamError(tt.statusCode)
require.Equal(t, tt.expected, got)
})
}
}
// ---------------------------------------------------------------------------
// TestCheckErrorPolicy_GeminiAccounts — verifies CheckErrorPolicy works
// correctly for Gemini platform accounts (API Key type).
// ---------------------------------------------------------------------------
func TestCheckErrorPolicy_GeminiAccounts(t *testing.T) {
tests := []struct {
name string
account *Account
statusCode int
body []byte
expected ErrorPolicyResult
}{
{
name: "gemini_apikey_custom_codes_hit",
account: &Account{
ID: 100,
Type: AccountTypeAPIKey,
Platform: PlatformGemini,
Credentials: map[string]any{
"custom_error_codes_enabled": true,
"custom_error_codes": []any{float64(429), float64(500)},
},
},
statusCode: 429,
body: []byte(`{"error":"rate limited"}`),
expected: ErrorPolicyMatched,
},
{
name: "gemini_apikey_custom_codes_miss",
account: &Account{
ID: 101,
Type: AccountTypeAPIKey,
Platform: PlatformGemini,
Credentials: map[string]any{
"custom_error_codes_enabled": true,
"custom_error_codes": []any{float64(429)},
},
},
statusCode: 500,
body: []byte(`{"error":"internal"}`),
expected: ErrorPolicySkipped,
},
{
name: "gemini_apikey_no_custom_codes_returns_none",
account: &Account{
ID: 102,
Type: AccountTypeAPIKey,
Platform: PlatformGemini,
},
statusCode: 500,
body: []byte(`{"error":"internal"}`),
expected: ErrorPolicyNone,
},
{
name: "gemini_apikey_temp_unschedulable_hit",
account: &Account{
ID: 103,
Type: AccountTypeAPIKey,
Platform: PlatformGemini,
Credentials: map[string]any{
"temp_unschedulable_enabled": true,
"temp_unschedulable_rules": []any{
map[string]any{
"error_code": float64(503),
"keywords": []any{"overloaded"},
"duration_minutes": float64(10),
},
},
},
},
statusCode: 503,
body: []byte(`overloaded service`),
expected: ErrorPolicyTempUnscheduled,
},
{
name: "gemini_apikey_temp_unschedulable_401_second_hit_returns_none",
account: &Account{
ID: 105,
Type: AccountTypeAPIKey,
Platform: PlatformGemini,
TempUnschedulableReason: `{"status_code":401,"until_unix":1735689600}`,
Credentials: map[string]any{
"temp_unschedulable_enabled": true,
"temp_unschedulable_rules": []any{
map[string]any{
"error_code": float64(401),
"keywords": []any{"unauthorized"},
"duration_minutes": float64(10),
},
},
},
},
statusCode: 401,
body: []byte(`unauthorized`),
expected: ErrorPolicyNone,
},
{
name: "gemini_custom_codes_override_temp_unschedulable",
account: &Account{
ID: 104,
Type: AccountTypeAPIKey,
Platform: PlatformGemini,
Credentials: map[string]any{
"custom_error_codes_enabled": true,
"custom_error_codes": []any{float64(503)},
"temp_unschedulable_enabled": true,
"temp_unschedulable_rules": []any{
map[string]any{
"error_code": float64(503),
"keywords": []any{"overloaded"},
"duration_minutes": float64(10),
},
},
},
},
statusCode: 503,
body: []byte(`overloaded`),
expected: ErrorPolicyMatched, // custom codes take precedence
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
repo := &errorPolicyRepoStub{}
svc := NewRateLimitService(repo, nil, &config.Config{}, nil, nil)
result := svc.CheckErrorPolicy(context.Background(), tt.account, tt.statusCode, tt.body)
require.Equal(t, tt.expected, result)
})
}
}
// ---------------------------------------------------------------------------
// TestGeminiErrorPolicyIntegration — verifies the Gemini error handling
// paths produce the correct behavior for each ErrorPolicyResult.
//
// These tests simulate the inline error policy switch in handleClaudeCompat
// and forwardNativeGemini by calling the same methods in the same order.
// ---------------------------------------------------------------------------
func TestGeminiErrorPolicyIntegration(t *testing.T) {
gin.SetMode(gin.TestMode)
tests := []struct {
name string
account *Account
statusCode int
respBody []byte
expectFailover bool // expect UpstreamFailoverError
expectHandleError bool // expect handleGeminiUpstreamError to be called
expectShouldFailover bool // for None path, whether shouldFailover triggers
expectModelScope string
}{
{
name: "custom_codes_matched_429_failover",
account: &Account{
ID: 200,
Type: AccountTypeAPIKey,
Platform: PlatformGemini,
Credentials: map[string]any{
"custom_error_codes_enabled": true,
"custom_error_codes": []any{float64(429)},
},
},
statusCode: 429,
respBody: []byte(`{"error":"rate limited"}`),
expectFailover: true,
expectHandleError: true,
},
{
name: "custom_codes_skipped_500_failover",
account: &Account{
ID: 201,
Type: AccountTypeAPIKey,
Platform: PlatformGemini,
Credentials: map[string]any{
"custom_error_codes_enabled": true,
"custom_error_codes": []any{float64(429)},
},
},
statusCode: 500,
respBody: []byte(`{"error":"internal"}`),
expectFailover: true,
expectHandleError: false,
},
{
name: "custom_codes_skipped_400_no_failover",
account: &Account{
ID: 205,
Type: AccountTypeAPIKey,
Platform: PlatformGemini,
Credentials: map[string]any{
"custom_error_codes_enabled": true,
"custom_error_codes": []any{float64(429)},
},
},
statusCode: 400,
respBody: []byte(`{"error":"bad request"}`),
expectFailover: false,
expectHandleError: false,
},
{
name: "temp_unschedulable_matched_failover",
account: &Account{
ID: 202,
Type: AccountTypeAPIKey,
Platform: PlatformGemini,
Credentials: map[string]any{
"temp_unschedulable_enabled": true,
"temp_unschedulable_rules": []any{
map[string]any{
"error_code": float64(503),
"keywords": []any{"overloaded"},
"duration_minutes": float64(10),
},
},
},
},
statusCode: 503,
respBody: []byte(`overloaded`),
expectFailover: true,
expectHandleError: false,
expectModelScope: "gemini-2.5-pro",
},
{
name: "no_policy_429_failover_via_shouldFailover",
account: &Account{
ID: 203,
Type: AccountTypeAPIKey,
Platform: PlatformGemini,
},
statusCode: 429,
respBody: []byte(`{"error":"rate limited"}`),
expectFailover: true,
expectHandleError: true,
expectShouldFailover: true,
},
{
name: "no_policy_400_no_failover",
account: &Account{
ID: 204,
Type: AccountTypeAPIKey,
Platform: PlatformGemini,
},
statusCode: 400,
respBody: []byte(`{"error":"bad request"}`),
expectFailover: false,
expectHandleError: true,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
repo := &geminiErrorPolicyRepo{}
rlSvc := NewRateLimitService(repo, nil, &config.Config{}, nil, nil)
svc := &GeminiMessagesCompatService{
accountRepo: repo,
rateLimitService: rlSvc,
}
writer := httptest.NewRecorder()
c, _ := gin.CreateTestContext(writer)
c.Request = httptest.NewRequest(http.MethodPost, "/v1/messages", nil)
// Simulate the Claude compat error handling path (same logic as native).
// This mirrors the inline switch in handleClaudeCompat.
var handleErrorCalled bool
var gotFailover bool
ctx := context.Background()
statusCode := tt.statusCode
respBody := tt.respBody
account := tt.account
headers := http.Header{}
if svc.rateLimitService != nil {
policy := svc.rateLimitService.CheckErrorPolicy(ctx, account, statusCode, respBody, "gemini-2.5-pro")
switch policy {
case ErrorPolicySkipped:
// Skipped → 不标记账号状态;可 failover 的状态码仍换号
handleErrorCalled = false
gotFailover = svc.skippedErrorPolicyFailoverError(c, account, statusCode, respBody, "req-test") != nil
goto verify
case ErrorPolicyMatched:
svc.handleGeminiUpstreamError(ctx, account, statusCode, headers, respBody)
handleErrorCalled = true
gotFailover = true
goto verify
case ErrorPolicyTempUnscheduled:
handleErrorCalled = false
gotFailover = true
goto verify
}
}
// ErrorPolicyNone → original logic
svc.handleGeminiUpstreamError(ctx, account, statusCode, headers, respBody)
handleErrorCalled = true
if svc.shouldFailoverGeminiUpstreamError(statusCode) {
gotFailover = true
}
verify:
require.Equal(t, tt.expectFailover, gotFailover, "failover mismatch")
require.Equal(t, tt.expectHandleError, handleErrorCalled, "handleGeminiUpstreamError call mismatch")
if tt.expectModelScope != "" {
require.Equal(t, 1, repo.setModelRateLimitedCalls)
require.Equal(t, tt.expectModelScope, repo.lastModelScope)
require.Zero(t, repo.setTempCalls)
require.Zero(t, repo.setRateLimitedCalls, "model temp rule must not be widened into an account rate limit")
}
if tt.expectShouldFailover {
require.True(t, svc.shouldFailoverGeminiUpstreamError(statusCode),
"shouldFailoverGeminiUpstreamError should return true for status %d", statusCode)
}
})
}
}
// ---------------------------------------------------------------------------
// TestSkippedErrorPolicyFailoverError — ErrorPolicySkipped(池模式、或自定义
// 错误码未命中)不豁免换号:可 failover 的状态码返回 UpstreamFailoverError,
// 仅池模式账号可携带同账号重试标记。
// ---------------------------------------------------------------------------
func TestSkippedErrorPolicyFailoverError(t *testing.T) {
gin.SetMode(gin.TestMode)
svc := &GeminiMessagesCompatService{}
poolAccount := func(extra map[string]any) *Account {
creds := map[string]any{"pool_mode": true}
for k, v := range extra {
creds[k] = v
}
return &Account{ID: 300, Type: AccountTypeAPIKey, Platform: PlatformGemini, Credentials: creds}
}
customCodesAccount := &Account{
ID: 301, Type: AccountTypeAPIKey, Platform: PlatformGemini,
Credentials: map[string]any{
"custom_error_codes_enabled": true,
"custom_error_codes": []any{float64(429)},
},
}
tests := []struct {
name string
account *Account
statusCode int
expectFailover bool
expectSameAccount bool
}{
{"pool_500_failover_no_same_account_retry", poolAccount(nil), 500, true, false},
{"pool_429_failover_with_same_account_retry", poolAccount(nil), 429, true, true},
{"pool_custom_retry_codes_500", poolAccount(map[string]any{
"pool_mode_retry_status_codes": []any{float64(500)},
}), 500, true, true},
{"pool_400_not_failover_worthy", poolAccount(nil), 400, false, false},
{"custom_codes_miss_500_failover_no_same_account_retry", customCodesAccount, 500, true, false},
{"custom_codes_miss_400_not_failover_worthy", customCodesAccount, 400, false, false},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
writer := httptest.NewRecorder()
c, _ := gin.CreateTestContext(writer)
c.Request = httptest.NewRequest(http.MethodPost, "/v1/messages", nil)
body := []byte(`{"error":{"code":"bad_response_status_code","message":"openai_error"}}`)
failoverErr := svc.skippedErrorPolicyFailoverError(c, tt.account, tt.statusCode, body, "req-1")
if !tt.expectFailover {
require.Nil(t, failoverErr)
return
}
require.NotNil(t, failoverErr)
require.Equal(t, tt.statusCode, failoverErr.StatusCode)
require.Equal(t, body, failoverErr.ResponseBody)
require.Equal(t, tt.expectSameAccount, failoverErr.RetryableOnSameAccount)
require.True(t, failoverErr.ShouldRetryNextAccount())
})
}
}
// ---------------------------------------------------------------------------
// TestGeminiErrorPolicy_NilRateLimitService — verifies nil safety
// ---------------------------------------------------------------------------
func TestGeminiErrorPolicy_NilRateLimitService(t *testing.T) {
svc := &GeminiMessagesCompatService{
rateLimitService: nil,
}
// When rateLimitService is nil, error policy is skipped → falls through to
// shouldFailoverGeminiUpstreamError (original logic).
// Verify this doesn't panic and follows expected behavior.
ctx := context.Background()
account := &Account{
ID: 300,
Type: AccountTypeAPIKey,
Platform: PlatformGemini,
Credentials: map[string]any{
"custom_error_codes_enabled": true,
"custom_error_codes": []any{float64(429)},
},
}
// The nil check should prevent CheckErrorPolicy from being called
if svc.rateLimitService != nil {
t.Fatal("rateLimitService should be nil for this test")
}
// shouldFailoverGeminiUpstreamError still works
require.True(t, svc.shouldFailoverGeminiUpstreamError(429))
require.False(t, svc.shouldFailoverGeminiUpstreamError(400))
// handleGeminiUpstreamError should not panic with nil rateLimitService
require.NotPanics(t, func() {
svc.handleGeminiUpstreamError(ctx, account, 500, http.Header{}, []byte(`error`))
})
}
// ---------------------------------------------------------------------------
// geminiErrorPolicyRepo — minimal AccountRepository stub for Gemini error
// policy tests. Embeds mockAccountRepoForGemini and adds tracking.
// ---------------------------------------------------------------------------
func TestHandleGeminiUpstreamError_GoogleOneCapacityExhaustedUsesTierCooldown(t *testing.T) {
repo := &rateLimit429AccountRepoStub{}
quotaSvc := NewGeminiQuotaService(&config.Config{}, nil)
rlSvc := NewRateLimitService(repo, nil, &config.Config{}, quotaSvc, nil)
svc := &GeminiMessagesCompatService{
accountRepo: repo,
rateLimitService: rlSvc,
}
account := &Account{
ID: 511,
Platform: PlatformGemini,
Type: AccountTypeOAuth,
Credentials: map[string]any{
"oauth_type": "google_one",
"tier_id": "google_ai_pro",
},
}
body := []byte(`{"error":{"code":429,"details":[{"@type":"type.googleapis.com/google.rpc.ErrorInfo","domain":"cloudcode-pa.googleapis.com","metadata":{"model":"gemini-3.1-pro-preview"},"reason":"MODEL_CAPACITY_EXHAUSTED"}],"message":"No capacity available for model gemini-3.1-pro-preview on the server","status":"RESOURCE_EXHAUSTED"}}`)
before := time.Now()
svc.handleGeminiUpstreamError(context.Background(), account, http.StatusTooManyRequests, http.Header{}, body)
after := time.Now()
require.Equal(t, 1, repo.rateLimitCalls)
require.Equal(t, int64(511), repo.lastRateLimitID)
require.WithinDuration(t, before.Add(5*time.Minute), repo.lastRateLimitReset, 2*time.Second)
require.True(t, repo.lastRateLimitReset.After(before))
require.True(t, repo.lastRateLimitReset.Before(after.Add(5*time.Minute).Add(2*time.Second)))
}
// ---------------------------------------------------------------------------
// TestHandleGeminiUpstreamError_PoolMode429 — 池模式账号的 429 不写账号级限流。
//
// 429 的标记点在重试循环内(handleClaudeCompat / forwardNativeGemini /
// chat completions 三条路径),先于 CheckErrorPolicy 执行,池模式豁免只能落在
// handleGeminiUpstreamError 自身;否则一次上游 429 会把账号锁到 PST 午夜,
// 即便重试已经成功返回客户端。
// ---------------------------------------------------------------------------
func TestHandleGeminiUpstreamError_PoolMode429(t *testing.T) {
// 中转上游的真实 429 文案:不含 "per day",也没有 quotaResetDelay,
// 解析失败后 apikey 账号会落到 PST 午夜兜底。
body := []byte(`{"error":{"code":429,"message":"You have exhausted your capacity on this model. Your quota will reset after 6h53m10s."}}`)
tests := []struct {
name string
account *Account
expectRateLimited bool
}{
{
name: "pool_mode_apikey_stays_in_pool",
account: &Account{
ID: 600,
Platform: PlatformGemini,
Type: AccountTypeAPIKey,
Credentials: map[string]any{"pool_mode": true},
},
expectRateLimited: false,
},
{
name: "custom_error_codes_hit_overrides_pool_mode",
account: &Account{
ID: 601,
Platform: PlatformGemini,
Type: AccountTypeAPIKey,
Credentials: map[string]any{
"pool_mode": true,
"custom_error_codes_enabled": true,
"custom_error_codes": []any{float64(429)},
},
},
expectRateLimited: true,
},
{
name: "custom_error_codes_miss_skips",
account: &Account{
ID: 602,
Platform: PlatformGemini,
Type: AccountTypeAPIKey,
Credentials: map[string]any{
"pool_mode": true,
"custom_error_codes_enabled": true,
"custom_error_codes": []any{float64(500)},
},
},
expectRateLimited: false,
},
{
name: "non_pool_apikey_still_rate_limited",
account: &Account{
ID: 603,
Platform: PlatformGemini,
Type: AccountTypeAPIKey,
},
expectRateLimited: true,
},
{
name: "oauth_account_ignores_pool_mode_flag",
account: &Account{
ID: 604,
Platform: PlatformGemini,
Type: AccountTypeOAuth,
Credentials: map[string]any{"pool_mode": true},
},
expectRateLimited: true,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
repo := &rateLimit429AccountRepoStub{}
svc := &GeminiMessagesCompatService{
accountRepo: repo,
rateLimitService: NewRateLimitService(repo, nil, &config.Config{}, nil, nil),
}
svc.handleGeminiUpstreamError(context.Background(), tt.account, http.StatusTooManyRequests, http.Header{}, body)
if !tt.expectRateLimited {
require.Zero(t, repo.rateLimitCalls, "池模式账号不应被标记账号级限流")
return
}
require.Equal(t, 1, repo.rateLimitCalls)
require.Equal(t, tt.account.ID, repo.lastRateLimitID)
require.True(t, repo.lastRateLimitReset.After(time.Now()))
})
}
}
type geminiErrorPolicyRepo struct {
mockAccountRepoForGemini
setErrorCalls int
setRateLimitedCalls int
setTempCalls int
setModelRateLimitedCalls int
lastModelScope string
}
func (r *geminiErrorPolicyRepo) SetError(_ context.Context, _ int64, _ string) error {
r.setErrorCalls++
return nil
}
func (r *geminiErrorPolicyRepo) SetRateLimited(_ context.Context, _ int64, _ time.Time) error {
r.setRateLimitedCalls++
return nil
}
func (r *geminiErrorPolicyRepo) SetTempUnschedulable(_ context.Context, _ int64, _ time.Time, _ string) error {
r.setTempCalls++
return nil
}
func (r *geminiErrorPolicyRepo) SetModelRateLimit(_ context.Context, _ int64, scope string, _ time.Time, _ ...string) error {
r.setModelRateLimitedCalls++
r.lastModelScope = scope
return nil
}