mirror of
https://github.com/Wei-Shaw/sub2api.git
synced 2026-10-07 13:18:18 +08:00
Merge pull request #6358 from Wei-Shaw/fix/openai-spark-model-429-rate-limit
fix(openai): scope Spark quota 429 to model
This commit is contained in:
@@ -12,6 +12,7 @@ const (
|
||||
modelRateLimitsKey = "model_rate_limits"
|
||||
antigravityGeminiModelRateLimitKey = "antigravity:gemini"
|
||||
openAIImageGenerationRateLimitKey = "openai:image_generation"
|
||||
openAICodexSparkRateLimitReason = "openai_codex_spark_rate_limit"
|
||||
// anthropicFableRateLimitKey 是 Anthropic 7d_oi(Fable 专属 7d 窗口)限流的
|
||||
// 家族级 scope:命中后所有 Fable 变体(含 [1m] 等后缀)都不再调度到该账号。
|
||||
anthropicFableRateLimitKey = "claude-fable-5"
|
||||
|
||||
@@ -162,6 +162,10 @@ func (s *OpenAIGatewayService) handleOpenAIAccountUpstreamError(ctx context.Cont
|
||||
s.rateLimitService.HandleTempUnschedulable(stateCtx, account, statusCode, responseBody, canonicalModel[0]) {
|
||||
return true
|
||||
}
|
||||
if statusCode == http.StatusTooManyRequests && s.rateLimitService != nil && len(canonicalModel) > 0 &&
|
||||
s.rateLimitService.HandleOpenAICodexSparkRateLimit(stateCtx, account, canonicalModel[0], statusCode, headers, responseBody) {
|
||||
return false
|
||||
}
|
||||
if statusCode == http.StatusTooManyRequests {
|
||||
s.markOpenAIOAuth429RateLimited(stateCtx, account, headers, responseBody)
|
||||
}
|
||||
|
||||
@@ -15,8 +15,11 @@ import (
|
||||
|
||||
type oauth429RateLimitRepo struct {
|
||||
mockAccountRepoForGemini
|
||||
setRateLimitedCalls int
|
||||
lastRateLimitedUntil time.Time
|
||||
setRateLimitedCalls int
|
||||
lastRateLimitedUntil time.Time
|
||||
setModelRateLimitCalls int
|
||||
lastModelRateLimitKey string
|
||||
lastModelRateLimitedUntil time.Time
|
||||
}
|
||||
|
||||
func (r *oauth429RateLimitRepo) SetRateLimited(_ context.Context, _ int64, until time.Time) error {
|
||||
@@ -25,6 +28,13 @@ func (r *oauth429RateLimitRepo) SetRateLimited(_ context.Context, _ int64, until
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *oauth429RateLimitRepo) SetModelRateLimit(_ context.Context, _ int64, scope string, until time.Time, _ ...string) error {
|
||||
r.setModelRateLimitCalls++
|
||||
r.lastModelRateLimitKey = scope
|
||||
r.lastModelRateLimitedUntil = until
|
||||
return nil
|
||||
}
|
||||
|
||||
func TestOpenAI429FastPath_KeepsOAuthAccountSchedulableDuringRetryWindow(t *testing.T) {
|
||||
repo := &oauth429RateLimitRepo{}
|
||||
rateLimits := NewRateLimitService(repo, nil, &config.Config{}, nil, nil)
|
||||
@@ -85,6 +95,176 @@ func TestOpenAI429FastPath_BlocksOAuthImmediatelyWhenSevenDayQuotaIsExhausted(t
|
||||
require.False(t, svc.ShouldRetryOpenAIOAuth429(account, headers, nil))
|
||||
}
|
||||
|
||||
func TestOpenAI429FastPath_SparkQuotaOnlyBlocksSparkModel(t *testing.T) {
|
||||
repo := &oauth429RateLimitRepo{}
|
||||
rateLimits := NewRateLimitService(repo, nil, &config.Config{}, nil, nil)
|
||||
svc := &OpenAIGatewayService{rateLimitService: rateLimits}
|
||||
rateLimits.SetAccountRuntimeBlocker(svc)
|
||||
account := &Account{ID: 425, Platform: PlatformOpenAI, Type: AccountTypeOAuth}
|
||||
headers := http.Header{}
|
||||
headers.Set("x-codex-primary-used-percent", "100")
|
||||
headers.Set("x-codex-primary-reset-after-seconds", "604800")
|
||||
headers.Set("x-codex-primary-window-minutes", "10080")
|
||||
headers.Set("x-codex-secondary-used-percent", "20")
|
||||
headers.Set("x-codex-secondary-reset-after-seconds", "3600")
|
||||
headers.Set("x-codex-secondary-window-minutes", "300")
|
||||
|
||||
shouldDisable := svc.handleOpenAIAccountUpstreamError(
|
||||
context.Background(), account, http.StatusTooManyRequests, headers,
|
||||
[]byte(`{"error":{"type":"rate_limit_error","code":"rate_limit_exceeded"}}`),
|
||||
"gpt-5.3-codex-spark",
|
||||
)
|
||||
|
||||
require.False(t, shouldDisable)
|
||||
require.False(t, svc.isOpenAIAccountRuntimeBlocked(account), "Spark quota must not create an account runtime block")
|
||||
require.Equal(t, 0, repo.setRateLimitedCalls, "Spark quota must not persist account-level rate limit")
|
||||
require.Equal(t, 1, repo.setModelRateLimitCalls)
|
||||
require.Equal(t, "gpt-5.3-codex-spark", repo.lastModelRateLimitKey)
|
||||
require.Greater(t, time.Until(repo.lastModelRateLimitedUntil), 6*24*time.Hour)
|
||||
}
|
||||
|
||||
func TestOpenAI429FastPath_SparkTransient429UsesShortFallback(t *testing.T) {
|
||||
repo := &oauth429RateLimitRepo{}
|
||||
rateLimits := NewRateLimitService(repo, nil, &config.Config{}, nil, nil)
|
||||
svc := &OpenAIGatewayService{rateLimitService: rateLimits}
|
||||
rateLimits.SetAccountRuntimeBlocker(svc)
|
||||
account := &Account{ID: 428, Platform: PlatformOpenAI, Type: AccountTypeOAuth}
|
||||
headers := http.Header{}
|
||||
headers.Set("x-codex-primary-used-percent", "37")
|
||||
headers.Set("x-codex-primary-reset-after-seconds", "604800")
|
||||
headers.Set("x-codex-primary-window-minutes", "10080")
|
||||
headers.Set("x-codex-secondary-used-percent", "20")
|
||||
headers.Set("x-codex-secondary-reset-after-seconds", "3600")
|
||||
headers.Set("x-codex-secondary-window-minutes", "300")
|
||||
|
||||
shouldDisable := svc.handleOpenAIAccountUpstreamError(
|
||||
context.Background(), account, http.StatusTooManyRequests, headers,
|
||||
[]byte(`{"error":{"type":"rate_limit_error","code":"rate_limit_exceeded"}}`),
|
||||
"gpt-5.3-codex-spark",
|
||||
)
|
||||
|
||||
require.False(t, shouldDisable)
|
||||
require.Equal(t, 1, repo.setModelRateLimitCalls)
|
||||
require.Less(t, time.Until(repo.lastModelRateLimitedUntil), time.Minute)
|
||||
require.Greater(t, time.Until(repo.lastModelRateLimitedUntil), time.Second)
|
||||
}
|
||||
|
||||
func TestOpenAIStream429_SparkQuotaUsesQuotaHeaders(t *testing.T) {
|
||||
repo := &oauth429RateLimitRepo{}
|
||||
rateLimits := NewRateLimitService(repo, nil, &config.Config{}, nil, nil)
|
||||
svc := &OpenAIGatewayService{rateLimitService: rateLimits}
|
||||
rateLimits.SetAccountRuntimeBlocker(svc)
|
||||
account := &Account{ID: 429, Platform: PlatformOpenAI, Type: AccountTypeOAuth}
|
||||
headers := http.Header{}
|
||||
headers.Set("x-codex-primary-used-percent", "100")
|
||||
headers.Set("x-codex-primary-reset-after-seconds", "604800")
|
||||
headers.Set("x-codex-primary-window-minutes", "10080")
|
||||
headers.Set("x-codex-secondary-used-percent", "20")
|
||||
headers.Set("x-codex-secondary-reset-after-seconds", "3600")
|
||||
headers.Set("x-codex-secondary-window-minutes", "300")
|
||||
payload := []byte(`{"type":"error","error":{"type":"rate_limit_error","code":"rate_limit_exceeded"}}`)
|
||||
|
||||
status, shouldDisable := svc.handleOpenAIStreamTerminalAccountSideEffects(
|
||||
nil, account, payload, "quota exhausted", headers, "gpt-5.3-codex-spark",
|
||||
)
|
||||
|
||||
require.Equal(t, http.StatusTooManyRequests, status)
|
||||
require.False(t, shouldDisable)
|
||||
require.Equal(t, 1, repo.setModelRateLimitCalls)
|
||||
require.Equal(t, "gpt-5.3-codex-spark", repo.lastModelRateLimitKey)
|
||||
require.Greater(t, time.Until(repo.lastModelRateLimitedUntil), 6*24*time.Hour)
|
||||
require.False(t, svc.isOpenAIAccountRuntimeBlocked(account))
|
||||
}
|
||||
|
||||
func TestOpenAIStreamFailover_Spark429KeepsModelScope(t *testing.T) {
|
||||
repo := &oauth429RateLimitRepo{}
|
||||
rateLimits := NewRateLimitService(repo, nil, &config.Config{}, nil, nil)
|
||||
svc := &OpenAIGatewayService{rateLimitService: rateLimits}
|
||||
rateLimits.SetAccountRuntimeBlocker(svc)
|
||||
account := &Account{ID: 432, Platform: PlatformOpenAI, Type: AccountTypeOAuth}
|
||||
headers := http.Header{}
|
||||
headers.Set("x-codex-primary-used-percent", "100")
|
||||
headers.Set("x-codex-primary-reset-after-seconds", "604800")
|
||||
headers.Set("x-codex-primary-window-minutes", "10080")
|
||||
headers.Set("x-codex-secondary-used-percent", "20")
|
||||
headers.Set("x-codex-secondary-reset-after-seconds", "3600")
|
||||
headers.Set("x-codex-secondary-window-minutes", "300")
|
||||
payload := []byte(`{"type":"error","error":{"type":"rate_limit_error","code":"rate_limit_exceeded"}}`)
|
||||
|
||||
failoverErr := svc.newOpenAIStreamFailoverErrorWithModel(
|
||||
nil, account, false, "", payload, "quota exhausted", "gpt-5.3-codex-spark", headers,
|
||||
)
|
||||
|
||||
require.Equal(t, http.StatusTooManyRequests, failoverErr.StatusCode)
|
||||
require.Equal(t, 0, repo.setRateLimitedCalls)
|
||||
require.Equal(t, 1, repo.setModelRateLimitCalls)
|
||||
require.Equal(t, "gpt-5.3-codex-spark", repo.lastModelRateLimitKey)
|
||||
require.False(t, svc.isOpenAIAccountRuntimeBlocked(account))
|
||||
}
|
||||
|
||||
func TestOpenAIWSErrorEvent_OrdinaryModelIgnoresHandshakeQuotaHeaders(t *testing.T) {
|
||||
repo := &oauth429RateLimitRepo{}
|
||||
rateLimits := NewRateLimitService(repo, nil, &config.Config{}, nil, nil)
|
||||
svc := &OpenAIGatewayService{rateLimitService: rateLimits}
|
||||
rateLimits.SetAccountRuntimeBlocker(svc)
|
||||
account := &Account{ID: 430, Platform: PlatformOpenAI, Type: AccountTypeOAuth}
|
||||
headers := http.Header{}
|
||||
headers.Set("x-codex-primary-used-percent", "100")
|
||||
headers.Set("x-codex-primary-reset-after-seconds", "604800")
|
||||
headers.Set("x-codex-primary-window-minutes", "10080")
|
||||
payload := []byte(`{"type":"error","error":{"type":"rate_limit_error","code":"rate_limit_exceeded"}}`)
|
||||
|
||||
svc.persistOpenAIWSRateLimitSignal(context.Background(), account, headers, payload, "rate_limit_exceeded", "rate_limit_error", "quota exhausted", "gpt-5.3-codex")
|
||||
|
||||
require.False(t, svc.isOpenAIAccountRuntimeBlocked(account))
|
||||
require.Zero(t, repo.setRateLimitedCalls)
|
||||
}
|
||||
|
||||
func TestOpenAIWSErrorEvent_SparkQuotaUsesHandshakeQuotaHeaders(t *testing.T) {
|
||||
repo := &oauth429RateLimitRepo{}
|
||||
rateLimits := NewRateLimitService(repo, nil, &config.Config{}, nil, nil)
|
||||
svc := &OpenAIGatewayService{rateLimitService: rateLimits}
|
||||
rateLimits.SetAccountRuntimeBlocker(svc)
|
||||
account := &Account{ID: 431, Platform: PlatformOpenAI, Type: AccountTypeOAuth}
|
||||
headers := http.Header{}
|
||||
headers.Set("x-codex-primary-used-percent", "100")
|
||||
headers.Set("x-codex-primary-reset-after-seconds", "604800")
|
||||
headers.Set("x-codex-primary-window-minutes", "10080")
|
||||
payload := []byte(`{"type":"error","error":{"type":"rate_limit_error","code":"rate_limit_exceeded"}}`)
|
||||
|
||||
svc.persistOpenAIWSRateLimitSignal(context.Background(), account, headers, payload, "rate_limit_exceeded", "rate_limit_error", "quota exhausted", "gpt-5.3-codex-spark")
|
||||
|
||||
require.Equal(t, 1, repo.setModelRateLimitCalls)
|
||||
require.Equal(t, "gpt-5.3-codex-spark", repo.lastModelRateLimitKey)
|
||||
require.Greater(t, time.Until(repo.lastModelRateLimitedUntil), 6*24*time.Hour)
|
||||
require.False(t, svc.isOpenAIAccountRuntimeBlocked(account))
|
||||
}
|
||||
|
||||
func TestOpenAI429FastPath_SparkShadowQuotaStaysModelScoped(t *testing.T) {
|
||||
repo := &oauth429RateLimitRepo{}
|
||||
rateLimits := NewRateLimitService(repo, nil, &config.Config{}, nil, nil)
|
||||
svc := &OpenAIGatewayService{rateLimitService: rateLimits}
|
||||
rateLimits.SetAccountRuntimeBlocker(svc)
|
||||
parentID := int64(426)
|
||||
shadow := &Account{ID: 427, Platform: PlatformOpenAI, Type: AccountTypeOAuth, ParentAccountID: &parentID, QuotaDimension: QuotaDimensionSpark}
|
||||
headers := http.Header{}
|
||||
headers.Set("x-codex-primary-used-percent", "100")
|
||||
headers.Set("x-codex-primary-reset-after-seconds", "604800")
|
||||
headers.Set("x-codex-primary-window-minutes", "10080")
|
||||
|
||||
shouldDisable := svc.handleOpenAIAccountUpstreamError(
|
||||
context.Background(), shadow, http.StatusTooManyRequests, headers,
|
||||
[]byte(`{"error":{"type":"rate_limit_error","code":"rate_limit_exceeded"}}`),
|
||||
"gpt-5.3-codex-spark",
|
||||
)
|
||||
|
||||
require.False(t, shouldDisable)
|
||||
require.False(t, svc.isOpenAIAccountRuntimeBlocked(shadow))
|
||||
require.Equal(t, 0, repo.setRateLimitedCalls)
|
||||
require.Equal(t, 1, repo.setModelRateLimitCalls)
|
||||
require.Equal(t, "gpt-5.3-codex-spark", repo.lastModelRateLimitKey)
|
||||
}
|
||||
|
||||
func TestOpenAI429FastPath_RetriesOAuthWhenNoQuotaSignalExists(t *testing.T) {
|
||||
svc := &OpenAIGatewayService{}
|
||||
account := &Account{ID: 424, Platform: PlatformOpenAI, Type: AccountTypeOAuth}
|
||||
|
||||
@@ -511,7 +511,7 @@ func (s *OpenAIGatewayService) handleChatBufferedStreamingResponse(
|
||||
}
|
||||
message := openAICompatFailedResponseMessage(finalResponse)
|
||||
if openAIStreamFailedEventShouldFailover(payload, message) {
|
||||
return nil, s.newOpenAIStreamFailoverError(c, account, false, requestID, payload, message, resp.Header)
|
||||
return nil, s.newOpenAIStreamFailoverErrorWithModel(c, account, false, requestID, payload, message, upstreamModel, resp.Header)
|
||||
}
|
||||
message = s.recordOpenAIStreamUpstreamError(c, account, false, requestID, "http_error", payload, message)
|
||||
// response.failed 到达在 HTTP 200 SSE 流上,无真实 HTTP 错误码;统一走语义
|
||||
@@ -766,7 +766,7 @@ func (s *OpenAIGatewayService) handleChatStreamingResponse(
|
||||
shouldFailover = openAIStreamErrorEventShouldFailover(payloadBytes, message)
|
||||
}
|
||||
if !clientOutputStarted && shouldFailover {
|
||||
streamFailoverErr = s.newOpenAIStreamFailoverError(c, account, false, requestID, payloadBytes, message, resp.Header)
|
||||
streamFailoverErr = s.newOpenAIStreamFailoverErrorWithModel(c, account, false, requestID, payloadBytes, message, upstreamModel, resp.Header)
|
||||
return true
|
||||
}
|
||||
message = s.recordOpenAIStreamUpstreamError(c, account, false, requestID, "http_error", payloadBytes, message)
|
||||
|
||||
@@ -605,7 +605,7 @@ func (s *OpenAIGatewayService) handleAnthropicBufferedStreamingResponse(
|
||||
}
|
||||
message := openAICompatFailedResponseMessage(finalResponse)
|
||||
if openAIStreamFailedEventShouldFailover(payload, message) {
|
||||
return nil, s.newOpenAIStreamFailoverError(c, account, false, requestID, payload, message, resp.Header)
|
||||
return nil, s.newOpenAIStreamFailoverErrorWithModel(c, account, false, requestID, payload, message, upstreamModel, resp.Header)
|
||||
}
|
||||
message = s.recordOpenAIStreamUpstreamError(c, account, false, requestID, "http_error", payload, message)
|
||||
// 统一走语义状态推断 + body 归一化(与 /v1/responses 路径一致),
|
||||
@@ -1038,7 +1038,7 @@ func (s *OpenAIGatewayService) handleAnthropicStreamingResponse(
|
||||
shouldFailover = openAIStreamErrorEventShouldFailover(payloadBytes, message)
|
||||
}
|
||||
if !clientOutputStarted && shouldFailover {
|
||||
streamFailoverErr = s.newOpenAIStreamFailoverError(c, account, false, requestID, payloadBytes, message, resp.Header)
|
||||
streamFailoverErr = s.newOpenAIStreamFailoverErrorWithModel(c, account, false, requestID, payloadBytes, message, upstreamModel, resp.Header)
|
||||
return true
|
||||
}
|
||||
message = s.recordOpenAIStreamUpstreamError(c, account, false, requestID, "http_error", payloadBytes, message)
|
||||
|
||||
@@ -1558,6 +1558,7 @@ func (s *OpenAIGatewayService) handleOpenAIStreamTerminalAccountSideEffects(
|
||||
payload []byte,
|
||||
message string,
|
||||
headers http.Header,
|
||||
canonicalModel ...string,
|
||||
) (int, bool) {
|
||||
statusCode := openAIStreamFailureStatus(payload, message)
|
||||
switch statusCode {
|
||||
@@ -1571,14 +1572,17 @@ func (s *OpenAIGatewayService) handleOpenAIStreamTerminalAccountSideEffects(
|
||||
if c != nil && c.Request != nil {
|
||||
ctx = c.Request.Context()
|
||||
}
|
||||
model := firstNonEmpty(canonicalModel...)
|
||||
if model == "" {
|
||||
model = firstNonEmpty(gjson.GetBytes(payload, "model").String(), gjson.GetBytes(payload, "response.model").String())
|
||||
}
|
||||
accountHeaders := headers
|
||||
if statusCode == http.StatusTooManyRequests {
|
||||
// The enclosing HTTP response succeeded. Its quota snapshot describes
|
||||
// normal account state and must not become the reset for a semantic 429
|
||||
// carried by a stream terminal event.
|
||||
accountHeaders = nil
|
||||
// 普通模型的流式 429 不能继承外层 HTTP 200 的全局 quota 快照;
|
||||
// 只有 OAuth/SetupToken 的 Spark 配额 429 才需要保留 headers 读取明确的 5h/7d reset。
|
||||
accountHeaders = openAIWSSemantic429Headers(account, model, headers)
|
||||
}
|
||||
return statusCode, s.handleOpenAIAccountUpstreamError(ctx, account, statusCode, accountHeaders, payload)
|
||||
return statusCode, s.handleOpenAIAccountUpstreamError(ctx, account, statusCode, accountHeaders, payload, model)
|
||||
default:
|
||||
return statusCode, false
|
||||
}
|
||||
@@ -1654,6 +1658,19 @@ func (s *OpenAIGatewayService) newOpenAIStreamFailoverError(
|
||||
payload []byte,
|
||||
message string,
|
||||
responseHeaders ...http.Header,
|
||||
) *UpstreamFailoverError {
|
||||
return s.newOpenAIStreamFailoverErrorWithModel(c, account, passthrough, upstreamRequestID, payload, message, "", responseHeaders...)
|
||||
}
|
||||
|
||||
func (s *OpenAIGatewayService) newOpenAIStreamFailoverErrorWithModel(
|
||||
c *gin.Context,
|
||||
account *Account,
|
||||
passthrough bool,
|
||||
upstreamRequestID string,
|
||||
payload []byte,
|
||||
message string,
|
||||
canonicalModel string,
|
||||
responseHeaders ...http.Header,
|
||||
) *UpstreamFailoverError {
|
||||
message = sanitizeUpstreamErrorMessage(strings.TrimSpace(message))
|
||||
if message == "" {
|
||||
@@ -1663,7 +1680,7 @@ func (s *OpenAIGatewayService) newOpenAIStreamFailoverError(
|
||||
if len(responseHeaders) > 0 && responseHeaders[0] != nil {
|
||||
headers = responseHeaders[0].Clone()
|
||||
}
|
||||
statusCode, shouldDisable := s.handleOpenAIStreamTerminalAccountSideEffects(c, account, payload, message, headers)
|
||||
statusCode, shouldDisable := s.handleOpenAIStreamTerminalAccountSideEffects(c, account, payload, message, headers, canonicalModel)
|
||||
// 流内 failed 事件承载于 HTTP 200;使用事件的语义状态更新账号健康,
|
||||
// 再由 failover 引擎按 StatusCode/RetryableOnSameAccount 决定恢复策略。
|
||||
message = s.recordOpenAIStreamUpstreamError(c, account, passthrough, upstreamRequestID, "failover", payload, message)
|
||||
@@ -1734,6 +1751,7 @@ func (s *OpenAIGatewayService) nonStreamingTerminalFailureFailover(
|
||||
terminalType string,
|
||||
payload []byte,
|
||||
message string,
|
||||
canonicalModel ...string,
|
||||
) *UpstreamFailoverError {
|
||||
if account == nil || IsResponseCommitted(c) {
|
||||
return nil
|
||||
@@ -1751,7 +1769,7 @@ func (s *OpenAIGatewayService) nonStreamingTerminalFailureFailover(
|
||||
headers = resp.Header
|
||||
upstreamRequestID = strings.TrimSpace(resp.Header.Get("x-request-id"))
|
||||
}
|
||||
return s.newOpenAIStreamFailoverError(c, account, passthrough, upstreamRequestID, payload, message, headers)
|
||||
return s.newOpenAIStreamFailoverErrorWithModel(c, account, passthrough, upstreamRequestID, payload, message, firstNonEmpty(canonicalModel...), headers)
|
||||
}
|
||||
|
||||
func (s *OpenAIGatewayService) handleStreamingResponsePassthrough(
|
||||
@@ -1873,7 +1891,7 @@ func (s *OpenAIGatewayService) handleStreamingResponsePassthrough(
|
||||
return
|
||||
}
|
||||
if bareErrorAccountSideEffectsPending {
|
||||
s.handleOpenAIStreamTerminalAccountSideEffects(c, account, bareErrorPayload, failedMessage, resp.Header)
|
||||
s.handleOpenAIStreamTerminalAccountSideEffects(c, account, bareErrorPayload, failedMessage, resp.Header, mappedModel)
|
||||
bareErrorAccountSideEffectsPending = false
|
||||
}
|
||||
if clientDisconnected || !writePendingLines() {
|
||||
@@ -2005,7 +2023,7 @@ func (s *OpenAIGatewayService) handleStreamingResponsePassthrough(
|
||||
// account health; EOF synthesis applies the pending effect.
|
||||
bareErrorAccountSideEffectsPending = true
|
||||
} else {
|
||||
s.handleOpenAIStreamTerminalAccountSideEffects(c, account, dataBytes, failedMessage, resp.Header)
|
||||
s.handleOpenAIStreamTerminalAccountSideEffects(c, account, dataBytes, failedMessage, resp.Header, mappedModel)
|
||||
bareErrorAccountSideEffectsPending = false
|
||||
}
|
||||
}
|
||||
@@ -2020,7 +2038,7 @@ func (s *OpenAIGatewayService) handleStreamingResponsePassthrough(
|
||||
}
|
||||
if shouldFailover {
|
||||
return resultWithUsage(),
|
||||
s.newOpenAIStreamFailoverError(c, account, true, upstreamRequestID, dataBytes, failedMessage, resp.Header)
|
||||
s.newOpenAIStreamFailoverErrorWithModel(c, account, true, upstreamRequestID, dataBytes, failedMessage, mappedModel, resp.Header)
|
||||
}
|
||||
if !cyberHit && !sawBareError {
|
||||
if status, errType, errMsg, matched := applyOpenAIStreamFailedErrorPassthroughRule(c, account.Platform, dataBytes, failedMessage); matched {
|
||||
@@ -2273,7 +2291,7 @@ func (s *OpenAIGatewayService) handlePassthroughSSEToJSON(resp *http.Response, c
|
||||
if compactErr := newOpenAICompactFallbackSignal(c, terminalPayload, msg); compactErr != nil {
|
||||
return nil, compactErr
|
||||
}
|
||||
if failoverErr := s.nonStreamingTerminalFailureFailover(c, resp, account, true, terminalType, terminalPayload, msg); failoverErr != nil {
|
||||
if failoverErr := s.nonStreamingTerminalFailureFailover(c, resp, account, true, terminalType, terminalPayload, msg, mappedModel); failoverErr != nil {
|
||||
return nil, failoverErr
|
||||
}
|
||||
return nil, s.writeOpenAINonStreamingProtocolError(resp, c, msg)
|
||||
|
||||
@@ -372,7 +372,7 @@ func (s *OpenAIGatewayService) handleStreamingResponseWithReasoning(ctx context.
|
||||
completeGuardedEvent(true)
|
||||
}
|
||||
if codexFailureTerminal && sawBareError && !sawResponseFailed && bareErrorAccountSideEffectsPending {
|
||||
s.handleOpenAIStreamTerminalAccountSideEffects(c, account, bareErrorPayload, failedMessage, resp.Header)
|
||||
s.handleOpenAIStreamTerminalAccountSideEffects(c, account, bareErrorPayload, failedMessage, resp.Header, mappedModel)
|
||||
bareErrorAccountSideEffectsPending = false
|
||||
}
|
||||
if codexFailureTerminal && sawBareError && !sawResponseFailed && !clientDisconnected {
|
||||
@@ -557,7 +557,7 @@ func (s *OpenAIGatewayService) handleStreamingResponseWithReasoning(ctx context.
|
||||
// Defer account health updates so the pair is applied once.
|
||||
bareErrorAccountSideEffectsPending = true
|
||||
} else {
|
||||
s.handleOpenAIStreamTerminalAccountSideEffects(c, account, dataBytes, failedMessage, resp.Header)
|
||||
s.handleOpenAIStreamTerminalAccountSideEffects(c, account, dataBytes, failedMessage, resp.Header, mappedModel)
|
||||
bareErrorAccountSideEffectsPending = false
|
||||
}
|
||||
}
|
||||
@@ -572,7 +572,7 @@ func (s *OpenAIGatewayService) handleStreamingResponseWithReasoning(ctx context.
|
||||
}
|
||||
if shouldFailover {
|
||||
sawFailedEvent = true
|
||||
streamEarlyErr = s.newOpenAIStreamFailoverError(c, account, false, upstreamRequestID, dataBytes, failedMessage, resp.Header)
|
||||
streamEarlyErr = s.newOpenAIStreamFailoverErrorWithModel(c, account, false, upstreamRequestID, dataBytes, failedMessage, mappedModel, resp.Header)
|
||||
return
|
||||
}
|
||||
if !cyberHit && !sawBareError {
|
||||
@@ -1684,7 +1684,7 @@ func (s *OpenAIGatewayService) handleSSEToJSON(resp *http.Response, c *gin.Conte
|
||||
if compactErr := newOpenAICompactFallbackSignal(c, terminalPayload, msg); compactErr != nil {
|
||||
return nil, compactErr
|
||||
}
|
||||
if failoverErr := s.nonStreamingTerminalFailureFailover(c, resp, account, false, terminalType, terminalPayload, msg); failoverErr != nil {
|
||||
if failoverErr := s.nonStreamingTerminalFailureFailover(c, resp, account, false, terminalType, terminalPayload, msg, mappedModel); failoverErr != nil {
|
||||
return nil, failoverErr
|
||||
}
|
||||
return nil, s.writeOpenAINonStreamingProtocolError(resp, c, msg)
|
||||
|
||||
@@ -866,7 +866,7 @@ func (s *OpenAIGatewayService) ProxyResponsesWebSocketFromClient(
|
||||
)
|
||||
var dialErr *openAIWSDialError
|
||||
if errors.As(acquireErr, &dialErr) && dialErr != nil && dialErr.StatusCode == http.StatusTooManyRequests {
|
||||
s.persistOpenAIWSRateLimitSignal(ctx, account, dialErr.ResponseHeaders, nil, "rate_limit_exceeded", "rate_limit_error", strings.TrimSpace(acquireErr.Error()))
|
||||
s.persistOpenAIWSRateLimitSignal(ctx, account, dialErr.ResponseHeaders, nil, "rate_limit_exceeded", "rate_limit_error", strings.TrimSpace(acquireErr.Error()), canonicalModel)
|
||||
return nil, s.newOpenAIWSRateLimitFailoverError(account, dialErr.ResponseHeaders, nil, acquireErr.Error())
|
||||
}
|
||||
if errors.Is(acquireErr, errOpenAIWSPreferredConnUnavailable) {
|
||||
@@ -1019,7 +1019,7 @@ func (s *OpenAIGatewayService) ProxyResponsesWebSocketFromClient(
|
||||
}
|
||||
}
|
||||
}
|
||||
s.persistOpenAIWSRateLimitSignal(ctx, account, lease.HandshakeHeaders(), upstreamMessage, errCodeRaw, errTypeRaw, errMsgRaw)
|
||||
s.persistOpenAIWSRateLimitSignal(ctx, account, lease.HandshakeHeaders(), upstreamMessage, errCodeRaw, errTypeRaw, errMsgRaw, mappedModel)
|
||||
fallbackReason, _ := classifyOpenAIWSErrorEventFromRaw(errCodeRaw, errTypeRaw, errMsgRaw)
|
||||
errCode, errType, errMessage := summarizeOpenAIWSErrorEventFieldsFromRaw(errCodeRaw, errTypeRaw, errMsgRaw)
|
||||
recoverablePrevNotFound := fallbackReason == openAIWSIngressStagePreviousResponseNotFound &&
|
||||
|
||||
@@ -131,7 +131,8 @@ func (s *OpenAIGatewayService) performOpenAIWSGeneratePrewarm(
|
||||
|
||||
if eventType == "error" {
|
||||
errCodeRaw, errTypeRaw, errMsgRaw := parseOpenAIWSErrorEventFields(message)
|
||||
s.persistOpenAIWSRateLimitSignal(ctx, account, lease.HandshakeHeaders(), message, errCodeRaw, errTypeRaw, errMsgRaw)
|
||||
prewarmModel, _ := reqBody["model"].(string)
|
||||
s.persistOpenAIWSRateLimitSignal(ctx, account, lease.HandshakeHeaders(), message, errCodeRaw, errTypeRaw, errMsgRaw, prewarmModel)
|
||||
errMsg := strings.TrimSpace(errMsgRaw)
|
||||
if errMsg == "" {
|
||||
errMsg = "OpenAI websocket prewarm error"
|
||||
@@ -324,6 +325,9 @@ func (s *OpenAIGatewayService) handleOpenAIWSErrorEventTransientFailure(ctx cont
|
||||
}
|
||||
status := openAIWSPayloadTransientStatus(payload)
|
||||
if status != 0 {
|
||||
if status == http.StatusTooManyRequests {
|
||||
headers = openAIWSSemantic429Headers(account, canonicalModel, headers)
|
||||
}
|
||||
s.handleOpenAIAccountUpstreamError(ctx, account, status, headers, payload, canonicalModel)
|
||||
}
|
||||
}
|
||||
@@ -336,13 +340,13 @@ func (s *OpenAIGatewayService) handleOpenAIWSFailureAccountSideEffects(ctx conte
|
||||
status := openAIStreamFailureStatus(payload, message)
|
||||
switch status {
|
||||
case http.StatusUnauthorized, http.StatusTooManyRequests, 529:
|
||||
s.handleOpenAIStreamTerminalAccountSideEffects(nil, account, payload, message, headers)
|
||||
s.handleOpenAIStreamTerminalAccountSideEffects(nil, account, payload, message, headers, canonicalModel)
|
||||
return true
|
||||
case http.StatusForbidden:
|
||||
if !openAIStream403AccountFailure(payload, message) {
|
||||
return false
|
||||
}
|
||||
s.handleOpenAIStreamTerminalAccountSideEffects(nil, account, payload, message, headers)
|
||||
s.handleOpenAIStreamTerminalAccountSideEffects(nil, account, payload, message, headers, canonicalModel)
|
||||
return true
|
||||
}
|
||||
|
||||
@@ -683,14 +687,31 @@ func isOpenAIWSRateLimitError(codeRaw, errTypeRaw, msgRaw string) bool {
|
||||
return false
|
||||
}
|
||||
|
||||
func (s *OpenAIGatewayService) persistOpenAIWSRateLimitSignal(ctx context.Context, account *Account, headers http.Header, responseBody []byte, codeRaw, errTypeRaw, msgRaw string) {
|
||||
func (s *OpenAIGatewayService) persistOpenAIWSRateLimitSignal(ctx context.Context, account *Account, headers http.Header, responseBody []byte, codeRaw, errTypeRaw, msgRaw string, canonicalModel ...string) {
|
||||
if s == nil || s.rateLimitService == nil || account == nil || account.Platform != PlatformOpenAI {
|
||||
return
|
||||
}
|
||||
if !isOpenAIWSRateLimitError(codeRaw, errTypeRaw, msgRaw) {
|
||||
return
|
||||
}
|
||||
s.handleOpenAIAccountUpstreamError(ctx, account, http.StatusTooManyRequests, headers, responseBody)
|
||||
model := firstNonEmpty(canonicalModel...)
|
||||
if model == "" {
|
||||
model = firstNonEmpty(gjson.GetBytes(responseBody, "model").String(), gjson.GetBytes(responseBody, "response.model").String())
|
||||
}
|
||||
// 非空 responseBody 表示已建立连接后收到的语义错误事件;握手响应头
|
||||
// 可能只是成功连接时的全局快照,不能用于普通模型的 429 账号级限流。
|
||||
// 实际拨号 HTTP 429 使用 nil responseBody,必须保留响应头。
|
||||
if len(responseBody) > 0 {
|
||||
headers = openAIWSSemantic429Headers(account, model, headers)
|
||||
}
|
||||
s.handleOpenAIAccountUpstreamError(ctx, account, http.StatusTooManyRequests, headers, responseBody, model)
|
||||
}
|
||||
|
||||
func openAIWSSemantic429Headers(account *Account, model string, headers http.Header) http.Header {
|
||||
if isCodexSparkModel(model) && isOpenAIOAuthAccount(account) {
|
||||
return headers
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *OpenAIGatewayService) newOpenAIWSRateLimitFailoverError(account *Account, headers http.Header, responseBody []byte, message string) *UpstreamFailoverError {
|
||||
|
||||
@@ -245,7 +245,7 @@ func (s *OpenAIGatewayService) forwardOpenAIWSV2(
|
||||
)
|
||||
var dialErr *openAIWSDialError
|
||||
if errors.As(err, &dialErr) && dialErr != nil && dialErr.StatusCode == http.StatusTooManyRequests {
|
||||
s.persistOpenAIWSRateLimitSignal(ctx, account, dialErr.ResponseHeaders, nil, "rate_limit_exceeded", "rate_limit_error", strings.TrimSpace(err.Error()))
|
||||
s.persistOpenAIWSRateLimitSignal(ctx, account, dialErr.ResponseHeaders, nil, "rate_limit_exceeded", "rate_limit_error", strings.TrimSpace(err.Error()), mappedModel)
|
||||
}
|
||||
return nil, wrapOpenAIWSFallback(classifyOpenAIWSAcquireError(err), err)
|
||||
}
|
||||
@@ -601,7 +601,7 @@ func (s *OpenAIGatewayService) forwardOpenAIWSV2(
|
||||
if eventType == "error" {
|
||||
s.handleOpenAIWSErrorEventTransientFailure(ctx, account, mappedModel, lease.HandshakeHeaders(), message)
|
||||
errCodeRaw, errTypeRaw, errMsgRaw := parseOpenAIWSErrorEventFields(message)
|
||||
s.persistOpenAIWSRateLimitSignal(ctx, account, lease.HandshakeHeaders(), message, errCodeRaw, errTypeRaw, errMsgRaw)
|
||||
s.persistOpenAIWSRateLimitSignal(ctx, account, lease.HandshakeHeaders(), message, errCodeRaw, errTypeRaw, errMsgRaw, mappedModel)
|
||||
errMsg := strings.TrimSpace(errMsgRaw)
|
||||
if errMsg == "" {
|
||||
errMsg = "Upstream websocket error"
|
||||
|
||||
@@ -695,7 +695,7 @@ func (s *OpenAIGatewayService) proxyOpenAIWSHTTPBridgeTurn(
|
||||
if account.Platform == PlatformGrok {
|
||||
return nil, newOpenAIUpstreamFailoverError(statusCode, resp.Header, upstreamMessage, errMessage, false)
|
||||
}
|
||||
return nil, s.newOpenAIStreamFailoverError(c, account, true, resp.Header.Get("x-request-id"), upstreamMessage, errMessage, resp.Header)
|
||||
return nil, s.newOpenAIStreamFailoverErrorWithModel(c, account, true, resp.Header.Get("x-request-id"), upstreamMessage, errMessage, mappedModel, resp.Header)
|
||||
}
|
||||
if account.Platform != PlatformGrok && !failureAccountSideEffectsApplied {
|
||||
if eventType == "response.failed" || (!officialOpenAIResponses && shouldFailover && !requestScopedCapacity) {
|
||||
|
||||
@@ -902,7 +902,7 @@ func (s *OpenAIGatewayService) proxyResponsesWebSocketV2Passthrough(
|
||||
)
|
||||
s.handleOpenAIWSDialTransientFailure(ctx, account, capturedSessionModel, dialErr)
|
||||
if statusCode == http.StatusTooManyRequests {
|
||||
s.persistOpenAIWSRateLimitSignal(ctx, account, handshakeHeaders, nil, "rate_limit_exceeded", "rate_limit_error", strings.TrimSpace(err.Error()))
|
||||
s.persistOpenAIWSRateLimitSignal(ctx, account, handshakeHeaders, nil, "rate_limit_exceeded", "rate_limit_error", strings.TrimSpace(err.Error()), capturedSessionModel)
|
||||
return s.newOpenAIWSRateLimitFailoverError(account, handshakeHeaders, nil, err.Error())
|
||||
}
|
||||
return s.mapOpenAIWSPassthroughDialError(err, statusCode, handshakeHeaders)
|
||||
@@ -1291,7 +1291,7 @@ func (s *OpenAIGatewayService) proxyResponsesWebSocketV2Passthrough(
|
||||
if wroteDownstream || !isOpenAIWSRateLimitError(errCodeRaw, errTypeRaw, errMsgRaw) {
|
||||
return nil
|
||||
}
|
||||
s.persistOpenAIWSRateLimitSignal(ctx, account, handshakeHeaders, payload, errCodeRaw, errTypeRaw, errMsgRaw)
|
||||
s.persistOpenAIWSRateLimitSignal(ctx, account, handshakeHeaders, payload, errCodeRaw, errTypeRaw, errMsgRaw, capturedSessionModel)
|
||||
logOpenAIWSV2Passthrough(
|
||||
"relay_rate_limit_failover account_id=%d err_code=%s err_type=%s err_message=%s",
|
||||
account.ID,
|
||||
|
||||
@@ -2192,6 +2192,43 @@ func (s *RateLimitService) HandleOpenAIImageRateLimit(ctx context.Context, accou
|
||||
return true
|
||||
}
|
||||
|
||||
// HandleOpenAICodexSparkRateLimit 将 Spark 独立配额窗口记录为模型级限流。
|
||||
// Spark 的 x-codex-* 使用率和 reset 时间只代表 Spark 模型维度,不能写入账号级
|
||||
// RateLimitResetAt,否则同一 OAuth 账号上的其他模型也会被错误停调。
|
||||
func (s *RateLimitService) HandleOpenAICodexSparkRateLimit(ctx context.Context, account *Account, requestedModel string, statusCode int, headers http.Header, responseBody []byte) bool {
|
||||
if s == nil || account == nil || s.accountRepo == nil || statusCode != http.StatusTooManyRequests || !isOpenAIOAuthAccount(account) {
|
||||
return false
|
||||
}
|
||||
if !isCodexSparkModel(requestedModel) || !account.ShouldHandleErrorCode(statusCode) {
|
||||
return false
|
||||
}
|
||||
|
||||
modelKey := normalizeCodexModel(modelRateLimitKeyForUpstreamModelNotFound(ctx, account, requestedModel))
|
||||
if modelKey == "" {
|
||||
return false
|
||||
}
|
||||
now := time.Now()
|
||||
disposition, resetAt := classifyOpenAIOAuth429(headers, responseBody)
|
||||
// Spark 只有明确耗尽 5h/7d 窗口时才能使用上游长 reset;普通瞬时 429
|
||||
// 即使携带全局 reset 头,也只能使用短时回避,避免错误冷却数天。
|
||||
if disposition != openAIOAuth429Quota5h && disposition != openAIOAuth429Quota7d {
|
||||
resetAt = nil
|
||||
}
|
||||
if resetAt == nil || !resetAt.After(now) {
|
||||
cooldown, ok := s.get429FallbackCooldown(ctx, account)
|
||||
if !ok || cooldown <= 0 {
|
||||
cooldown = time.Duration(defaultRateLimit429CooldownSeconds) * time.Second
|
||||
}
|
||||
reset := now.Add(cooldown)
|
||||
resetAt = &reset
|
||||
}
|
||||
if err := s.accountRepo.SetModelRateLimit(ctx, account.ID, modelKey, *resetAt, openAICodexSparkRateLimitReason); err != nil {
|
||||
slog.Warn("openai_codex_spark_model_rate_limit_set_failed", "account_id", account.ID, "model", modelKey, "error", err)
|
||||
}
|
||||
slog.Info("openai_codex_spark_model_rate_limited", "account_id", account.ID, "model", modelKey, "reset_at", *resetAt)
|
||||
return true
|
||||
}
|
||||
|
||||
func (s *RateLimitService) HandleOpenAIImageCapabilityLoss(ctx context.Context, account *Account, statusCode int, responseBody []byte) bool {
|
||||
if s == nil || account == nil || s.accountRepo == nil {
|
||||
return false
|
||||
|
||||
Reference in New Issue
Block a user