Merge pull request #6358 from Wei-Shaw/fix/openai-spark-model-429-rate-limit

fix(openai): scope Spark quota 429 to model
This commit is contained in:
Wesley Liddick
2026-08-29 15:44:37 +08:00
committed by GitHub
13 changed files with 294 additions and 33 deletions
@@ -12,6 +12,7 @@ const (
modelRateLimitsKey = "model_rate_limits"
antigravityGeminiModelRateLimitKey = "antigravity:gemini"
openAIImageGenerationRateLimitKey = "openai:image_generation"
openAICodexSparkRateLimitReason = "openai_codex_spark_rate_limit"
// anthropicFableRateLimitKey 是 Anthropic 7d_oi(Fable 专属 7d 窗口)限流的
// 家族级 scope:命中后所有 Fable 变体(含 [1m] 等后缀)都不再调度到该账号。
anthropicFableRateLimitKey = "claude-fable-5"
@@ -162,6 +162,10 @@ func (s *OpenAIGatewayService) handleOpenAIAccountUpstreamError(ctx context.Cont
s.rateLimitService.HandleTempUnschedulable(stateCtx, account, statusCode, responseBody, canonicalModel[0]) {
return true
}
if statusCode == http.StatusTooManyRequests && s.rateLimitService != nil && len(canonicalModel) > 0 &&
s.rateLimitService.HandleOpenAICodexSparkRateLimit(stateCtx, account, canonicalModel[0], statusCode, headers, responseBody) {
return false
}
if statusCode == http.StatusTooManyRequests {
s.markOpenAIOAuth429RateLimited(stateCtx, account, headers, responseBody)
}
@@ -15,8 +15,11 @@ import (
type oauth429RateLimitRepo struct {
mockAccountRepoForGemini
setRateLimitedCalls int
lastRateLimitedUntil time.Time
setRateLimitedCalls int
lastRateLimitedUntil time.Time
setModelRateLimitCalls int
lastModelRateLimitKey string
lastModelRateLimitedUntil time.Time
}
func (r *oauth429RateLimitRepo) SetRateLimited(_ context.Context, _ int64, until time.Time) error {
@@ -25,6 +28,13 @@ func (r *oauth429RateLimitRepo) SetRateLimited(_ context.Context, _ int64, until
return nil
}
func (r *oauth429RateLimitRepo) SetModelRateLimit(_ context.Context, _ int64, scope string, until time.Time, _ ...string) error {
r.setModelRateLimitCalls++
r.lastModelRateLimitKey = scope
r.lastModelRateLimitedUntil = until
return nil
}
func TestOpenAI429FastPath_KeepsOAuthAccountSchedulableDuringRetryWindow(t *testing.T) {
repo := &oauth429RateLimitRepo{}
rateLimits := NewRateLimitService(repo, nil, &config.Config{}, nil, nil)
@@ -85,6 +95,176 @@ func TestOpenAI429FastPath_BlocksOAuthImmediatelyWhenSevenDayQuotaIsExhausted(t
require.False(t, svc.ShouldRetryOpenAIOAuth429(account, headers, nil))
}
func TestOpenAI429FastPath_SparkQuotaOnlyBlocksSparkModel(t *testing.T) {
repo := &oauth429RateLimitRepo{}
rateLimits := NewRateLimitService(repo, nil, &config.Config{}, nil, nil)
svc := &OpenAIGatewayService{rateLimitService: rateLimits}
rateLimits.SetAccountRuntimeBlocker(svc)
account := &Account{ID: 425, Platform: PlatformOpenAI, Type: AccountTypeOAuth}
headers := http.Header{}
headers.Set("x-codex-primary-used-percent", "100")
headers.Set("x-codex-primary-reset-after-seconds", "604800")
headers.Set("x-codex-primary-window-minutes", "10080")
headers.Set("x-codex-secondary-used-percent", "20")
headers.Set("x-codex-secondary-reset-after-seconds", "3600")
headers.Set("x-codex-secondary-window-minutes", "300")
shouldDisable := svc.handleOpenAIAccountUpstreamError(
context.Background(), account, http.StatusTooManyRequests, headers,
[]byte(`{"error":{"type":"rate_limit_error","code":"rate_limit_exceeded"}}`),
"gpt-5.3-codex-spark",
)
require.False(t, shouldDisable)
require.False(t, svc.isOpenAIAccountRuntimeBlocked(account), "Spark quota must not create an account runtime block")
require.Equal(t, 0, repo.setRateLimitedCalls, "Spark quota must not persist account-level rate limit")
require.Equal(t, 1, repo.setModelRateLimitCalls)
require.Equal(t, "gpt-5.3-codex-spark", repo.lastModelRateLimitKey)
require.Greater(t, time.Until(repo.lastModelRateLimitedUntil), 6*24*time.Hour)
}
func TestOpenAI429FastPath_SparkTransient429UsesShortFallback(t *testing.T) {
repo := &oauth429RateLimitRepo{}
rateLimits := NewRateLimitService(repo, nil, &config.Config{}, nil, nil)
svc := &OpenAIGatewayService{rateLimitService: rateLimits}
rateLimits.SetAccountRuntimeBlocker(svc)
account := &Account{ID: 428, Platform: PlatformOpenAI, Type: AccountTypeOAuth}
headers := http.Header{}
headers.Set("x-codex-primary-used-percent", "37")
headers.Set("x-codex-primary-reset-after-seconds", "604800")
headers.Set("x-codex-primary-window-minutes", "10080")
headers.Set("x-codex-secondary-used-percent", "20")
headers.Set("x-codex-secondary-reset-after-seconds", "3600")
headers.Set("x-codex-secondary-window-minutes", "300")
shouldDisable := svc.handleOpenAIAccountUpstreamError(
context.Background(), account, http.StatusTooManyRequests, headers,
[]byte(`{"error":{"type":"rate_limit_error","code":"rate_limit_exceeded"}}`),
"gpt-5.3-codex-spark",
)
require.False(t, shouldDisable)
require.Equal(t, 1, repo.setModelRateLimitCalls)
require.Less(t, time.Until(repo.lastModelRateLimitedUntil), time.Minute)
require.Greater(t, time.Until(repo.lastModelRateLimitedUntil), time.Second)
}
func TestOpenAIStream429_SparkQuotaUsesQuotaHeaders(t *testing.T) {
repo := &oauth429RateLimitRepo{}
rateLimits := NewRateLimitService(repo, nil, &config.Config{}, nil, nil)
svc := &OpenAIGatewayService{rateLimitService: rateLimits}
rateLimits.SetAccountRuntimeBlocker(svc)
account := &Account{ID: 429, Platform: PlatformOpenAI, Type: AccountTypeOAuth}
headers := http.Header{}
headers.Set("x-codex-primary-used-percent", "100")
headers.Set("x-codex-primary-reset-after-seconds", "604800")
headers.Set("x-codex-primary-window-minutes", "10080")
headers.Set("x-codex-secondary-used-percent", "20")
headers.Set("x-codex-secondary-reset-after-seconds", "3600")
headers.Set("x-codex-secondary-window-minutes", "300")
payload := []byte(`{"type":"error","error":{"type":"rate_limit_error","code":"rate_limit_exceeded"}}`)
status, shouldDisable := svc.handleOpenAIStreamTerminalAccountSideEffects(
nil, account, payload, "quota exhausted", headers, "gpt-5.3-codex-spark",
)
require.Equal(t, http.StatusTooManyRequests, status)
require.False(t, shouldDisable)
require.Equal(t, 1, repo.setModelRateLimitCalls)
require.Equal(t, "gpt-5.3-codex-spark", repo.lastModelRateLimitKey)
require.Greater(t, time.Until(repo.lastModelRateLimitedUntil), 6*24*time.Hour)
require.False(t, svc.isOpenAIAccountRuntimeBlocked(account))
}
func TestOpenAIStreamFailover_Spark429KeepsModelScope(t *testing.T) {
repo := &oauth429RateLimitRepo{}
rateLimits := NewRateLimitService(repo, nil, &config.Config{}, nil, nil)
svc := &OpenAIGatewayService{rateLimitService: rateLimits}
rateLimits.SetAccountRuntimeBlocker(svc)
account := &Account{ID: 432, Platform: PlatformOpenAI, Type: AccountTypeOAuth}
headers := http.Header{}
headers.Set("x-codex-primary-used-percent", "100")
headers.Set("x-codex-primary-reset-after-seconds", "604800")
headers.Set("x-codex-primary-window-minutes", "10080")
headers.Set("x-codex-secondary-used-percent", "20")
headers.Set("x-codex-secondary-reset-after-seconds", "3600")
headers.Set("x-codex-secondary-window-minutes", "300")
payload := []byte(`{"type":"error","error":{"type":"rate_limit_error","code":"rate_limit_exceeded"}}`)
failoverErr := svc.newOpenAIStreamFailoverErrorWithModel(
nil, account, false, "", payload, "quota exhausted", "gpt-5.3-codex-spark", headers,
)
require.Equal(t, http.StatusTooManyRequests, failoverErr.StatusCode)
require.Equal(t, 0, repo.setRateLimitedCalls)
require.Equal(t, 1, repo.setModelRateLimitCalls)
require.Equal(t, "gpt-5.3-codex-spark", repo.lastModelRateLimitKey)
require.False(t, svc.isOpenAIAccountRuntimeBlocked(account))
}
func TestOpenAIWSErrorEvent_OrdinaryModelIgnoresHandshakeQuotaHeaders(t *testing.T) {
repo := &oauth429RateLimitRepo{}
rateLimits := NewRateLimitService(repo, nil, &config.Config{}, nil, nil)
svc := &OpenAIGatewayService{rateLimitService: rateLimits}
rateLimits.SetAccountRuntimeBlocker(svc)
account := &Account{ID: 430, Platform: PlatformOpenAI, Type: AccountTypeOAuth}
headers := http.Header{}
headers.Set("x-codex-primary-used-percent", "100")
headers.Set("x-codex-primary-reset-after-seconds", "604800")
headers.Set("x-codex-primary-window-minutes", "10080")
payload := []byte(`{"type":"error","error":{"type":"rate_limit_error","code":"rate_limit_exceeded"}}`)
svc.persistOpenAIWSRateLimitSignal(context.Background(), account, headers, payload, "rate_limit_exceeded", "rate_limit_error", "quota exhausted", "gpt-5.3-codex")
require.False(t, svc.isOpenAIAccountRuntimeBlocked(account))
require.Zero(t, repo.setRateLimitedCalls)
}
func TestOpenAIWSErrorEvent_SparkQuotaUsesHandshakeQuotaHeaders(t *testing.T) {
repo := &oauth429RateLimitRepo{}
rateLimits := NewRateLimitService(repo, nil, &config.Config{}, nil, nil)
svc := &OpenAIGatewayService{rateLimitService: rateLimits}
rateLimits.SetAccountRuntimeBlocker(svc)
account := &Account{ID: 431, Platform: PlatformOpenAI, Type: AccountTypeOAuth}
headers := http.Header{}
headers.Set("x-codex-primary-used-percent", "100")
headers.Set("x-codex-primary-reset-after-seconds", "604800")
headers.Set("x-codex-primary-window-minutes", "10080")
payload := []byte(`{"type":"error","error":{"type":"rate_limit_error","code":"rate_limit_exceeded"}}`)
svc.persistOpenAIWSRateLimitSignal(context.Background(), account, headers, payload, "rate_limit_exceeded", "rate_limit_error", "quota exhausted", "gpt-5.3-codex-spark")
require.Equal(t, 1, repo.setModelRateLimitCalls)
require.Equal(t, "gpt-5.3-codex-spark", repo.lastModelRateLimitKey)
require.Greater(t, time.Until(repo.lastModelRateLimitedUntil), 6*24*time.Hour)
require.False(t, svc.isOpenAIAccountRuntimeBlocked(account))
}
func TestOpenAI429FastPath_SparkShadowQuotaStaysModelScoped(t *testing.T) {
repo := &oauth429RateLimitRepo{}
rateLimits := NewRateLimitService(repo, nil, &config.Config{}, nil, nil)
svc := &OpenAIGatewayService{rateLimitService: rateLimits}
rateLimits.SetAccountRuntimeBlocker(svc)
parentID := int64(426)
shadow := &Account{ID: 427, Platform: PlatformOpenAI, Type: AccountTypeOAuth, ParentAccountID: &parentID, QuotaDimension: QuotaDimensionSpark}
headers := http.Header{}
headers.Set("x-codex-primary-used-percent", "100")
headers.Set("x-codex-primary-reset-after-seconds", "604800")
headers.Set("x-codex-primary-window-minutes", "10080")
shouldDisable := svc.handleOpenAIAccountUpstreamError(
context.Background(), shadow, http.StatusTooManyRequests, headers,
[]byte(`{"error":{"type":"rate_limit_error","code":"rate_limit_exceeded"}}`),
"gpt-5.3-codex-spark",
)
require.False(t, shouldDisable)
require.False(t, svc.isOpenAIAccountRuntimeBlocked(shadow))
require.Equal(t, 0, repo.setRateLimitedCalls)
require.Equal(t, 1, repo.setModelRateLimitCalls)
require.Equal(t, "gpt-5.3-codex-spark", repo.lastModelRateLimitKey)
}
func TestOpenAI429FastPath_RetriesOAuthWhenNoQuotaSignalExists(t *testing.T) {
svc := &OpenAIGatewayService{}
account := &Account{ID: 424, Platform: PlatformOpenAI, Type: AccountTypeOAuth}
@@ -511,7 +511,7 @@ func (s *OpenAIGatewayService) handleChatBufferedStreamingResponse(
}
message := openAICompatFailedResponseMessage(finalResponse)
if openAIStreamFailedEventShouldFailover(payload, message) {
return nil, s.newOpenAIStreamFailoverError(c, account, false, requestID, payload, message, resp.Header)
return nil, s.newOpenAIStreamFailoverErrorWithModel(c, account, false, requestID, payload, message, upstreamModel, resp.Header)
}
message = s.recordOpenAIStreamUpstreamError(c, account, false, requestID, "http_error", payload, message)
// response.failed 到达在 HTTP 200 SSE 流上,无真实 HTTP 错误码;统一走语义
@@ -766,7 +766,7 @@ func (s *OpenAIGatewayService) handleChatStreamingResponse(
shouldFailover = openAIStreamErrorEventShouldFailover(payloadBytes, message)
}
if !clientOutputStarted && shouldFailover {
streamFailoverErr = s.newOpenAIStreamFailoverError(c, account, false, requestID, payloadBytes, message, resp.Header)
streamFailoverErr = s.newOpenAIStreamFailoverErrorWithModel(c, account, false, requestID, payloadBytes, message, upstreamModel, resp.Header)
return true
}
message = s.recordOpenAIStreamUpstreamError(c, account, false, requestID, "http_error", payloadBytes, message)
@@ -605,7 +605,7 @@ func (s *OpenAIGatewayService) handleAnthropicBufferedStreamingResponse(
}
message := openAICompatFailedResponseMessage(finalResponse)
if openAIStreamFailedEventShouldFailover(payload, message) {
return nil, s.newOpenAIStreamFailoverError(c, account, false, requestID, payload, message, resp.Header)
return nil, s.newOpenAIStreamFailoverErrorWithModel(c, account, false, requestID, payload, message, upstreamModel, resp.Header)
}
message = s.recordOpenAIStreamUpstreamError(c, account, false, requestID, "http_error", payload, message)
// 统一走语义状态推断 + body 归一化(与 /v1/responses 路径一致),
@@ -1038,7 +1038,7 @@ func (s *OpenAIGatewayService) handleAnthropicStreamingResponse(
shouldFailover = openAIStreamErrorEventShouldFailover(payloadBytes, message)
}
if !clientOutputStarted && shouldFailover {
streamFailoverErr = s.newOpenAIStreamFailoverError(c, account, false, requestID, payloadBytes, message, resp.Header)
streamFailoverErr = s.newOpenAIStreamFailoverErrorWithModel(c, account, false, requestID, payloadBytes, message, upstreamModel, resp.Header)
return true
}
message = s.recordOpenAIStreamUpstreamError(c, account, false, requestID, "http_error", payloadBytes, message)
@@ -1558,6 +1558,7 @@ func (s *OpenAIGatewayService) handleOpenAIStreamTerminalAccountSideEffects(
payload []byte,
message string,
headers http.Header,
canonicalModel ...string,
) (int, bool) {
statusCode := openAIStreamFailureStatus(payload, message)
switch statusCode {
@@ -1571,14 +1572,17 @@ func (s *OpenAIGatewayService) handleOpenAIStreamTerminalAccountSideEffects(
if c != nil && c.Request != nil {
ctx = c.Request.Context()
}
model := firstNonEmpty(canonicalModel...)
if model == "" {
model = firstNonEmpty(gjson.GetBytes(payload, "model").String(), gjson.GetBytes(payload, "response.model").String())
}
accountHeaders := headers
if statusCode == http.StatusTooManyRequests {
// The enclosing HTTP response succeeded. Its quota snapshot describes
// normal account state and must not become the reset for a semantic 429
// carried by a stream terminal event.
accountHeaders = nil
// 普通模型的流式 429 不能继承外层 HTTP 200 的全局 quota 快照;
// 只有 OAuth/SetupToken 的 Spark 配额 429 才需要保留 headers 读取明确的 5h/7d reset。
accountHeaders = openAIWSSemantic429Headers(account, model, headers)
}
return statusCode, s.handleOpenAIAccountUpstreamError(ctx, account, statusCode, accountHeaders, payload)
return statusCode, s.handleOpenAIAccountUpstreamError(ctx, account, statusCode, accountHeaders, payload, model)
default:
return statusCode, false
}
@@ -1654,6 +1658,19 @@ func (s *OpenAIGatewayService) newOpenAIStreamFailoverError(
payload []byte,
message string,
responseHeaders ...http.Header,
) *UpstreamFailoverError {
return s.newOpenAIStreamFailoverErrorWithModel(c, account, passthrough, upstreamRequestID, payload, message, "", responseHeaders...)
}
func (s *OpenAIGatewayService) newOpenAIStreamFailoverErrorWithModel(
c *gin.Context,
account *Account,
passthrough bool,
upstreamRequestID string,
payload []byte,
message string,
canonicalModel string,
responseHeaders ...http.Header,
) *UpstreamFailoverError {
message = sanitizeUpstreamErrorMessage(strings.TrimSpace(message))
if message == "" {
@@ -1663,7 +1680,7 @@ func (s *OpenAIGatewayService) newOpenAIStreamFailoverError(
if len(responseHeaders) > 0 && responseHeaders[0] != nil {
headers = responseHeaders[0].Clone()
}
statusCode, shouldDisable := s.handleOpenAIStreamTerminalAccountSideEffects(c, account, payload, message, headers)
statusCode, shouldDisable := s.handleOpenAIStreamTerminalAccountSideEffects(c, account, payload, message, headers, canonicalModel)
// 流内 failed 事件承载于 HTTP 200;使用事件的语义状态更新账号健康,
// 再由 failover 引擎按 StatusCode/RetryableOnSameAccount 决定恢复策略。
message = s.recordOpenAIStreamUpstreamError(c, account, passthrough, upstreamRequestID, "failover", payload, message)
@@ -1734,6 +1751,7 @@ func (s *OpenAIGatewayService) nonStreamingTerminalFailureFailover(
terminalType string,
payload []byte,
message string,
canonicalModel ...string,
) *UpstreamFailoverError {
if account == nil || IsResponseCommitted(c) {
return nil
@@ -1751,7 +1769,7 @@ func (s *OpenAIGatewayService) nonStreamingTerminalFailureFailover(
headers = resp.Header
upstreamRequestID = strings.TrimSpace(resp.Header.Get("x-request-id"))
}
return s.newOpenAIStreamFailoverError(c, account, passthrough, upstreamRequestID, payload, message, headers)
return s.newOpenAIStreamFailoverErrorWithModel(c, account, passthrough, upstreamRequestID, payload, message, firstNonEmpty(canonicalModel...), headers)
}
func (s *OpenAIGatewayService) handleStreamingResponsePassthrough(
@@ -1873,7 +1891,7 @@ func (s *OpenAIGatewayService) handleStreamingResponsePassthrough(
return
}
if bareErrorAccountSideEffectsPending {
s.handleOpenAIStreamTerminalAccountSideEffects(c, account, bareErrorPayload, failedMessage, resp.Header)
s.handleOpenAIStreamTerminalAccountSideEffects(c, account, bareErrorPayload, failedMessage, resp.Header, mappedModel)
bareErrorAccountSideEffectsPending = false
}
if clientDisconnected || !writePendingLines() {
@@ -2005,7 +2023,7 @@ func (s *OpenAIGatewayService) handleStreamingResponsePassthrough(
// account health; EOF synthesis applies the pending effect.
bareErrorAccountSideEffectsPending = true
} else {
s.handleOpenAIStreamTerminalAccountSideEffects(c, account, dataBytes, failedMessage, resp.Header)
s.handleOpenAIStreamTerminalAccountSideEffects(c, account, dataBytes, failedMessage, resp.Header, mappedModel)
bareErrorAccountSideEffectsPending = false
}
}
@@ -2020,7 +2038,7 @@ func (s *OpenAIGatewayService) handleStreamingResponsePassthrough(
}
if shouldFailover {
return resultWithUsage(),
s.newOpenAIStreamFailoverError(c, account, true, upstreamRequestID, dataBytes, failedMessage, resp.Header)
s.newOpenAIStreamFailoverErrorWithModel(c, account, true, upstreamRequestID, dataBytes, failedMessage, mappedModel, resp.Header)
}
if !cyberHit && !sawBareError {
if status, errType, errMsg, matched := applyOpenAIStreamFailedErrorPassthroughRule(c, account.Platform, dataBytes, failedMessage); matched {
@@ -2273,7 +2291,7 @@ func (s *OpenAIGatewayService) handlePassthroughSSEToJSON(resp *http.Response, c
if compactErr := newOpenAICompactFallbackSignal(c, terminalPayload, msg); compactErr != nil {
return nil, compactErr
}
if failoverErr := s.nonStreamingTerminalFailureFailover(c, resp, account, true, terminalType, terminalPayload, msg); failoverErr != nil {
if failoverErr := s.nonStreamingTerminalFailureFailover(c, resp, account, true, terminalType, terminalPayload, msg, mappedModel); failoverErr != nil {
return nil, failoverErr
}
return nil, s.writeOpenAINonStreamingProtocolError(resp, c, msg)
@@ -372,7 +372,7 @@ func (s *OpenAIGatewayService) handleStreamingResponseWithReasoning(ctx context.
completeGuardedEvent(true)
}
if codexFailureTerminal && sawBareError && !sawResponseFailed && bareErrorAccountSideEffectsPending {
s.handleOpenAIStreamTerminalAccountSideEffects(c, account, bareErrorPayload, failedMessage, resp.Header)
s.handleOpenAIStreamTerminalAccountSideEffects(c, account, bareErrorPayload, failedMessage, resp.Header, mappedModel)
bareErrorAccountSideEffectsPending = false
}
if codexFailureTerminal && sawBareError && !sawResponseFailed && !clientDisconnected {
@@ -557,7 +557,7 @@ func (s *OpenAIGatewayService) handleStreamingResponseWithReasoning(ctx context.
// Defer account health updates so the pair is applied once.
bareErrorAccountSideEffectsPending = true
} else {
s.handleOpenAIStreamTerminalAccountSideEffects(c, account, dataBytes, failedMessage, resp.Header)
s.handleOpenAIStreamTerminalAccountSideEffects(c, account, dataBytes, failedMessage, resp.Header, mappedModel)
bareErrorAccountSideEffectsPending = false
}
}
@@ -572,7 +572,7 @@ func (s *OpenAIGatewayService) handleStreamingResponseWithReasoning(ctx context.
}
if shouldFailover {
sawFailedEvent = true
streamEarlyErr = s.newOpenAIStreamFailoverError(c, account, false, upstreamRequestID, dataBytes, failedMessage, resp.Header)
streamEarlyErr = s.newOpenAIStreamFailoverErrorWithModel(c, account, false, upstreamRequestID, dataBytes, failedMessage, mappedModel, resp.Header)
return
}
if !cyberHit && !sawBareError {
@@ -1684,7 +1684,7 @@ func (s *OpenAIGatewayService) handleSSEToJSON(resp *http.Response, c *gin.Conte
if compactErr := newOpenAICompactFallbackSignal(c, terminalPayload, msg); compactErr != nil {
return nil, compactErr
}
if failoverErr := s.nonStreamingTerminalFailureFailover(c, resp, account, false, terminalType, terminalPayload, msg); failoverErr != nil {
if failoverErr := s.nonStreamingTerminalFailureFailover(c, resp, account, false, terminalType, terminalPayload, msg, mappedModel); failoverErr != nil {
return nil, failoverErr
}
return nil, s.writeOpenAINonStreamingProtocolError(resp, c, msg)
@@ -866,7 +866,7 @@ func (s *OpenAIGatewayService) ProxyResponsesWebSocketFromClient(
)
var dialErr *openAIWSDialError
if errors.As(acquireErr, &dialErr) && dialErr != nil && dialErr.StatusCode == http.StatusTooManyRequests {
s.persistOpenAIWSRateLimitSignal(ctx, account, dialErr.ResponseHeaders, nil, "rate_limit_exceeded", "rate_limit_error", strings.TrimSpace(acquireErr.Error()))
s.persistOpenAIWSRateLimitSignal(ctx, account, dialErr.ResponseHeaders, nil, "rate_limit_exceeded", "rate_limit_error", strings.TrimSpace(acquireErr.Error()), canonicalModel)
return nil, s.newOpenAIWSRateLimitFailoverError(account, dialErr.ResponseHeaders, nil, acquireErr.Error())
}
if errors.Is(acquireErr, errOpenAIWSPreferredConnUnavailable) {
@@ -1019,7 +1019,7 @@ func (s *OpenAIGatewayService) ProxyResponsesWebSocketFromClient(
}
}
}
s.persistOpenAIWSRateLimitSignal(ctx, account, lease.HandshakeHeaders(), upstreamMessage, errCodeRaw, errTypeRaw, errMsgRaw)
s.persistOpenAIWSRateLimitSignal(ctx, account, lease.HandshakeHeaders(), upstreamMessage, errCodeRaw, errTypeRaw, errMsgRaw, mappedModel)
fallbackReason, _ := classifyOpenAIWSErrorEventFromRaw(errCodeRaw, errTypeRaw, errMsgRaw)
errCode, errType, errMessage := summarizeOpenAIWSErrorEventFieldsFromRaw(errCodeRaw, errTypeRaw, errMsgRaw)
recoverablePrevNotFound := fallbackReason == openAIWSIngressStagePreviousResponseNotFound &&
@@ -131,7 +131,8 @@ func (s *OpenAIGatewayService) performOpenAIWSGeneratePrewarm(
if eventType == "error" {
errCodeRaw, errTypeRaw, errMsgRaw := parseOpenAIWSErrorEventFields(message)
s.persistOpenAIWSRateLimitSignal(ctx, account, lease.HandshakeHeaders(), message, errCodeRaw, errTypeRaw, errMsgRaw)
prewarmModel, _ := reqBody["model"].(string)
s.persistOpenAIWSRateLimitSignal(ctx, account, lease.HandshakeHeaders(), message, errCodeRaw, errTypeRaw, errMsgRaw, prewarmModel)
errMsg := strings.TrimSpace(errMsgRaw)
if errMsg == "" {
errMsg = "OpenAI websocket prewarm error"
@@ -324,6 +325,9 @@ func (s *OpenAIGatewayService) handleOpenAIWSErrorEventTransientFailure(ctx cont
}
status := openAIWSPayloadTransientStatus(payload)
if status != 0 {
if status == http.StatusTooManyRequests {
headers = openAIWSSemantic429Headers(account, canonicalModel, headers)
}
s.handleOpenAIAccountUpstreamError(ctx, account, status, headers, payload, canonicalModel)
}
}
@@ -336,13 +340,13 @@ func (s *OpenAIGatewayService) handleOpenAIWSFailureAccountSideEffects(ctx conte
status := openAIStreamFailureStatus(payload, message)
switch status {
case http.StatusUnauthorized, http.StatusTooManyRequests, 529:
s.handleOpenAIStreamTerminalAccountSideEffects(nil, account, payload, message, headers)
s.handleOpenAIStreamTerminalAccountSideEffects(nil, account, payload, message, headers, canonicalModel)
return true
case http.StatusForbidden:
if !openAIStream403AccountFailure(payload, message) {
return false
}
s.handleOpenAIStreamTerminalAccountSideEffects(nil, account, payload, message, headers)
s.handleOpenAIStreamTerminalAccountSideEffects(nil, account, payload, message, headers, canonicalModel)
return true
}
@@ -683,14 +687,31 @@ func isOpenAIWSRateLimitError(codeRaw, errTypeRaw, msgRaw string) bool {
return false
}
func (s *OpenAIGatewayService) persistOpenAIWSRateLimitSignal(ctx context.Context, account *Account, headers http.Header, responseBody []byte, codeRaw, errTypeRaw, msgRaw string) {
func (s *OpenAIGatewayService) persistOpenAIWSRateLimitSignal(ctx context.Context, account *Account, headers http.Header, responseBody []byte, codeRaw, errTypeRaw, msgRaw string, canonicalModel ...string) {
if s == nil || s.rateLimitService == nil || account == nil || account.Platform != PlatformOpenAI {
return
}
if !isOpenAIWSRateLimitError(codeRaw, errTypeRaw, msgRaw) {
return
}
s.handleOpenAIAccountUpstreamError(ctx, account, http.StatusTooManyRequests, headers, responseBody)
model := firstNonEmpty(canonicalModel...)
if model == "" {
model = firstNonEmpty(gjson.GetBytes(responseBody, "model").String(), gjson.GetBytes(responseBody, "response.model").String())
}
// 非空 responseBody 表示已建立连接后收到的语义错误事件;握手响应头
// 可能只是成功连接时的全局快照,不能用于普通模型的 429 账号级限流。
// 实际拨号 HTTP 429 使用 nil responseBody,必须保留响应头。
if len(responseBody) > 0 {
headers = openAIWSSemantic429Headers(account, model, headers)
}
s.handleOpenAIAccountUpstreamError(ctx, account, http.StatusTooManyRequests, headers, responseBody, model)
}
func openAIWSSemantic429Headers(account *Account, model string, headers http.Header) http.Header {
if isCodexSparkModel(model) && isOpenAIOAuthAccount(account) {
return headers
}
return nil
}
func (s *OpenAIGatewayService) newOpenAIWSRateLimitFailoverError(account *Account, headers http.Header, responseBody []byte, message string) *UpstreamFailoverError {
@@ -245,7 +245,7 @@ func (s *OpenAIGatewayService) forwardOpenAIWSV2(
)
var dialErr *openAIWSDialError
if errors.As(err, &dialErr) && dialErr != nil && dialErr.StatusCode == http.StatusTooManyRequests {
s.persistOpenAIWSRateLimitSignal(ctx, account, dialErr.ResponseHeaders, nil, "rate_limit_exceeded", "rate_limit_error", strings.TrimSpace(err.Error()))
s.persistOpenAIWSRateLimitSignal(ctx, account, dialErr.ResponseHeaders, nil, "rate_limit_exceeded", "rate_limit_error", strings.TrimSpace(err.Error()), mappedModel)
}
return nil, wrapOpenAIWSFallback(classifyOpenAIWSAcquireError(err), err)
}
@@ -601,7 +601,7 @@ func (s *OpenAIGatewayService) forwardOpenAIWSV2(
if eventType == "error" {
s.handleOpenAIWSErrorEventTransientFailure(ctx, account, mappedModel, lease.HandshakeHeaders(), message)
errCodeRaw, errTypeRaw, errMsgRaw := parseOpenAIWSErrorEventFields(message)
s.persistOpenAIWSRateLimitSignal(ctx, account, lease.HandshakeHeaders(), message, errCodeRaw, errTypeRaw, errMsgRaw)
s.persistOpenAIWSRateLimitSignal(ctx, account, lease.HandshakeHeaders(), message, errCodeRaw, errTypeRaw, errMsgRaw, mappedModel)
errMsg := strings.TrimSpace(errMsgRaw)
if errMsg == "" {
errMsg = "Upstream websocket error"
@@ -695,7 +695,7 @@ func (s *OpenAIGatewayService) proxyOpenAIWSHTTPBridgeTurn(
if account.Platform == PlatformGrok {
return nil, newOpenAIUpstreamFailoverError(statusCode, resp.Header, upstreamMessage, errMessage, false)
}
return nil, s.newOpenAIStreamFailoverError(c, account, true, resp.Header.Get("x-request-id"), upstreamMessage, errMessage, resp.Header)
return nil, s.newOpenAIStreamFailoverErrorWithModel(c, account, true, resp.Header.Get("x-request-id"), upstreamMessage, errMessage, mappedModel, resp.Header)
}
if account.Platform != PlatformGrok && !failureAccountSideEffectsApplied {
if eventType == "response.failed" || (!officialOpenAIResponses && shouldFailover && !requestScopedCapacity) {
@@ -902,7 +902,7 @@ func (s *OpenAIGatewayService) proxyResponsesWebSocketV2Passthrough(
)
s.handleOpenAIWSDialTransientFailure(ctx, account, capturedSessionModel, dialErr)
if statusCode == http.StatusTooManyRequests {
s.persistOpenAIWSRateLimitSignal(ctx, account, handshakeHeaders, nil, "rate_limit_exceeded", "rate_limit_error", strings.TrimSpace(err.Error()))
s.persistOpenAIWSRateLimitSignal(ctx, account, handshakeHeaders, nil, "rate_limit_exceeded", "rate_limit_error", strings.TrimSpace(err.Error()), capturedSessionModel)
return s.newOpenAIWSRateLimitFailoverError(account, handshakeHeaders, nil, err.Error())
}
return s.mapOpenAIWSPassthroughDialError(err, statusCode, handshakeHeaders)
@@ -1291,7 +1291,7 @@ func (s *OpenAIGatewayService) proxyResponsesWebSocketV2Passthrough(
if wroteDownstream || !isOpenAIWSRateLimitError(errCodeRaw, errTypeRaw, errMsgRaw) {
return nil
}
s.persistOpenAIWSRateLimitSignal(ctx, account, handshakeHeaders, payload, errCodeRaw, errTypeRaw, errMsgRaw)
s.persistOpenAIWSRateLimitSignal(ctx, account, handshakeHeaders, payload, errCodeRaw, errTypeRaw, errMsgRaw, capturedSessionModel)
logOpenAIWSV2Passthrough(
"relay_rate_limit_failover account_id=%d err_code=%s err_type=%s err_message=%s",
account.ID,
@@ -2192,6 +2192,43 @@ func (s *RateLimitService) HandleOpenAIImageRateLimit(ctx context.Context, accou
return true
}
// HandleOpenAICodexSparkRateLimit 将 Spark 独立配额窗口记录为模型级限流。
// Spark 的 x-codex-* 使用率和 reset 时间只代表 Spark 模型维度,不能写入账号级
// RateLimitResetAt,否则同一 OAuth 账号上的其他模型也会被错误停调。
func (s *RateLimitService) HandleOpenAICodexSparkRateLimit(ctx context.Context, account *Account, requestedModel string, statusCode int, headers http.Header, responseBody []byte) bool {
if s == nil || account == nil || s.accountRepo == nil || statusCode != http.StatusTooManyRequests || !isOpenAIOAuthAccount(account) {
return false
}
if !isCodexSparkModel(requestedModel) || !account.ShouldHandleErrorCode(statusCode) {
return false
}
modelKey := normalizeCodexModel(modelRateLimitKeyForUpstreamModelNotFound(ctx, account, requestedModel))
if modelKey == "" {
return false
}
now := time.Now()
disposition, resetAt := classifyOpenAIOAuth429(headers, responseBody)
// Spark 只有明确耗尽 5h/7d 窗口时才能使用上游长 reset;普通瞬时 429
// 即使携带全局 reset 头,也只能使用短时回避,避免错误冷却数天。
if disposition != openAIOAuth429Quota5h && disposition != openAIOAuth429Quota7d {
resetAt = nil
}
if resetAt == nil || !resetAt.After(now) {
cooldown, ok := s.get429FallbackCooldown(ctx, account)
if !ok || cooldown <= 0 {
cooldown = time.Duration(defaultRateLimit429CooldownSeconds) * time.Second
}
reset := now.Add(cooldown)
resetAt = &reset
}
if err := s.accountRepo.SetModelRateLimit(ctx, account.ID, modelKey, *resetAt, openAICodexSparkRateLimitReason); err != nil {
slog.Warn("openai_codex_spark_model_rate_limit_set_failed", "account_id", account.ID, "model", modelKey, "error", err)
}
slog.Info("openai_codex_spark_model_rate_limited", "account_id", account.ID, "model", modelKey, "reset_at", *resetAt)
return true
}
func (s *RateLimitService) HandleOpenAIImageCapabilityLoss(ctx context.Context, account *Account, statusCode int, responseBody []byte) bool {
if s == nil || account == nil || s.accountRepo == nil {
return false