diff --git a/backend/internal/service/grok_upstream_errors.go b/backend/internal/service/grok_upstream_errors.go index 880cd702b0..0039ea4cdc 100644 --- a/backend/internal/service/grok_upstream_errors.go +++ b/backend/internal/service/grok_upstream_errors.go @@ -191,10 +191,17 @@ func grokContentPolicyClientMessage(responseBody []byte) string { // shouldFailoverGrokUpstreamError is the body-aware counterpart of the // status-only failover helper. Grok content refusals must stay on the current // account and be returned to the caller instead of consuming the account pool. +// Free-usage / empty-output / billing bodies also failover even when the HTTP +// status alone would not (e.g. 400 with free-usage-exhausted). func (s *OpenAIGatewayService) shouldFailoverGrokUpstreamError(statusCode int, responseBody []byte) bool { if isGrokContentPolicyRejection(statusCode, responseBody) { return false } + decision := classifyGrokUpstreamFailure(statusCode, responseBody, "") + switch decision.Class { + case GrokFailureFreeUsage, GrokFailureEmptyUpstream, GrokFailureBilling, GrokFailureModelCapacity: + return decision.ShouldFailover + } return s.shouldFailoverUpstreamError(statusCode) } diff --git a/backend/internal/service/grok_upstream_failure.go b/backend/internal/service/grok_upstream_failure.go new file mode 100644 index 0000000000..86ed02e93b --- /dev/null +++ b/backend/internal/service/grok_upstream_failure.go @@ -0,0 +1,494 @@ +package service + +import ( + "context" + "encoding/json" + "net/http" + "regexp" + "strconv" + "strings" + "time" + + "github.com/tidwall/gjson" +) + +// Grok upstream failure classes used to decide temp-unschedulable cooldowns and +// pre-commit account failover. Classification is body-first so free-usage and +// empty-output wording still win when the proxy rewrites status codes. +type GrokUpstreamFailureClass string + +const ( + GrokFailureNone GrokUpstreamFailureClass = "" + GrokFailureFreeUsage GrokUpstreamFailureClass = "subscription:free-usage-exhausted" + GrokFailureBilling GrokUpstreamFailureClass = "billing_quota" + GrokFailureEmptyUpstream GrokUpstreamFailureClass = "empty_upstream" + GrokFailureModelCapacity GrokUpstreamFailureClass = "model_capacity" + GrokFailureRateLimit GrokUpstreamFailureClass = "rate_limit" + GrokFailureAuth GrokUpstreamFailureClass = "auth_error" + GrokFailureServer GrokUpstreamFailureClass = "server_error" +) + +// GrokUpstreamFailureDecision is a pure classification result. Callers map it +// onto existing account state helpers (tempUnscheduleGrok / rateLimitGrok). +// BlockModel is retained for observability; the current scheduler does not +// implement per-model soft-blocks, so free-usage deliberately never sets it. +type GrokUpstreamFailureDecision struct { + Class GrokUpstreamFailureClass + Model string + Cooldown time.Duration + ShouldCooldown bool + // ShouldFailover recommends trying another account before writing a + // terminal response (pre-commit only). Content-policy rejections are + // handled separately and never reach this classifier for failover. + ShouldFailover bool + // BlockModel is true only for empty-output when a model id is known. + // Free-usage never sets this: the account cools, not a single model. + BlockModel bool + Reason string + TokensActual *int64 + TokensLimit *int64 +} + +var ( + reGrokTokenPair = regexp.MustCompile(`(?i)tokens?\s*(?:\(actual\s*/\s*limit\))?\s*[:=]?\s*(\d+)\s*/\s*(\d+)`) + reGrokModelFor = regexp.MustCompile(`(?i)(?:for\s+model|model|模型)\s*[::]?\s*([a-z0-9][a-z0-9._-]{2,80})`) + reGrokResetsWindow = regexp.MustCompile(`(?i)resets?\s+over\s+a\s+rolling\s+(\d+)\s*-\s*hour`) +) + +// classifyGrokUpstreamFailure decides cooldown/failover from status + body. +// Priority (body/code first, status second): +// 1. free-usage exhausted → account cool, no model block, failover +// 2. billing hard quota → longer cool, failover +// 3. empty model output → short cool + optional model soft-block marker, failover +// 4. model capacity → short cool, failover +// 5. bare rate-limit / 429 without free-usage language → cool, failover +// 6. bare 5xx → brief cool, failover +// 7. validation / client errors without quota language → no cool +// +// Content-policy 403s must be filtered by the caller before invoking this. +func classifyGrokUpstreamFailure(statusCode int, responseBody []byte, requestedModel string) GrokUpstreamFailureDecision { + text, code, low := grokUpstreamErrorCorpus(statusCode, responseBody) + model := extractGrokFailureModel(text, responseBody, requestedModel) + actual, limit, hasTokens := parseGrokTokenPair(text) + if !hasTokens { + actual, limit, hasTokens = parseGrokTokenPair(string(responseBody)) + } + + // --- Free usage / rolling quota exhausted --- + if isGrokFreeUsageExhaustedText(low) || isGrokFreeUsageCode(code) || isGrokFreeUsageCode(text) { + d := GrokUpstreamFailureDecision{ + Class: GrokFailureFreeUsage, + Model: model, + Cooldown: grokFreeUsageCooldownDuration(low), + ShouldCooldown: true, + ShouldFailover: true, + BlockModel: false, + Reason: firstNonEmpty(text, code, "free usage exhausted"), + } + if hasTokens { + a, b := actual, limit + d.TokensActual = &a + d.TokensLimit = &b + } + return d + } + + // Billing / hard quota (not free-tier rolling). + // Cooldown stays at 30m to match the existing Grok 402/spending-limit + // handler (longer cools would change ops behavior without a settings knob). + if isGrokBillingQuotaText(low) || statusCode == http.StatusPaymentRequired { + reason := firstNonEmpty(text, "billing quota") + if statusCode == http.StatusPaymentRequired && text == "" { + reason = "payment required" + } + return GrokUpstreamFailureDecision{ + Class: GrokFailureBilling, + Model: model, + Cooldown: 30 * time.Minute, + ShouldCooldown: true, + ShouldFailover: true, + BlockModel: model != "", + Reason: reason, + } + } + + // Empty HTTP 200 / empty model output (often rewritten to synthetic 502). + if isGrokEmptyModelOutputText(low) || isGrokEmptyModelOutputCode(code) { + return GrokUpstreamFailureDecision{ + Class: GrokFailureEmptyUpstream, + Model: model, + Cooldown: 4 * time.Minute, + ShouldCooldown: true, + ShouldFailover: true, + BlockModel: model != "", + Reason: firstNonEmpty(text, "empty model output"), + } + } + + // Model capacity / overloaded. + if isGrokModelCapacityText(low) { + return GrokUpstreamFailureDecision{ + Class: GrokFailureModelCapacity, + Model: model, + Cooldown: 3 * time.Minute, + ShouldCooldown: true, + ShouldFailover: true, + BlockModel: false, + Reason: firstNonEmpty(text, "model capacity"), + } + } + + // Rate limit without free-usage language. + if statusCode == http.StatusTooManyRequests || isGrokRateLimitText(low) { + return GrokUpstreamFailureDecision{ + Class: GrokFailureRateLimit, + Model: model, + Cooldown: 10 * time.Minute, + ShouldCooldown: true, + ShouldFailover: true, + BlockModel: false, + Reason: firstNonEmpty(text, "rate limit"), + } + } + + // Upstream 5xx — brief cool. Empty-output synthetic 502 already handled above. + if statusCode >= 500 && statusCode <= 599 { + return GrokUpstreamFailureDecision{ + Class: GrokFailureServer, + Cooldown: 2 * time.Minute, + ShouldCooldown: true, + ShouldFailover: true, + Reason: firstNonEmpty(text, "server error"), + } + } + + return GrokUpstreamFailureDecision{Reason: text} +} + +func grokUpstreamErrorCorpus(statusCode int, responseBody []byte) (text, code, low string) { + raw := strings.TrimSpace(string(responseBody)) + if unwrappedStatus, unwrappedBody, ok := unwrapGrokUpstreamErrorText(raw); ok { + if statusCode <= 0 { + statusCode = unwrappedStatus + } + raw = unwrappedBody + } + text = raw + codeFromJSON, msgFromJSON := parseGrokUpstreamErrorJSON(raw) + if msgFromJSON != "" { + if text == "" || len(msgFromJSON) > len(text)/2 || looksLikeGrokQuotaMessage(msgFromJSON) { + text = msgFromJSON + } + } + // Prefer structured fields from the original body when present. + if len(responseBody) > 0 { + if m := strings.TrimSpace(firstNonEmpty( + gjson.GetBytes(responseBody, "error.message").String(), + gjson.GetBytes(responseBody, "message").String(), + gjson.GetBytes(responseBody, "error").String(), + )); m != "" && (text == "" || looksLikeGrokQuotaMessage(m)) { + text = m + } + if c := strings.TrimSpace(firstNonEmpty( + gjson.GetBytes(responseBody, "error.code").String(), + gjson.GetBytes(responseBody, "code").String(), + )); c != "" { + codeFromJSON = c + } + } + code = codeFromJSON + low = strings.ToLower(strings.TrimSpace(text)) + if code != "" && !strings.Contains(low, strings.ToLower(code)) { + low = strings.ToLower(code) + " " + low + } + return text, code, low +} + +func unwrapGrokUpstreamErrorText(errText string) (status int, body string, ok bool) { + text := strings.TrimSpace(errText) + if text == "" { + return 0, "", false + } + lower := strings.ToLower(text) + for _, p := range []string{"upstream status ", "status "} { + if !strings.HasPrefix(lower, p) { + continue + } + rest := strings.TrimSpace(text[len(p):]) + i := 0 + for i < len(rest) && rest[i] >= '0' && rest[i] <= '9' { + status = status*10 + int(rest[i]-'0') + i++ + } + if status <= 0 || i == 0 { + return 0, "", false + } + rest = strings.TrimSpace(rest[i:]) + if strings.HasPrefix(rest, ":") { + rest = strings.TrimSpace(rest[1:]) + } + return status, rest, true + } + return 0, "", false +} + +func parseGrokUpstreamErrorJSON(errText string) (code, message string) { + text := strings.TrimSpace(errText) + if text == "" || text[0] != '{' { + return "", "" + } + var payload map[string]any + if json.Unmarshal([]byte(text), &payload) != nil { + return "", "" + } + if v, ok := payload["code"].(string); ok { + code = v + } + if v, ok := payload["message"].(string); ok { + message = v + } + if errObj, ok := payload["error"].(map[string]any); ok { + if v, ok := errObj["code"].(string); ok && code == "" { + code = v + } + if v, ok := errObj["message"].(string); ok && message == "" { + message = v + } + } + if errStr, ok := payload["error"].(string); ok && message == "" { + message = errStr + } + return strings.TrimSpace(code), strings.TrimSpace(message) +} + +func looksLikeGrokQuotaMessage(s string) bool { + low := strings.ToLower(s) + return strings.Contains(low, "quota") || + strings.Contains(low, "usage") || + strings.Contains(low, "credit") || + strings.Contains(low, "额度") || + strings.Contains(low, "free") +} + +func isGrokFreeUsageCode(code string) bool { + c := strings.ToLower(strings.TrimSpace(code)) + if c == "" { + return false + } + if strings.Contains(c, "subscription:free-usage-exhausted") || + strings.Contains(c, "free-usage-exhausted") || + strings.Contains(c, "free_usage_exhausted") || + strings.Contains(c, "usage-limit-exceeded") || + strings.Contains(c, "usage_limit_exceeded") { + return true + } + return (strings.Contains(c, "free-usage") || strings.Contains(c, "free_usage")) && + (strings.Contains(c, "exhaust") || strings.Contains(c, "exceed") || strings.Contains(c, "limit")) +} + +func isGrokFreeUsageExhaustedText(low string) bool { + if low == "" { + return false + } + if strings.Contains(low, "free-usage-exhausted") || + strings.Contains(low, "free_usage_exhausted") || + strings.Contains(low, "subscription:free-usage") || + strings.Contains(low, "usage-limit-exceeded") || + strings.Contains(low, "usage_limit_exceeded") || + strings.Contains(low, "free-tier-limit") || + strings.Contains(low, "free_tier_limit") { + return true + } + if strings.Contains(low, "free usage") || + strings.Contains(low, "included free usage") || + strings.Contains(low, "used all the included free") || + strings.Contains(low, "you've used all the included free") || + strings.Contains(low, "you have used all the included free") || + strings.Contains(low, "free quota") || + strings.Contains(low, "no remaining free") || + strings.Contains(low, "out of free") || + strings.Contains(low, "usage resets over a rolling") || + (strings.Contains(low, "free tier") && (strings.Contains(low, "exhaust") || strings.Contains(low, "limit") || strings.Contains(low, "exceed"))) { + return true + } + for _, p := range []string{ + "额度耗尽", "额度用完", "额度不足", "额度已用尽", "额度已耗尽", + "免费额度", "免费用量", "用量用完", "用量耗尽", "用量超限", "用量已用尽", + "配额耗尽", "配额已用尽", "配额不足", "配额超限", "配额用完", + "没有额度", "没额度", "无额度", "可用额度不足", "模型额度", + "临时额度", "额度已满", "额度超限", "额度达到上限", + "模型额度用完", "模型额度耗尽", "账号额度用完", "账号额度耗尽", + "额度不够", "没额度了", "额度没了", "用完额度", "耗尽额度", + } { + if strings.Contains(low, p) { + return true + } + } + if (strings.Contains(low, "quota") && (strings.Contains(low, "exhaust") || strings.Contains(low, "exceed") || strings.Contains(low, "limit"))) || + (strings.Contains(low, "usage") && (strings.Contains(low, "exhaust") || strings.Contains(low, "exceed")) && (strings.Contains(low, "limit") || strings.Contains(low, "free") || strings.Contains(low, "model"))) { + if strings.Contains(low, "free") || strings.Contains(low, "rolling") || + strings.Contains(low, "24-hour") || strings.Contains(low, "24 hour") || + strings.Contains(low, "model") || strings.Contains(low, "subscription") || + strings.Contains(low, "included") || strings.Contains(low, "tokens") { + return true + } + } + if a, b, ok := parseGrokTokenPair(low); ok && b > 0 && a >= b { + if strings.Contains(low, "free") || strings.Contains(low, "subscription") || + strings.Contains(low, "included") || strings.Contains(low, "model") || + strings.Contains(low, "usage") || strings.Contains(low, "quota") || + strings.Contains(low, "rolling") { + return true + } + } + return false +} + +func isGrokBillingQuotaText(low string) bool { + if low == "" { + return false + } + if strings.Contains(low, "insufficient_quota") { + return true + } + if strings.Contains(low, "billing") && strings.Contains(low, "quota") { + return true + } + if strings.Contains(low, "payment") && (strings.Contains(low, "required") || strings.Contains(low, "fail")) { + return true + } + if strings.Contains(low, "spending limit") || strings.Contains(low, "run out of credits") || strings.Contains(low, "out of credits") { + return true + } + if strings.Contains(low, "余额不足") || strings.Contains(low, "欠费") || strings.Contains(low, "需要付费") { + return true + } + return false +} + +func isGrokModelCapacityText(low string) bool { + return strings.Contains(low, "capacity") || + strings.Contains(low, "overloaded") || + strings.Contains(low, "server_busy") || + strings.Contains(low, "too many concurrent") || + strings.Contains(low, "engine_overloaded") +} + +func isGrokRateLimitText(low string) bool { + return strings.Contains(low, "rate limit") || + strings.Contains(low, "rate_limit") || + strings.Contains(low, "too many requests") || + strings.Contains(low, "请求过于频繁") || + strings.Contains(low, "速率限制") +} + +func isGrokEmptyModelOutputText(low string) bool { + if low == "" { + return false + } + return strings.Contains(low, "empty model output") || + strings.Contains(low, "no content/tool_calls") || + strings.Contains(low, "no client-visible content") || + strings.Contains(low, "empty_upstream") || + strings.Contains(low, "empty upstream") +} + +func isGrokEmptyModelOutputCode(code string) bool { + c := strings.ToLower(strings.TrimSpace(code)) + if c == "" { + return false + } + return c == "empty_upstream" || + c == "empty-model-output" || + c == "empty_model_output" || + strings.Contains(c, "empty_upstream") || + strings.Contains(c, "empty-model-output") +} + +func grokFreeUsageCooldownDuration(low string) time.Duration { + if m := reGrokResetsWindow.FindStringSubmatch(low); len(m) == 2 { + hours, _ := strconv.Atoi(m[1]) + if hours > 0 { + d := time.Duration(hours) * time.Hour / 12 + if d < 20*time.Minute { + d = 20 * time.Minute + } + if d > 6*time.Hour { + d = 6 * time.Hour + } + return d + } + } + return 2 * time.Hour +} + +func parseGrokTokenPair(errText string) (actual, limit int64, ok bool) { + m := reGrokTokenPair.FindStringSubmatch(errText) + if len(m) != 3 { + return 0, 0, false + } + a, errA := strconv.ParseInt(m[1], 10, 64) + b, errB := strconv.ParseInt(m[2], 10, 64) + if errA != nil || errB != nil { + return 0, 0, false + } + return a, b, true +} + +func extractGrokFailureModel(text string, responseBody []byte, fallback string) string { + if m := reGrokModelFor.FindStringSubmatch(text); len(m) == 2 { + return strings.TrimSpace(m[1]) + } + if len(responseBody) > 0 { + if m := strings.TrimSpace(firstNonEmpty( + gjson.GetBytes(responseBody, "error.model").String(), + gjson.GetBytes(responseBody, "model").String(), + )); m != "" { + return m + } + } + return strings.TrimSpace(fallback) +} + +// applyGrokUpstreamFailureDecision maps a classification onto existing account +// health helpers. Returns true when the decision fully handled the error path +// (caller should not apply the status-code switch defaults again). +func (s *OpenAIGatewayService) applyGrokUpstreamFailureDecision( + ctx context.Context, + account *Account, + decision GrokUpstreamFailureDecision, +) bool { + if s == nil || account == nil || !decision.ShouldCooldown || decision.Cooldown <= 0 { + return false + } + // Keep reasons short and stable for ops UI / temp_unschedulable_reason. + var reason string + switch decision.Class { + case GrokFailureFreeUsage: + reason = "grok free usage exhausted" + case GrokFailureBilling: + low := strings.ToLower(decision.Reason) + if strings.Contains(low, "spending") || strings.Contains(low, "credits") { + reason = "grok spending limit" + } else { + // Keep the historical 402/payment reason for ops UI + regression tests. + reason = "grok payment required" + } + case GrokFailureEmptyUpstream: + reason = "grok empty model output" + case GrokFailureModelCapacity: + reason = "grok model capacity" + case GrokFailureRateLimit: + // Pure 429 without free-usage language keeps the existing rate-limit + // snapshot path (Retry-After / quota headers). Body-only rate-limit + // phrasing still cools here via ShouldCooldown from the classifier, but + // the handler only invokes this for non-RateLimit classes. + return false + case GrokFailureServer: + reason = "grok upstream temporary error" + default: + return false + } + s.tempUnscheduleGrok(ctx, account, decision.Cooldown, reason) + return true +} diff --git a/backend/internal/service/grok_upstream_failure_test.go b/backend/internal/service/grok_upstream_failure_test.go new file mode 100644 index 0000000000..c51b4053a6 --- /dev/null +++ b/backend/internal/service/grok_upstream_failure_test.go @@ -0,0 +1,167 @@ +//go:build unit + +package service + +import ( + "context" + "net/http" + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +func TestClassifyGrokUpstreamFailure_FreeUsage(t *testing.T) { + cases := []struct { + name string + status int + body string + }{ + { + name: "code free-usage-exhausted", + status: http.StatusTooManyRequests, + body: `{"error":{"code":"subscription:free-usage-exhausted","message":"You've used all the included free usage for model grok-4.5. Usage resets over a rolling 24-hour window."}}`, + }, + { + name: "chinese body without 429", + status: http.StatusBadRequest, + body: `{"error":{"message":"模型额度用完,请稍后再试"}}`, + }, + { + name: "token pair with free marker", + status: http.StatusOK, + body: `{"error":{"message":"free usage tokens (actual / limit): 2000000 / 2000000 for model grok-4.5"}}`, + }, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + d := classifyGrokUpstreamFailure(tc.status, []byte(tc.body), "grok-4.5") + require.Equal(t, GrokFailureFreeUsage, d.Class) + require.True(t, d.ShouldCooldown) + require.True(t, d.ShouldFailover) + require.False(t, d.BlockModel, "free-usage must not soft-block models") + require.GreaterOrEqual(t, d.Cooldown, 20*time.Minute) + }) + } +} + +func TestClassifyGrokUpstreamFailure_EmptyUpstream(t *testing.T) { + d := classifyGrokUpstreamFailure(http.StatusBadGateway, []byte(`empty model output: no content/tool_calls`), "grok-4.5") + require.Equal(t, GrokFailureEmptyUpstream, d.Class) + require.True(t, d.ShouldCooldown) + require.True(t, d.ShouldFailover) + require.True(t, d.BlockModel) + require.Equal(t, 4*time.Minute, d.Cooldown) +} + +func TestClassifyGrokUpstreamFailure_Billing(t *testing.T) { + d := classifyGrokUpstreamFailure(http.StatusForbidden, []byte(`{"code":"personal-team-blocked:spending-limit","error":"spending limit reached"}`), "") + require.Equal(t, GrokFailureBilling, d.Class) + require.True(t, d.ShouldCooldown) + require.True(t, d.ShouldFailover) +} + +func TestClassifyGrokUpstreamFailure_ValidationNoCool(t *testing.T) { + d := classifyGrokUpstreamFailure(http.StatusBadRequest, []byte(`{"error":{"message":"invalid tool schema"}}`), "") + require.Equal(t, GrokFailureNone, d.Class) + require.False(t, d.ShouldCooldown) + require.False(t, d.ShouldFailover) +} + +func TestClassifyGrokUpstreamFailure_FreeUsageWinsOver5xx(t *testing.T) { + // Proxy may rewrite free-usage into synthetic 502; body must win. + d := classifyGrokUpstreamFailure(http.StatusBadGateway, []byte(`subscription:free-usage-exhausted for model grok-4.3`), "grok-4.3") + require.Equal(t, GrokFailureFreeUsage, d.Class) + require.NotEqual(t, GrokFailureServer, d.Class) +} + +func TestShouldFailoverGrokUpstreamError_FreeUsageBody(t *testing.T) { + svc := &OpenAIGatewayService{} + body := []byte(`{"error":{"code":"subscription:free-usage-exhausted","message":"free usage exhausted"}}`) + require.True(t, svc.shouldFailoverGrokUpstreamError(http.StatusBadRequest, body)) +} + +func TestShouldFailoverGrokUpstreamError_ContentPolicyStillNoFailover(t *testing.T) { + svc := &OpenAIGatewayService{} + body := []byte(`{"error":{"code":"new_sensitive","message":"text is sensitive"}}`) + require.False(t, svc.shouldFailoverGrokUpstreamError(http.StatusForbidden, body)) +} + +func TestHandleGrokAccountUpstreamError_FreeUsageBodyCoolsAccount(t *testing.T) { + repo := &grokQuotaAccountRepo{} + svc := &OpenAIGatewayService{accountRepo: repo} + account := &Account{ID: 9101, Platform: PlatformGrok, Type: AccountTypeOAuth} + before := time.Now() + body := []byte(`{"error":{"code":"subscription:free-usage-exhausted","message":"You've used all the included free usage. Usage resets over a rolling 24-hour window."}}`) + + svc.handleGrokAccountUpstreamError(context.Background(), account, http.StatusBadRequest, nil, body) + + require.Equal(t, 1, repo.tempUnschedCalls) + require.Equal(t, "grok free usage exhausted", repo.lastTempUnschedReason) + // 24h rolling → 2h cool + require.Greater(t, repo.lastTempUnschedUntil, before.Add(119*time.Minute)) + require.Less(t, repo.lastTempUnschedUntil, before.Add(121*time.Minute)) +} + +func TestHandleGrokAccountUpstreamError_EmptyOutputCoolsAccount(t *testing.T) { + repo := &grokQuotaAccountRepo{} + svc := &OpenAIGatewayService{accountRepo: repo} + account := &Account{ID: 9102, Platform: PlatformGrok, Type: AccountTypeOAuth} + before := time.Now() + + svc.handleGrokAccountUpstreamError( + context.Background(), account, http.StatusBadGateway, nil, + []byte(`empty model output: no content/tool_calls`), + ) + + require.Equal(t, 1, repo.tempUnschedCalls) + require.Equal(t, "grok empty model output", repo.lastTempUnschedReason) + require.WithinDuration(t, before.Add(4*time.Minute), repo.lastTempUnschedUntil, time.Second) +} + +func TestHandleGrokAccountUpstreamError_FreeUsageDoesNotCoolPoolMode(t *testing.T) { + repo := &grokQuotaAccountRepo{} + svc := &OpenAIGatewayService{accountRepo: repo} + account := &Account{ + ID: 9103, + Platform: PlatformGrok, + Type: AccountTypeAPIKey, + Credentials: map[string]any{ + "pool_mode": true, + }, + } + body := []byte(`{"error":{"code":"subscription:free-usage-exhausted","message":"free usage exhausted"}}`) + + svc.handleGrokAccountUpstreamError(context.Background(), account, http.StatusBadRequest, nil, body) + + require.Zero(t, repo.tempUnschedCalls) + require.False(t, svc.isOpenAIAccountRuntimeBlocked(account)) +} + +func TestHandleGrokAccountUpstreamError_ContentPolicyStillNoMutation(t *testing.T) { + repo := &grokQuotaAccountRepo{} + svc := &OpenAIGatewayService{accountRepo: repo} + account := &Account{ID: 9104, Platform: PlatformGrok, Type: AccountTypeOAuth} + body := []byte(`{"error":{"code":"new_sensitive","message":"text is sensitive"}}`) + + svc.handleGrokAccountUpstreamError(context.Background(), account, http.StatusForbidden, nil, body) + + require.Zero(t, repo.tempUnschedCalls) +} + +func TestHandleGrokAccountUpstreamError_Entitlement403Unchanged(t *testing.T) { + repo := &grokQuotaAccountRepo{} + svc := &OpenAIGatewayService{accountRepo: repo} + account := &Account{ID: 9105, Platform: PlatformGrok, Type: AccountTypeOAuth} + before := time.Now() + + svc.handleGrokAccountUpstreamError( + context.Background(), account, http.StatusForbidden, nil, + []byte(`{"error":{"message":"subscription required"}}`), + ) + + require.Equal(t, 1, repo.tempUnschedCalls) + require.Equal(t, "grok access or entitlement denied", repo.lastTempUnschedReason) + require.Greater(t, repo.lastTempUnschedUntil, before.Add(29*time.Minute)) + require.Less(t, repo.lastTempUnschedUntil, before.Add(31*time.Minute)) +} diff --git a/backend/internal/service/openai_gateway_grok.go b/backend/internal/service/openai_gateway_grok.go index 0d3f024628..8140c14cf8 100644 --- a/backend/internal/service/openai_gateway_grok.go +++ b/backend/internal/service/openai_gateway_grok.go @@ -1356,6 +1356,19 @@ func (s *OpenAIGatewayService) handleGrokAccountUpstreamError(ctx context.Contex } now := time.Now() s.updateGrokUsageSnapshot(ctx, account, parseGrokQuotaSnapshot(headers, statusCode, now)) + + // Body-first free-usage / empty / billing / capacity must run before the + // status switch so non-429 free-usage bodies still cool the account. + // Pool-mode still skips durable mutation unless an explicit temp rule matches. + decision := classifyGrokUpstreamFailure(statusCode, responseBody, "") + if decision.ShouldCooldown && decision.Class != GrokFailureNone && decision.Class != GrokFailureRateLimit { + if account.IsPoolMode() { + // Allow configured temp rules (403) below; skip default body cools. + } else if s.applyGrokUpstreamFailureDecision(ctx, account, decision) { + return + } + } + if statusCode == http.StatusForbidden && s.applyGrokForbiddenPolicy(ctx, account, responseBody) { return } @@ -1367,11 +1380,10 @@ func (s *OpenAIGatewayService) handleGrokAccountUpstreamError(ctx context.Contex case http.StatusUnauthorized: s.tempUnscheduleGrok(ctx, account, 10*time.Minute, "grok credentials unauthorized") case http.StatusPaymentRequired: - // 402: temporarily unschedulable with a clear payment-required reason. + // 402 without a body-classified billing decision: keep the legacy 30m cool. s.tempUnscheduleGrok(ctx, account, 30*time.Minute, "grok payment required") case http.StatusForbidden: - // Spending-limit 403 (personal-team-blocked:spending-limit) is billing exhaustion, - // not a generic entitlement denial — still temp-unschedule with a distinct reason. + // Spending-limit already handled by body classifier when phrasing matches. if isGrokSpendingLimitError(responseBody) { s.tempUnscheduleGrok(ctx, account, 30*time.Minute, "grok spending limit") return @@ -1379,6 +1391,7 @@ func (s *OpenAIGatewayService) handleGrokAccountUpstreamError(ctx context.Contex s.tempUnscheduleGrok(ctx, account, 30*time.Minute, "grok access or entitlement denied") case http.StatusTooManyRequests: // updateGrokUsageSnapshot installs rate-limit state for non-pool accounts. + // Free-usage 429 was already cooled above via body classification. default: if statusCode >= 500 { s.tempUnscheduleGrok(ctx, account, 2*time.Minute, "grok upstream temporary error") diff --git a/backend/internal/service/openai_gateway_grok_cache.go b/backend/internal/service/openai_gateway_grok_cache.go index 4c6923e799..696e7352ec 100644 --- a/backend/internal/service/openai_gateway_grok_cache.go +++ b/backend/internal/service/openai_gateway_grok_cache.go @@ -118,6 +118,12 @@ func explicitGrokCacheSeed(c *gin.Context, body []byte, explicitKey string) stri if seed == "" { seed = strings.TrimSpace(explicitKey) } + // previous_response_id is last-resort: multi-turn Responses without an + // explicit session still share one cache identity (model is already in the + // isolated seed). Message ids are rejected by the seed helper. + if seed == "" && len(body) > 0 { + seed = grokPreviousResponseSessionSeed(body) + } return seed } diff --git a/backend/internal/service/openai_gateway_grok_cache_test.go b/backend/internal/service/openai_gateway_grok_cache_test.go index dc3beaa461..3251bb7485 100644 --- a/backend/internal/service/openai_gateway_grok_cache_test.go +++ b/backend/internal/service/openai_gateway_grok_cache_test.go @@ -24,6 +24,36 @@ func newGrokCacheTestContext(apiKeyID int64) *gin.Context { return c } +func TestGrokPreviousResponseSessionSeed(t *testing.T) { + require.Equal(t, "grok-prev-resp:resp_abc123", grokPreviousResponseSessionSeed([]byte(`{"previous_response_id":"resp_abc123"}`))) + require.Empty(t, grokPreviousResponseSessionSeed([]byte(`{"previous_response_id":"msg_abc123"}`))) + require.Empty(t, grokPreviousResponseSessionSeed([]byte(`{"previous_response_id":""}`))) + require.Empty(t, grokPreviousResponseSessionSeed([]byte(`{}`))) +} + +func TestResolveGrokCacheIdentityUsesPreviousResponseIDWhenNoOtherSeed(t *testing.T) { + gin.SetMode(gin.TestMode) + c := newGrokCacheTestContext(301) + // No prompt_cache_key / headers / reusable prefix — only previous_response_id. + body := []byte(`{"model":"grok","input":[{"role":"user","content":"follow up"}],"previous_response_id":"resp_chain_001"}`) + got := resolveGrokCacheIdentity(c, body, "", "grok-4.5") + require.NotEmpty(t, got) + + // Same previous_response_id → same identity (model already in isolated seed). + again := resolveGrokCacheIdentity(c, body, "", "grok-4.5") + require.Equal(t, got, again) + + // Different model → different identity (model scope). + otherModel := resolveGrokCacheIdentity(c, body, "", "grok-4.3") + require.NotEqual(t, got, otherModel) + + // prompt_cache_key still wins over previous_response_id. + withCache := []byte(`{"model":"grok","prompt_cache_key":"client-session","previous_response_id":"resp_chain_001","input":[{"role":"user","content":"x"}]}`) + cacheID := resolveGrokCacheIdentity(c, withCache, "", "grok-4.5") + require.NotEmpty(t, cacheID) + require.NotEqual(t, got, cacheID) +} + func TestResolveGrokCacheIdentityStableAcrossAppendOnlyTurns(t *testing.T) { gin.SetMode(gin.TestMode) c := newGrokCacheTestContext(101) diff --git a/backend/internal/service/openai_gateway_scheduling.go b/backend/internal/service/openai_gateway_scheduling.go index 23c04daf0a..0baf9f2e8e 100644 --- a/backend/internal/service/openai_gateway_scheduling.go +++ b/backend/internal/service/openai_gateway_scheduling.go @@ -75,6 +75,12 @@ func explicitOpenAISessionID(c *gin.Context, body []byte) string { // with Grok's native conversation header only for requests authenticated to a // Grok group. This keeps an unrelated x-grok-conv-id header from changing // scheduling or upstream session behavior for non-Grok groups. +// +// For Grok groups only, previous_response_id is a last-resort sticky seed so +// multi-turn Responses chains stay on the same OAuth account when no explicit +// session/conversation/prompt_cache_key is present. Non-Grok groups omit this +// so HTTP OpenAI paths that delete previous_response_id before upstream are +// unchanged. func explicitOpenAIRequestSessionID(c *gin.Context, body []byte) string { if c == nil { return "" @@ -87,9 +93,27 @@ func explicitOpenAIRequestSessionID(c *gin.Context, body []byte) string { if sessionID == "" && len(body) > 0 { sessionID = strings.TrimSpace(gjson.GetBytes(body, "prompt_cache_key").String()) } + if sessionID == "" && isGrokRequestContext(c) && len(body) > 0 { + sessionID = grokPreviousResponseSessionSeed(body) + } return sessionID } +// grokPreviousResponseSessionSeed returns a stable sticky seed from a Responses +// previous_response_id. Only resp_* response ids are accepted; message ids and +// unknown shapes must not pin sticky routing or prompt-cache identity. +func grokPreviousResponseSessionSeed(body []byte) string { + id := strings.TrimSpace(gjson.GetBytes(body, "previous_response_id").String()) + if id == "" { + return "" + } + if ClassifyOpenAIPreviousResponseIDKind(id) != OpenAIPreviousResponseIDKindResponseID { + return "" + } + // Namespace so content-derived seeds never collide with response ids. + return "grok-prev-resp:" + id +} + // GenerateExplicitSessionHash generates a sticky-session hash only from explicit // client session signals. It intentionally skips content-derived fallback and is // used by stateless endpoints such as /v1/images. diff --git a/frontend/src/components/account/OAuthAuthorizationFlow.vue b/frontend/src/components/account/OAuthAuthorizationFlow.vue index 98ae7fe519..129f04bd0f 100644 --- a/frontend/src/components/account/OAuthAuthorizationFlow.vue +++ b/frontend/src/components/account/OAuthAuthorizationFlow.vue @@ -924,6 +924,11 @@ interface Props { showEmailPasswordOption?: boolean showManualOption?: boolean initialInputMethod?: AuthInputMethod + /** + * Prefill for Grok email----password reauth. Password is never stored; + * pass only the email (or "email----") so the operator types the password. + */ + initialEmailPassword?: string platform?: AccountPlatform // Platform type for different UI/text showProjectId?: boolean // New prop to control project ID visibility } @@ -949,6 +954,7 @@ const props = withDefaults(defineProps(), { showEmailPasswordOption: false, showManualOption: true, initialInputMethod: 'manual', + initialEmailPassword: '', platform: 'anthropic', showProjectId: true }) @@ -1010,7 +1016,7 @@ const sessionTokenInput = ref('') const codexSessionInput = ref('') const codexPATInput = ref('') const ssoCookieInput = ref('') -const emailPasswordInput = ref('') +const emailPasswordInput = ref(props.initialEmailPassword || '') const showHelpDialog = ref(false) const oauthState = ref('') const projectId = ref('') @@ -1085,6 +1091,16 @@ watch(() => props.initialInputMethod, (newVal) => { inputMethod.value = newVal }) +watch( + () => props.initialEmailPassword, + (newVal) => { + // Only prefill when the field is empty so we never overwrite operator input. + if (newVal && !emailPasswordInput.value.trim()) { + emailPasswordInput.value = newVal + } + } +) + watch(inputMethod, (newVal) => { emit('update:inputMethod', newVal) }) diff --git a/frontend/src/components/admin/account/ReAuthAccountModal.vue b/frontend/src/components/admin/account/ReAuthAccountModal.vue index 028c229d6a..a476a5684c 100644 --- a/frontend/src/components/admin/account/ReAuthAccountModal.vue +++ b/frontend/src/components/admin/account/ReAuthAccountModal.vue @@ -137,6 +137,8 @@ :method-label="t('admin.accounts.inputMethod')" :platform="isOpenAI ? 'openai' : isGemini ? 'gemini' : isAntigravity ? 'antigravity' : isGrok ? 'grok' : 'anthropic'" :show-project-id="isGemini && geminiOAuthType === 'code_assist'" + :initial-input-method="grokInitialInputMethod" + :initial-email-password="grokPrefillEmailPassword" @generate-url="handleGenerateUrl" @cookie-auth="handleCookieAuth" @validate-refresh-token="handleGrokValidateRefreshToken" @@ -255,6 +257,40 @@ const isAnthropic = computed(() => props.account?.platform === 'anthropic') const isAntigravity = computed(() => props.account?.platform === 'antigravity') const isGrok = computed(() => props.account?.platform === 'grok') +/** Stored Grok email for reauth prefill (password is never stored). */ +const grokAccountEmail = computed(() => { + if (!isGrok.value || !props.account) return '' + const creds = (props.account.credentials || {}) as Record + const email = typeof creds.email === 'string' ? creds.email.trim() : '' + return email +}) + +/** + * Prefill "email----" so the operator only types the password. + * Empty when no email is known (full email----password required). + */ +const grokPrefillEmailPassword = computed(() => { + const email = grokAccountEmail.value + return email ? `${email}----` : '' +}) + +/** + * Grok reauth default tab: + * - password first when we know the email (common reauth path) + * - refresh_token when email unknown but RT may still work + * - email_password otherwise + */ +const grokInitialInputMethod = computed(() => { + if (!isGrok.value) return 'manual' + if (grokAccountEmail.value) return 'email_password' + const creds = (props.account?.credentials || {}) as Record + const hasRT = + (typeof creds.refresh_token === 'string' && creds.refresh_token.trim() !== '') || + (typeof creds.has_refresh_token === 'boolean' && creds.has_refresh_token) + if (hasRT) return 'refresh_token' + return 'email_password' +}) + // Computed - current OAuth state based on platform const currentAuthUrl = computed(() => { if (isOpenAILike.value) return openaiOAuth.authUrl.value diff --git a/frontend/src/components/admin/account/__tests__/ReAuthAccountModal.grok.spec.ts b/frontend/src/components/admin/account/__tests__/ReAuthAccountModal.grok.spec.ts index dd8c368766..1291354c59 100644 --- a/frontend/src/components/admin/account/__tests__/ReAuthAccountModal.grok.spec.ts +++ b/frontend/src/components/admin/account/__tests__/ReAuthAccountModal.grok.spec.ts @@ -32,4 +32,13 @@ describe('ReAuthAccountModal Grok re-auth paths', () => { expect(source).toContain("method === 'email_password'") expect(source).toContain("method === 'refresh_token'") }) + + it('prefills email---- and defaults to password method when email is known', () => { + expect(source).toContain('grokPrefillEmailPassword') + expect(source).toContain('grokInitialInputMethod') + expect(source).toContain(':initial-email-password="grokPrefillEmailPassword"') + expect(source).toContain(':initial-input-method="grokInitialInputMethod"') + expect(source).toContain('email----') + expect(source).toContain("return 'email_password'") + }) })