feat(grok): 对齐 free-usage/empty 失败分类与 sticky/ReAuth 体验

从 grokcli 吸收 body-first 失败分类(free-usage/empty/billing),
接到现有 temp_unsched 与 failover,保留 content-policy 与 pool_mode。
Grok 粘连/缓存在无显式 session 时可用 previous_response_id;
ReAuth 预填 email---- 并默认密码 tab(密码不落库)。
This commit is contained in:
IanShaw027
2026-08-07 13:47:09 +08:00
parent 9bc99e8b6e
commit e12e0dc1a6
10 changed files with 806 additions and 4 deletions
@@ -191,10 +191,17 @@ func grokContentPolicyClientMessage(responseBody []byte) string {
// shouldFailoverGrokUpstreamError is the body-aware counterpart of the
// status-only failover helper. Grok content refusals must stay on the current
// account and be returned to the caller instead of consuming the account pool.
// Free-usage / empty-output / billing bodies also failover even when the HTTP
// status alone would not (e.g. 400 with free-usage-exhausted).
func (s *OpenAIGatewayService) shouldFailoverGrokUpstreamError(statusCode int, responseBody []byte) bool {
if isGrokContentPolicyRejection(statusCode, responseBody) {
return false
}
decision := classifyGrokUpstreamFailure(statusCode, responseBody, "")
switch decision.Class {
case GrokFailureFreeUsage, GrokFailureEmptyUpstream, GrokFailureBilling, GrokFailureModelCapacity:
return decision.ShouldFailover
}
return s.shouldFailoverUpstreamError(statusCode)
}
@@ -0,0 +1,494 @@
package service
import (
"context"
"encoding/json"
"net/http"
"regexp"
"strconv"
"strings"
"time"
"github.com/tidwall/gjson"
)
// Grok upstream failure classes used to decide temp-unschedulable cooldowns and
// pre-commit account failover. Classification is body-first so free-usage and
// empty-output wording still win when the proxy rewrites status codes.
type GrokUpstreamFailureClass string
const (
GrokFailureNone GrokUpstreamFailureClass = ""
GrokFailureFreeUsage GrokUpstreamFailureClass = "subscription:free-usage-exhausted"
GrokFailureBilling GrokUpstreamFailureClass = "billing_quota"
GrokFailureEmptyUpstream GrokUpstreamFailureClass = "empty_upstream"
GrokFailureModelCapacity GrokUpstreamFailureClass = "model_capacity"
GrokFailureRateLimit GrokUpstreamFailureClass = "rate_limit"
GrokFailureAuth GrokUpstreamFailureClass = "auth_error"
GrokFailureServer GrokUpstreamFailureClass = "server_error"
)
// GrokUpstreamFailureDecision is a pure classification result. Callers map it
// onto existing account state helpers (tempUnscheduleGrok / rateLimitGrok).
// BlockModel is retained for observability; the current scheduler does not
// implement per-model soft-blocks, so free-usage deliberately never sets it.
type GrokUpstreamFailureDecision struct {
Class GrokUpstreamFailureClass
Model string
Cooldown time.Duration
ShouldCooldown bool
// ShouldFailover recommends trying another account before writing a
// terminal response (pre-commit only). Content-policy rejections are
// handled separately and never reach this classifier for failover.
ShouldFailover bool
// BlockModel is true only for empty-output when a model id is known.
// Free-usage never sets this: the account cools, not a single model.
BlockModel bool
Reason string
TokensActual *int64
TokensLimit *int64
}
var (
reGrokTokenPair = regexp.MustCompile(`(?i)tokens?\s*(?:\(actual\s*/\s*limit\))?\s*[:=]?\s*(\d+)\s*/\s*(\d+)`)
reGrokModelFor = regexp.MustCompile(`(?i)(?:for\s+model|model|模型)\s*[::]?\s*([a-z0-9][a-z0-9._-]{2,80})`)
reGrokResetsWindow = regexp.MustCompile(`(?i)resets?\s+over\s+a\s+rolling\s+(\d+)\s*-\s*hour`)
)
// classifyGrokUpstreamFailure decides cooldown/failover from status + body.
// Priority (body/code first, status second):
// 1. free-usage exhausted → account cool, no model block, failover
// 2. billing hard quota → longer cool, failover
// 3. empty model output → short cool + optional model soft-block marker, failover
// 4. model capacity → short cool, failover
// 5. bare rate-limit / 429 without free-usage language → cool, failover
// 6. bare 5xx → brief cool, failover
// 7. validation / client errors without quota language → no cool
//
// Content-policy 403s must be filtered by the caller before invoking this.
func classifyGrokUpstreamFailure(statusCode int, responseBody []byte, requestedModel string) GrokUpstreamFailureDecision {
text, code, low := grokUpstreamErrorCorpus(statusCode, responseBody)
model := extractGrokFailureModel(text, responseBody, requestedModel)
actual, limit, hasTokens := parseGrokTokenPair(text)
if !hasTokens {
actual, limit, hasTokens = parseGrokTokenPair(string(responseBody))
}
// --- Free usage / rolling quota exhausted ---
if isGrokFreeUsageExhaustedText(low) || isGrokFreeUsageCode(code) || isGrokFreeUsageCode(text) {
d := GrokUpstreamFailureDecision{
Class: GrokFailureFreeUsage,
Model: model,
Cooldown: grokFreeUsageCooldownDuration(low),
ShouldCooldown: true,
ShouldFailover: true,
BlockModel: false,
Reason: firstNonEmpty(text, code, "free usage exhausted"),
}
if hasTokens {
a, b := actual, limit
d.TokensActual = &a
d.TokensLimit = &b
}
return d
}
// Billing / hard quota (not free-tier rolling).
// Cooldown stays at 30m to match the existing Grok 402/spending-limit
// handler (longer cools would change ops behavior without a settings knob).
if isGrokBillingQuotaText(low) || statusCode == http.StatusPaymentRequired {
reason := firstNonEmpty(text, "billing quota")
if statusCode == http.StatusPaymentRequired && text == "" {
reason = "payment required"
}
return GrokUpstreamFailureDecision{
Class: GrokFailureBilling,
Model: model,
Cooldown: 30 * time.Minute,
ShouldCooldown: true,
ShouldFailover: true,
BlockModel: model != "",
Reason: reason,
}
}
// Empty HTTP 200 / empty model output (often rewritten to synthetic 502).
if isGrokEmptyModelOutputText(low) || isGrokEmptyModelOutputCode(code) {
return GrokUpstreamFailureDecision{
Class: GrokFailureEmptyUpstream,
Model: model,
Cooldown: 4 * time.Minute,
ShouldCooldown: true,
ShouldFailover: true,
BlockModel: model != "",
Reason: firstNonEmpty(text, "empty model output"),
}
}
// Model capacity / overloaded.
if isGrokModelCapacityText(low) {
return GrokUpstreamFailureDecision{
Class: GrokFailureModelCapacity,
Model: model,
Cooldown: 3 * time.Minute,
ShouldCooldown: true,
ShouldFailover: true,
BlockModel: false,
Reason: firstNonEmpty(text, "model capacity"),
}
}
// Rate limit without free-usage language.
if statusCode == http.StatusTooManyRequests || isGrokRateLimitText(low) {
return GrokUpstreamFailureDecision{
Class: GrokFailureRateLimit,
Model: model,
Cooldown: 10 * time.Minute,
ShouldCooldown: true,
ShouldFailover: true,
BlockModel: false,
Reason: firstNonEmpty(text, "rate limit"),
}
}
// Upstream 5xx — brief cool. Empty-output synthetic 502 already handled above.
if statusCode >= 500 && statusCode <= 599 {
return GrokUpstreamFailureDecision{
Class: GrokFailureServer,
Cooldown: 2 * time.Minute,
ShouldCooldown: true,
ShouldFailover: true,
Reason: firstNonEmpty(text, "server error"),
}
}
return GrokUpstreamFailureDecision{Reason: text}
}
func grokUpstreamErrorCorpus(statusCode int, responseBody []byte) (text, code, low string) {
raw := strings.TrimSpace(string(responseBody))
if unwrappedStatus, unwrappedBody, ok := unwrapGrokUpstreamErrorText(raw); ok {
if statusCode <= 0 {
statusCode = unwrappedStatus
}
raw = unwrappedBody
}
text = raw
codeFromJSON, msgFromJSON := parseGrokUpstreamErrorJSON(raw)
if msgFromJSON != "" {
if text == "" || len(msgFromJSON) > len(text)/2 || looksLikeGrokQuotaMessage(msgFromJSON) {
text = msgFromJSON
}
}
// Prefer structured fields from the original body when present.
if len(responseBody) > 0 {
if m := strings.TrimSpace(firstNonEmpty(
gjson.GetBytes(responseBody, "error.message").String(),
gjson.GetBytes(responseBody, "message").String(),
gjson.GetBytes(responseBody, "error").String(),
)); m != "" && (text == "" || looksLikeGrokQuotaMessage(m)) {
text = m
}
if c := strings.TrimSpace(firstNonEmpty(
gjson.GetBytes(responseBody, "error.code").String(),
gjson.GetBytes(responseBody, "code").String(),
)); c != "" {
codeFromJSON = c
}
}
code = codeFromJSON
low = strings.ToLower(strings.TrimSpace(text))
if code != "" && !strings.Contains(low, strings.ToLower(code)) {
low = strings.ToLower(code) + " " + low
}
return text, code, low
}
func unwrapGrokUpstreamErrorText(errText string) (status int, body string, ok bool) {
text := strings.TrimSpace(errText)
if text == "" {
return 0, "", false
}
lower := strings.ToLower(text)
for _, p := range []string{"upstream status ", "status "} {
if !strings.HasPrefix(lower, p) {
continue
}
rest := strings.TrimSpace(text[len(p):])
i := 0
for i < len(rest) && rest[i] >= '0' && rest[i] <= '9' {
status = status*10 + int(rest[i]-'0')
i++
}
if status <= 0 || i == 0 {
return 0, "", false
}
rest = strings.TrimSpace(rest[i:])
if strings.HasPrefix(rest, ":") {
rest = strings.TrimSpace(rest[1:])
}
return status, rest, true
}
return 0, "", false
}
func parseGrokUpstreamErrorJSON(errText string) (code, message string) {
text := strings.TrimSpace(errText)
if text == "" || text[0] != '{' {
return "", ""
}
var payload map[string]any
if json.Unmarshal([]byte(text), &payload) != nil {
return "", ""
}
if v, ok := payload["code"].(string); ok {
code = v
}
if v, ok := payload["message"].(string); ok {
message = v
}
if errObj, ok := payload["error"].(map[string]any); ok {
if v, ok := errObj["code"].(string); ok && code == "" {
code = v
}
if v, ok := errObj["message"].(string); ok && message == "" {
message = v
}
}
if errStr, ok := payload["error"].(string); ok && message == "" {
message = errStr
}
return strings.TrimSpace(code), strings.TrimSpace(message)
}
func looksLikeGrokQuotaMessage(s string) bool {
low := strings.ToLower(s)
return strings.Contains(low, "quota") ||
strings.Contains(low, "usage") ||
strings.Contains(low, "credit") ||
strings.Contains(low, "额度") ||
strings.Contains(low, "free")
}
func isGrokFreeUsageCode(code string) bool {
c := strings.ToLower(strings.TrimSpace(code))
if c == "" {
return false
}
if strings.Contains(c, "subscription:free-usage-exhausted") ||
strings.Contains(c, "free-usage-exhausted") ||
strings.Contains(c, "free_usage_exhausted") ||
strings.Contains(c, "usage-limit-exceeded") ||
strings.Contains(c, "usage_limit_exceeded") {
return true
}
return (strings.Contains(c, "free-usage") || strings.Contains(c, "free_usage")) &&
(strings.Contains(c, "exhaust") || strings.Contains(c, "exceed") || strings.Contains(c, "limit"))
}
func isGrokFreeUsageExhaustedText(low string) bool {
if low == "" {
return false
}
if strings.Contains(low, "free-usage-exhausted") ||
strings.Contains(low, "free_usage_exhausted") ||
strings.Contains(low, "subscription:free-usage") ||
strings.Contains(low, "usage-limit-exceeded") ||
strings.Contains(low, "usage_limit_exceeded") ||
strings.Contains(low, "free-tier-limit") ||
strings.Contains(low, "free_tier_limit") {
return true
}
if strings.Contains(low, "free usage") ||
strings.Contains(low, "included free usage") ||
strings.Contains(low, "used all the included free") ||
strings.Contains(low, "you've used all the included free") ||
strings.Contains(low, "you have used all the included free") ||
strings.Contains(low, "free quota") ||
strings.Contains(low, "no remaining free") ||
strings.Contains(low, "out of free") ||
strings.Contains(low, "usage resets over a rolling") ||
(strings.Contains(low, "free tier") && (strings.Contains(low, "exhaust") || strings.Contains(low, "limit") || strings.Contains(low, "exceed"))) {
return true
}
for _, p := range []string{
"额度耗尽", "额度用完", "额度不足", "额度已用尽", "额度已耗尽",
"免费额度", "免费用量", "用量用完", "用量耗尽", "用量超限", "用量已用尽",
"配额耗尽", "配额已用尽", "配额不足", "配额超限", "配额用完",
"没有额度", "没额度", "无额度", "可用额度不足", "模型额度",
"临时额度", "额度已满", "额度超限", "额度达到上限",
"模型额度用完", "模型额度耗尽", "账号额度用完", "账号额度耗尽",
"额度不够", "没额度了", "额度没了", "用完额度", "耗尽额度",
} {
if strings.Contains(low, p) {
return true
}
}
if (strings.Contains(low, "quota") && (strings.Contains(low, "exhaust") || strings.Contains(low, "exceed") || strings.Contains(low, "limit"))) ||
(strings.Contains(low, "usage") && (strings.Contains(low, "exhaust") || strings.Contains(low, "exceed")) && (strings.Contains(low, "limit") || strings.Contains(low, "free") || strings.Contains(low, "model"))) {
if strings.Contains(low, "free") || strings.Contains(low, "rolling") ||
strings.Contains(low, "24-hour") || strings.Contains(low, "24 hour") ||
strings.Contains(low, "model") || strings.Contains(low, "subscription") ||
strings.Contains(low, "included") || strings.Contains(low, "tokens") {
return true
}
}
if a, b, ok := parseGrokTokenPair(low); ok && b > 0 && a >= b {
if strings.Contains(low, "free") || strings.Contains(low, "subscription") ||
strings.Contains(low, "included") || strings.Contains(low, "model") ||
strings.Contains(low, "usage") || strings.Contains(low, "quota") ||
strings.Contains(low, "rolling") {
return true
}
}
return false
}
func isGrokBillingQuotaText(low string) bool {
if low == "" {
return false
}
if strings.Contains(low, "insufficient_quota") {
return true
}
if strings.Contains(low, "billing") && strings.Contains(low, "quota") {
return true
}
if strings.Contains(low, "payment") && (strings.Contains(low, "required") || strings.Contains(low, "fail")) {
return true
}
if strings.Contains(low, "spending limit") || strings.Contains(low, "run out of credits") || strings.Contains(low, "out of credits") {
return true
}
if strings.Contains(low, "余额不足") || strings.Contains(low, "欠费") || strings.Contains(low, "需要付费") {
return true
}
return false
}
func isGrokModelCapacityText(low string) bool {
return strings.Contains(low, "capacity") ||
strings.Contains(low, "overloaded") ||
strings.Contains(low, "server_busy") ||
strings.Contains(low, "too many concurrent") ||
strings.Contains(low, "engine_overloaded")
}
func isGrokRateLimitText(low string) bool {
return strings.Contains(low, "rate limit") ||
strings.Contains(low, "rate_limit") ||
strings.Contains(low, "too many requests") ||
strings.Contains(low, "请求过于频繁") ||
strings.Contains(low, "速率限制")
}
func isGrokEmptyModelOutputText(low string) bool {
if low == "" {
return false
}
return strings.Contains(low, "empty model output") ||
strings.Contains(low, "no content/tool_calls") ||
strings.Contains(low, "no client-visible content") ||
strings.Contains(low, "empty_upstream") ||
strings.Contains(low, "empty upstream")
}
func isGrokEmptyModelOutputCode(code string) bool {
c := strings.ToLower(strings.TrimSpace(code))
if c == "" {
return false
}
return c == "empty_upstream" ||
c == "empty-model-output" ||
c == "empty_model_output" ||
strings.Contains(c, "empty_upstream") ||
strings.Contains(c, "empty-model-output")
}
func grokFreeUsageCooldownDuration(low string) time.Duration {
if m := reGrokResetsWindow.FindStringSubmatch(low); len(m) == 2 {
hours, _ := strconv.Atoi(m[1])
if hours > 0 {
d := time.Duration(hours) * time.Hour / 12
if d < 20*time.Minute {
d = 20 * time.Minute
}
if d > 6*time.Hour {
d = 6 * time.Hour
}
return d
}
}
return 2 * time.Hour
}
func parseGrokTokenPair(errText string) (actual, limit int64, ok bool) {
m := reGrokTokenPair.FindStringSubmatch(errText)
if len(m) != 3 {
return 0, 0, false
}
a, errA := strconv.ParseInt(m[1], 10, 64)
b, errB := strconv.ParseInt(m[2], 10, 64)
if errA != nil || errB != nil {
return 0, 0, false
}
return a, b, true
}
func extractGrokFailureModel(text string, responseBody []byte, fallback string) string {
if m := reGrokModelFor.FindStringSubmatch(text); len(m) == 2 {
return strings.TrimSpace(m[1])
}
if len(responseBody) > 0 {
if m := strings.TrimSpace(firstNonEmpty(
gjson.GetBytes(responseBody, "error.model").String(),
gjson.GetBytes(responseBody, "model").String(),
)); m != "" {
return m
}
}
return strings.TrimSpace(fallback)
}
// applyGrokUpstreamFailureDecision maps a classification onto existing account
// health helpers. Returns true when the decision fully handled the error path
// (caller should not apply the status-code switch defaults again).
func (s *OpenAIGatewayService) applyGrokUpstreamFailureDecision(
ctx context.Context,
account *Account,
decision GrokUpstreamFailureDecision,
) bool {
if s == nil || account == nil || !decision.ShouldCooldown || decision.Cooldown <= 0 {
return false
}
// Keep reasons short and stable for ops UI / temp_unschedulable_reason.
var reason string
switch decision.Class {
case GrokFailureFreeUsage:
reason = "grok free usage exhausted"
case GrokFailureBilling:
low := strings.ToLower(decision.Reason)
if strings.Contains(low, "spending") || strings.Contains(low, "credits") {
reason = "grok spending limit"
} else {
// Keep the historical 402/payment reason for ops UI + regression tests.
reason = "grok payment required"
}
case GrokFailureEmptyUpstream:
reason = "grok empty model output"
case GrokFailureModelCapacity:
reason = "grok model capacity"
case GrokFailureRateLimit:
// Pure 429 without free-usage language keeps the existing rate-limit
// snapshot path (Retry-After / quota headers). Body-only rate-limit
// phrasing still cools here via ShouldCooldown from the classifier, but
// the handler only invokes this for non-RateLimit classes.
return false
case GrokFailureServer:
reason = "grok upstream temporary error"
default:
return false
}
s.tempUnscheduleGrok(ctx, account, decision.Cooldown, reason)
return true
}
@@ -0,0 +1,167 @@
//go:build unit
package service
import (
"context"
"net/http"
"testing"
"time"
"github.com/stretchr/testify/require"
)
func TestClassifyGrokUpstreamFailure_FreeUsage(t *testing.T) {
cases := []struct {
name string
status int
body string
}{
{
name: "code free-usage-exhausted",
status: http.StatusTooManyRequests,
body: `{"error":{"code":"subscription:free-usage-exhausted","message":"You've used all the included free usage for model grok-4.5. Usage resets over a rolling 24-hour window."}}`,
},
{
name: "chinese body without 429",
status: http.StatusBadRequest,
body: `{"error":{"message":"模型额度用完,请稍后再试"}}`,
},
{
name: "token pair with free marker",
status: http.StatusOK,
body: `{"error":{"message":"free usage tokens (actual / limit): 2000000 / 2000000 for model grok-4.5"}}`,
},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
d := classifyGrokUpstreamFailure(tc.status, []byte(tc.body), "grok-4.5")
require.Equal(t, GrokFailureFreeUsage, d.Class)
require.True(t, d.ShouldCooldown)
require.True(t, d.ShouldFailover)
require.False(t, d.BlockModel, "free-usage must not soft-block models")
require.GreaterOrEqual(t, d.Cooldown, 20*time.Minute)
})
}
}
func TestClassifyGrokUpstreamFailure_EmptyUpstream(t *testing.T) {
d := classifyGrokUpstreamFailure(http.StatusBadGateway, []byte(`empty model output: no content/tool_calls`), "grok-4.5")
require.Equal(t, GrokFailureEmptyUpstream, d.Class)
require.True(t, d.ShouldCooldown)
require.True(t, d.ShouldFailover)
require.True(t, d.BlockModel)
require.Equal(t, 4*time.Minute, d.Cooldown)
}
func TestClassifyGrokUpstreamFailure_Billing(t *testing.T) {
d := classifyGrokUpstreamFailure(http.StatusForbidden, []byte(`{"code":"personal-team-blocked:spending-limit","error":"spending limit reached"}`), "")
require.Equal(t, GrokFailureBilling, d.Class)
require.True(t, d.ShouldCooldown)
require.True(t, d.ShouldFailover)
}
func TestClassifyGrokUpstreamFailure_ValidationNoCool(t *testing.T) {
d := classifyGrokUpstreamFailure(http.StatusBadRequest, []byte(`{"error":{"message":"invalid tool schema"}}`), "")
require.Equal(t, GrokFailureNone, d.Class)
require.False(t, d.ShouldCooldown)
require.False(t, d.ShouldFailover)
}
func TestClassifyGrokUpstreamFailure_FreeUsageWinsOver5xx(t *testing.T) {
// Proxy may rewrite free-usage into synthetic 502; body must win.
d := classifyGrokUpstreamFailure(http.StatusBadGateway, []byte(`subscription:free-usage-exhausted for model grok-4.3`), "grok-4.3")
require.Equal(t, GrokFailureFreeUsage, d.Class)
require.NotEqual(t, GrokFailureServer, d.Class)
}
func TestShouldFailoverGrokUpstreamError_FreeUsageBody(t *testing.T) {
svc := &OpenAIGatewayService{}
body := []byte(`{"error":{"code":"subscription:free-usage-exhausted","message":"free usage exhausted"}}`)
require.True(t, svc.shouldFailoverGrokUpstreamError(http.StatusBadRequest, body))
}
func TestShouldFailoverGrokUpstreamError_ContentPolicyStillNoFailover(t *testing.T) {
svc := &OpenAIGatewayService{}
body := []byte(`{"error":{"code":"new_sensitive","message":"text is sensitive"}}`)
require.False(t, svc.shouldFailoverGrokUpstreamError(http.StatusForbidden, body))
}
func TestHandleGrokAccountUpstreamError_FreeUsageBodyCoolsAccount(t *testing.T) {
repo := &grokQuotaAccountRepo{}
svc := &OpenAIGatewayService{accountRepo: repo}
account := &Account{ID: 9101, Platform: PlatformGrok, Type: AccountTypeOAuth}
before := time.Now()
body := []byte(`{"error":{"code":"subscription:free-usage-exhausted","message":"You've used all the included free usage. Usage resets over a rolling 24-hour window."}}`)
svc.handleGrokAccountUpstreamError(context.Background(), account, http.StatusBadRequest, nil, body)
require.Equal(t, 1, repo.tempUnschedCalls)
require.Equal(t, "grok free usage exhausted", repo.lastTempUnschedReason)
// 24h rolling → 2h cool
require.Greater(t, repo.lastTempUnschedUntil, before.Add(119*time.Minute))
require.Less(t, repo.lastTempUnschedUntil, before.Add(121*time.Minute))
}
func TestHandleGrokAccountUpstreamError_EmptyOutputCoolsAccount(t *testing.T) {
repo := &grokQuotaAccountRepo{}
svc := &OpenAIGatewayService{accountRepo: repo}
account := &Account{ID: 9102, Platform: PlatformGrok, Type: AccountTypeOAuth}
before := time.Now()
svc.handleGrokAccountUpstreamError(
context.Background(), account, http.StatusBadGateway, nil,
[]byte(`empty model output: no content/tool_calls`),
)
require.Equal(t, 1, repo.tempUnschedCalls)
require.Equal(t, "grok empty model output", repo.lastTempUnschedReason)
require.WithinDuration(t, before.Add(4*time.Minute), repo.lastTempUnschedUntil, time.Second)
}
func TestHandleGrokAccountUpstreamError_FreeUsageDoesNotCoolPoolMode(t *testing.T) {
repo := &grokQuotaAccountRepo{}
svc := &OpenAIGatewayService{accountRepo: repo}
account := &Account{
ID: 9103,
Platform: PlatformGrok,
Type: AccountTypeAPIKey,
Credentials: map[string]any{
"pool_mode": true,
},
}
body := []byte(`{"error":{"code":"subscription:free-usage-exhausted","message":"free usage exhausted"}}`)
svc.handleGrokAccountUpstreamError(context.Background(), account, http.StatusBadRequest, nil, body)
require.Zero(t, repo.tempUnschedCalls)
require.False(t, svc.isOpenAIAccountRuntimeBlocked(account))
}
func TestHandleGrokAccountUpstreamError_ContentPolicyStillNoMutation(t *testing.T) {
repo := &grokQuotaAccountRepo{}
svc := &OpenAIGatewayService{accountRepo: repo}
account := &Account{ID: 9104, Platform: PlatformGrok, Type: AccountTypeOAuth}
body := []byte(`{"error":{"code":"new_sensitive","message":"text is sensitive"}}`)
svc.handleGrokAccountUpstreamError(context.Background(), account, http.StatusForbidden, nil, body)
require.Zero(t, repo.tempUnschedCalls)
}
func TestHandleGrokAccountUpstreamError_Entitlement403Unchanged(t *testing.T) {
repo := &grokQuotaAccountRepo{}
svc := &OpenAIGatewayService{accountRepo: repo}
account := &Account{ID: 9105, Platform: PlatformGrok, Type: AccountTypeOAuth}
before := time.Now()
svc.handleGrokAccountUpstreamError(
context.Background(), account, http.StatusForbidden, nil,
[]byte(`{"error":{"message":"subscription required"}}`),
)
require.Equal(t, 1, repo.tempUnschedCalls)
require.Equal(t, "grok access or entitlement denied", repo.lastTempUnschedReason)
require.Greater(t, repo.lastTempUnschedUntil, before.Add(29*time.Minute))
require.Less(t, repo.lastTempUnschedUntil, before.Add(31*time.Minute))
}
@@ -1356,6 +1356,19 @@ func (s *OpenAIGatewayService) handleGrokAccountUpstreamError(ctx context.Contex
}
now := time.Now()
s.updateGrokUsageSnapshot(ctx, account, parseGrokQuotaSnapshot(headers, statusCode, now))
// Body-first free-usage / empty / billing / capacity must run before the
// status switch so non-429 free-usage bodies still cool the account.
// Pool-mode still skips durable mutation unless an explicit temp rule matches.
decision := classifyGrokUpstreamFailure(statusCode, responseBody, "")
if decision.ShouldCooldown && decision.Class != GrokFailureNone && decision.Class != GrokFailureRateLimit {
if account.IsPoolMode() {
// Allow configured temp rules (403) below; skip default body cools.
} else if s.applyGrokUpstreamFailureDecision(ctx, account, decision) {
return
}
}
if statusCode == http.StatusForbidden && s.applyGrokForbiddenPolicy(ctx, account, responseBody) {
return
}
@@ -1367,11 +1380,10 @@ func (s *OpenAIGatewayService) handleGrokAccountUpstreamError(ctx context.Contex
case http.StatusUnauthorized:
s.tempUnscheduleGrok(ctx, account, 10*time.Minute, "grok credentials unauthorized")
case http.StatusPaymentRequired:
// 402: temporarily unschedulable with a clear payment-required reason.
// 402 without a body-classified billing decision: keep the legacy 30m cool.
s.tempUnscheduleGrok(ctx, account, 30*time.Minute, "grok payment required")
case http.StatusForbidden:
// Spending-limit 403 (personal-team-blocked:spending-limit) is billing exhaustion,
// not a generic entitlement denial — still temp-unschedule with a distinct reason.
// Spending-limit already handled by body classifier when phrasing matches.
if isGrokSpendingLimitError(responseBody) {
s.tempUnscheduleGrok(ctx, account, 30*time.Minute, "grok spending limit")
return
@@ -1379,6 +1391,7 @@ func (s *OpenAIGatewayService) handleGrokAccountUpstreamError(ctx context.Contex
s.tempUnscheduleGrok(ctx, account, 30*time.Minute, "grok access or entitlement denied")
case http.StatusTooManyRequests:
// updateGrokUsageSnapshot installs rate-limit state for non-pool accounts.
// Free-usage 429 was already cooled above via body classification.
default:
if statusCode >= 500 {
s.tempUnscheduleGrok(ctx, account, 2*time.Minute, "grok upstream temporary error")
@@ -118,6 +118,12 @@ func explicitGrokCacheSeed(c *gin.Context, body []byte, explicitKey string) stri
if seed == "" {
seed = strings.TrimSpace(explicitKey)
}
// previous_response_id is last-resort: multi-turn Responses without an
// explicit session still share one cache identity (model is already in the
// isolated seed). Message ids are rejected by the seed helper.
if seed == "" && len(body) > 0 {
seed = grokPreviousResponseSessionSeed(body)
}
return seed
}
@@ -24,6 +24,36 @@ func newGrokCacheTestContext(apiKeyID int64) *gin.Context {
return c
}
func TestGrokPreviousResponseSessionSeed(t *testing.T) {
require.Equal(t, "grok-prev-resp:resp_abc123", grokPreviousResponseSessionSeed([]byte(`{"previous_response_id":"resp_abc123"}`)))
require.Empty(t, grokPreviousResponseSessionSeed([]byte(`{"previous_response_id":"msg_abc123"}`)))
require.Empty(t, grokPreviousResponseSessionSeed([]byte(`{"previous_response_id":""}`)))
require.Empty(t, grokPreviousResponseSessionSeed([]byte(`{}`)))
}
func TestResolveGrokCacheIdentityUsesPreviousResponseIDWhenNoOtherSeed(t *testing.T) {
gin.SetMode(gin.TestMode)
c := newGrokCacheTestContext(301)
// No prompt_cache_key / headers / reusable prefix — only previous_response_id.
body := []byte(`{"model":"grok","input":[{"role":"user","content":"follow up"}],"previous_response_id":"resp_chain_001"}`)
got := resolveGrokCacheIdentity(c, body, "", "grok-4.5")
require.NotEmpty(t, got)
// Same previous_response_id → same identity (model already in isolated seed).
again := resolveGrokCacheIdentity(c, body, "", "grok-4.5")
require.Equal(t, got, again)
// Different model → different identity (model scope).
otherModel := resolveGrokCacheIdentity(c, body, "", "grok-4.3")
require.NotEqual(t, got, otherModel)
// prompt_cache_key still wins over previous_response_id.
withCache := []byte(`{"model":"grok","prompt_cache_key":"client-session","previous_response_id":"resp_chain_001","input":[{"role":"user","content":"x"}]}`)
cacheID := resolveGrokCacheIdentity(c, withCache, "", "grok-4.5")
require.NotEmpty(t, cacheID)
require.NotEqual(t, got, cacheID)
}
func TestResolveGrokCacheIdentityStableAcrossAppendOnlyTurns(t *testing.T) {
gin.SetMode(gin.TestMode)
c := newGrokCacheTestContext(101)
@@ -75,6 +75,12 @@ func explicitOpenAISessionID(c *gin.Context, body []byte) string {
// with Grok's native conversation header only for requests authenticated to a
// Grok group. This keeps an unrelated x-grok-conv-id header from changing
// scheduling or upstream session behavior for non-Grok groups.
//
// For Grok groups only, previous_response_id is a last-resort sticky seed so
// multi-turn Responses chains stay on the same OAuth account when no explicit
// session/conversation/prompt_cache_key is present. Non-Grok groups omit this
// so HTTP OpenAI paths that delete previous_response_id before upstream are
// unchanged.
func explicitOpenAIRequestSessionID(c *gin.Context, body []byte) string {
if c == nil {
return ""
@@ -87,9 +93,27 @@ func explicitOpenAIRequestSessionID(c *gin.Context, body []byte) string {
if sessionID == "" && len(body) > 0 {
sessionID = strings.TrimSpace(gjson.GetBytes(body, "prompt_cache_key").String())
}
if sessionID == "" && isGrokRequestContext(c) && len(body) > 0 {
sessionID = grokPreviousResponseSessionSeed(body)
}
return sessionID
}
// grokPreviousResponseSessionSeed returns a stable sticky seed from a Responses
// previous_response_id. Only resp_* response ids are accepted; message ids and
// unknown shapes must not pin sticky routing or prompt-cache identity.
func grokPreviousResponseSessionSeed(body []byte) string {
id := strings.TrimSpace(gjson.GetBytes(body, "previous_response_id").String())
if id == "" {
return ""
}
if ClassifyOpenAIPreviousResponseIDKind(id) != OpenAIPreviousResponseIDKindResponseID {
return ""
}
// Namespace so content-derived seeds never collide with response ids.
return "grok-prev-resp:" + id
}
// GenerateExplicitSessionHash generates a sticky-session hash only from explicit
// client session signals. It intentionally skips content-derived fallback and is
// used by stateless endpoints such as /v1/images.
@@ -924,6 +924,11 @@ interface Props {
showEmailPasswordOption?: boolean
showManualOption?: boolean
initialInputMethod?: AuthInputMethod
/**
* Prefill for Grok email----password reauth. Password is never stored;
* pass only the email (or "email----") so the operator types the password.
*/
initialEmailPassword?: string
platform?: AccountPlatform // Platform type for different UI/text
showProjectId?: boolean // New prop to control project ID visibility
}
@@ -949,6 +954,7 @@ const props = withDefaults(defineProps<Props>(), {
showEmailPasswordOption: false,
showManualOption: true,
initialInputMethod: 'manual',
initialEmailPassword: '',
platform: 'anthropic',
showProjectId: true
})
@@ -1010,7 +1016,7 @@ const sessionTokenInput = ref('')
const codexSessionInput = ref('')
const codexPATInput = ref('')
const ssoCookieInput = ref('')
const emailPasswordInput = ref('')
const emailPasswordInput = ref(props.initialEmailPassword || '')
const showHelpDialog = ref(false)
const oauthState = ref('')
const projectId = ref('')
@@ -1085,6 +1091,16 @@ watch(() => props.initialInputMethod, (newVal) => {
inputMethod.value = newVal
})
watch(
() => props.initialEmailPassword,
(newVal) => {
// Only prefill when the field is empty so we never overwrite operator input.
if (newVal && !emailPasswordInput.value.trim()) {
emailPasswordInput.value = newVal
}
}
)
watch(inputMethod, (newVal) => {
emit('update:inputMethod', newVal)
})
@@ -137,6 +137,8 @@
:method-label="t('admin.accounts.inputMethod')"
:platform="isOpenAI ? 'openai' : isGemini ? 'gemini' : isAntigravity ? 'antigravity' : isGrok ? 'grok' : 'anthropic'"
:show-project-id="isGemini && geminiOAuthType === 'code_assist'"
:initial-input-method="grokInitialInputMethod"
:initial-email-password="grokPrefillEmailPassword"
@generate-url="handleGenerateUrl"
@cookie-auth="handleCookieAuth"
@validate-refresh-token="handleGrokValidateRefreshToken"
@@ -255,6 +257,40 @@ const isAnthropic = computed(() => props.account?.platform === 'anthropic')
const isAntigravity = computed(() => props.account?.platform === 'antigravity')
const isGrok = computed(() => props.account?.platform === 'grok')
/** Stored Grok email for reauth prefill (password is never stored). */
const grokAccountEmail = computed(() => {
if (!isGrok.value || !props.account) return ''
const creds = (props.account.credentials || {}) as Record<string, unknown>
const email = typeof creds.email === 'string' ? creds.email.trim() : ''
return email
})
/**
* Prefill "email----" so the operator only types the password.
* Empty when no email is known (full email----password required).
*/
const grokPrefillEmailPassword = computed(() => {
const email = grokAccountEmail.value
return email ? `${email}----` : ''
})
/**
* Grok reauth default tab:
* - password first when we know the email (common reauth path)
* - refresh_token when email unknown but RT may still work
* - email_password otherwise
*/
const grokInitialInputMethod = computed<AuthInputMethod>(() => {
if (!isGrok.value) return 'manual'
if (grokAccountEmail.value) return 'email_password'
const creds = (props.account?.credentials || {}) as Record<string, unknown>
const hasRT =
(typeof creds.refresh_token === 'string' && creds.refresh_token.trim() !== '') ||
(typeof creds.has_refresh_token === 'boolean' && creds.has_refresh_token)
if (hasRT) return 'refresh_token'
return 'email_password'
})
// Computed - current OAuth state based on platform
const currentAuthUrl = computed(() => {
if (isOpenAILike.value) return openaiOAuth.authUrl.value
@@ -32,4 +32,13 @@ describe('ReAuthAccountModal Grok re-auth paths', () => {
expect(source).toContain("method === 'email_password'")
expect(source).toContain("method === 'refresh_token'")
})
it('prefills email---- and defaults to password method when email is known', () => {
expect(source).toContain('grokPrefillEmailPassword')
expect(source).toContain('grokInitialInputMethod')
expect(source).toContain(':initial-email-password="grokPrefillEmailPassword"')
expect(source).toContain(':initial-input-method="grokInitialInputMethod"')
expect(source).toContain('email----')
expect(source).toContain("return 'email_password'")
})
})