mirror of
https://github.com/Wei-Shaw/sub2api.git
synced 2026-10-07 14:58:23 +08:00
feat(grok): 对齐 free-usage/empty 失败分类与 sticky/ReAuth 体验
从 grokcli 吸收 body-first 失败分类(free-usage/empty/billing), 接到现有 temp_unsched 与 failover,保留 content-policy 与 pool_mode。 Grok 粘连/缓存在无显式 session 时可用 previous_response_id; ReAuth 预填 email---- 并默认密码 tab(密码不落库)。
This commit is contained in:
@@ -191,10 +191,17 @@ func grokContentPolicyClientMessage(responseBody []byte) string {
|
||||
// shouldFailoverGrokUpstreamError is the body-aware counterpart of the
|
||||
// status-only failover helper. Grok content refusals must stay on the current
|
||||
// account and be returned to the caller instead of consuming the account pool.
|
||||
// Free-usage / empty-output / billing bodies also failover even when the HTTP
|
||||
// status alone would not (e.g. 400 with free-usage-exhausted).
|
||||
func (s *OpenAIGatewayService) shouldFailoverGrokUpstreamError(statusCode int, responseBody []byte) bool {
|
||||
if isGrokContentPolicyRejection(statusCode, responseBody) {
|
||||
return false
|
||||
}
|
||||
decision := classifyGrokUpstreamFailure(statusCode, responseBody, "")
|
||||
switch decision.Class {
|
||||
case GrokFailureFreeUsage, GrokFailureEmptyUpstream, GrokFailureBilling, GrokFailureModelCapacity:
|
||||
return decision.ShouldFailover
|
||||
}
|
||||
return s.shouldFailoverUpstreamError(statusCode)
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,494 @@
|
||||
package service
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/tidwall/gjson"
|
||||
)
|
||||
|
||||
// Grok upstream failure classes used to decide temp-unschedulable cooldowns and
|
||||
// pre-commit account failover. Classification is body-first so free-usage and
|
||||
// empty-output wording still win when the proxy rewrites status codes.
|
||||
type GrokUpstreamFailureClass string
|
||||
|
||||
const (
|
||||
GrokFailureNone GrokUpstreamFailureClass = ""
|
||||
GrokFailureFreeUsage GrokUpstreamFailureClass = "subscription:free-usage-exhausted"
|
||||
GrokFailureBilling GrokUpstreamFailureClass = "billing_quota"
|
||||
GrokFailureEmptyUpstream GrokUpstreamFailureClass = "empty_upstream"
|
||||
GrokFailureModelCapacity GrokUpstreamFailureClass = "model_capacity"
|
||||
GrokFailureRateLimit GrokUpstreamFailureClass = "rate_limit"
|
||||
GrokFailureAuth GrokUpstreamFailureClass = "auth_error"
|
||||
GrokFailureServer GrokUpstreamFailureClass = "server_error"
|
||||
)
|
||||
|
||||
// GrokUpstreamFailureDecision is a pure classification result. Callers map it
|
||||
// onto existing account state helpers (tempUnscheduleGrok / rateLimitGrok).
|
||||
// BlockModel is retained for observability; the current scheduler does not
|
||||
// implement per-model soft-blocks, so free-usage deliberately never sets it.
|
||||
type GrokUpstreamFailureDecision struct {
|
||||
Class GrokUpstreamFailureClass
|
||||
Model string
|
||||
Cooldown time.Duration
|
||||
ShouldCooldown bool
|
||||
// ShouldFailover recommends trying another account before writing a
|
||||
// terminal response (pre-commit only). Content-policy rejections are
|
||||
// handled separately and never reach this classifier for failover.
|
||||
ShouldFailover bool
|
||||
// BlockModel is true only for empty-output when a model id is known.
|
||||
// Free-usage never sets this: the account cools, not a single model.
|
||||
BlockModel bool
|
||||
Reason string
|
||||
TokensActual *int64
|
||||
TokensLimit *int64
|
||||
}
|
||||
|
||||
var (
|
||||
reGrokTokenPair = regexp.MustCompile(`(?i)tokens?\s*(?:\(actual\s*/\s*limit\))?\s*[:=]?\s*(\d+)\s*/\s*(\d+)`)
|
||||
reGrokModelFor = regexp.MustCompile(`(?i)(?:for\s+model|model|模型)\s*[::]?\s*([a-z0-9][a-z0-9._-]{2,80})`)
|
||||
reGrokResetsWindow = regexp.MustCompile(`(?i)resets?\s+over\s+a\s+rolling\s+(\d+)\s*-\s*hour`)
|
||||
)
|
||||
|
||||
// classifyGrokUpstreamFailure decides cooldown/failover from status + body.
|
||||
// Priority (body/code first, status second):
|
||||
// 1. free-usage exhausted → account cool, no model block, failover
|
||||
// 2. billing hard quota → longer cool, failover
|
||||
// 3. empty model output → short cool + optional model soft-block marker, failover
|
||||
// 4. model capacity → short cool, failover
|
||||
// 5. bare rate-limit / 429 without free-usage language → cool, failover
|
||||
// 6. bare 5xx → brief cool, failover
|
||||
// 7. validation / client errors without quota language → no cool
|
||||
//
|
||||
// Content-policy 403s must be filtered by the caller before invoking this.
|
||||
func classifyGrokUpstreamFailure(statusCode int, responseBody []byte, requestedModel string) GrokUpstreamFailureDecision {
|
||||
text, code, low := grokUpstreamErrorCorpus(statusCode, responseBody)
|
||||
model := extractGrokFailureModel(text, responseBody, requestedModel)
|
||||
actual, limit, hasTokens := parseGrokTokenPair(text)
|
||||
if !hasTokens {
|
||||
actual, limit, hasTokens = parseGrokTokenPair(string(responseBody))
|
||||
}
|
||||
|
||||
// --- Free usage / rolling quota exhausted ---
|
||||
if isGrokFreeUsageExhaustedText(low) || isGrokFreeUsageCode(code) || isGrokFreeUsageCode(text) {
|
||||
d := GrokUpstreamFailureDecision{
|
||||
Class: GrokFailureFreeUsage,
|
||||
Model: model,
|
||||
Cooldown: grokFreeUsageCooldownDuration(low),
|
||||
ShouldCooldown: true,
|
||||
ShouldFailover: true,
|
||||
BlockModel: false,
|
||||
Reason: firstNonEmpty(text, code, "free usage exhausted"),
|
||||
}
|
||||
if hasTokens {
|
||||
a, b := actual, limit
|
||||
d.TokensActual = &a
|
||||
d.TokensLimit = &b
|
||||
}
|
||||
return d
|
||||
}
|
||||
|
||||
// Billing / hard quota (not free-tier rolling).
|
||||
// Cooldown stays at 30m to match the existing Grok 402/spending-limit
|
||||
// handler (longer cools would change ops behavior without a settings knob).
|
||||
if isGrokBillingQuotaText(low) || statusCode == http.StatusPaymentRequired {
|
||||
reason := firstNonEmpty(text, "billing quota")
|
||||
if statusCode == http.StatusPaymentRequired && text == "" {
|
||||
reason = "payment required"
|
||||
}
|
||||
return GrokUpstreamFailureDecision{
|
||||
Class: GrokFailureBilling,
|
||||
Model: model,
|
||||
Cooldown: 30 * time.Minute,
|
||||
ShouldCooldown: true,
|
||||
ShouldFailover: true,
|
||||
BlockModel: model != "",
|
||||
Reason: reason,
|
||||
}
|
||||
}
|
||||
|
||||
// Empty HTTP 200 / empty model output (often rewritten to synthetic 502).
|
||||
if isGrokEmptyModelOutputText(low) || isGrokEmptyModelOutputCode(code) {
|
||||
return GrokUpstreamFailureDecision{
|
||||
Class: GrokFailureEmptyUpstream,
|
||||
Model: model,
|
||||
Cooldown: 4 * time.Minute,
|
||||
ShouldCooldown: true,
|
||||
ShouldFailover: true,
|
||||
BlockModel: model != "",
|
||||
Reason: firstNonEmpty(text, "empty model output"),
|
||||
}
|
||||
}
|
||||
|
||||
// Model capacity / overloaded.
|
||||
if isGrokModelCapacityText(low) {
|
||||
return GrokUpstreamFailureDecision{
|
||||
Class: GrokFailureModelCapacity,
|
||||
Model: model,
|
||||
Cooldown: 3 * time.Minute,
|
||||
ShouldCooldown: true,
|
||||
ShouldFailover: true,
|
||||
BlockModel: false,
|
||||
Reason: firstNonEmpty(text, "model capacity"),
|
||||
}
|
||||
}
|
||||
|
||||
// Rate limit without free-usage language.
|
||||
if statusCode == http.StatusTooManyRequests || isGrokRateLimitText(low) {
|
||||
return GrokUpstreamFailureDecision{
|
||||
Class: GrokFailureRateLimit,
|
||||
Model: model,
|
||||
Cooldown: 10 * time.Minute,
|
||||
ShouldCooldown: true,
|
||||
ShouldFailover: true,
|
||||
BlockModel: false,
|
||||
Reason: firstNonEmpty(text, "rate limit"),
|
||||
}
|
||||
}
|
||||
|
||||
// Upstream 5xx — brief cool. Empty-output synthetic 502 already handled above.
|
||||
if statusCode >= 500 && statusCode <= 599 {
|
||||
return GrokUpstreamFailureDecision{
|
||||
Class: GrokFailureServer,
|
||||
Cooldown: 2 * time.Minute,
|
||||
ShouldCooldown: true,
|
||||
ShouldFailover: true,
|
||||
Reason: firstNonEmpty(text, "server error"),
|
||||
}
|
||||
}
|
||||
|
||||
return GrokUpstreamFailureDecision{Reason: text}
|
||||
}
|
||||
|
||||
func grokUpstreamErrorCorpus(statusCode int, responseBody []byte) (text, code, low string) {
|
||||
raw := strings.TrimSpace(string(responseBody))
|
||||
if unwrappedStatus, unwrappedBody, ok := unwrapGrokUpstreamErrorText(raw); ok {
|
||||
if statusCode <= 0 {
|
||||
statusCode = unwrappedStatus
|
||||
}
|
||||
raw = unwrappedBody
|
||||
}
|
||||
text = raw
|
||||
codeFromJSON, msgFromJSON := parseGrokUpstreamErrorJSON(raw)
|
||||
if msgFromJSON != "" {
|
||||
if text == "" || len(msgFromJSON) > len(text)/2 || looksLikeGrokQuotaMessage(msgFromJSON) {
|
||||
text = msgFromJSON
|
||||
}
|
||||
}
|
||||
// Prefer structured fields from the original body when present.
|
||||
if len(responseBody) > 0 {
|
||||
if m := strings.TrimSpace(firstNonEmpty(
|
||||
gjson.GetBytes(responseBody, "error.message").String(),
|
||||
gjson.GetBytes(responseBody, "message").String(),
|
||||
gjson.GetBytes(responseBody, "error").String(),
|
||||
)); m != "" && (text == "" || looksLikeGrokQuotaMessage(m)) {
|
||||
text = m
|
||||
}
|
||||
if c := strings.TrimSpace(firstNonEmpty(
|
||||
gjson.GetBytes(responseBody, "error.code").String(),
|
||||
gjson.GetBytes(responseBody, "code").String(),
|
||||
)); c != "" {
|
||||
codeFromJSON = c
|
||||
}
|
||||
}
|
||||
code = codeFromJSON
|
||||
low = strings.ToLower(strings.TrimSpace(text))
|
||||
if code != "" && !strings.Contains(low, strings.ToLower(code)) {
|
||||
low = strings.ToLower(code) + " " + low
|
||||
}
|
||||
return text, code, low
|
||||
}
|
||||
|
||||
func unwrapGrokUpstreamErrorText(errText string) (status int, body string, ok bool) {
|
||||
text := strings.TrimSpace(errText)
|
||||
if text == "" {
|
||||
return 0, "", false
|
||||
}
|
||||
lower := strings.ToLower(text)
|
||||
for _, p := range []string{"upstream status ", "status "} {
|
||||
if !strings.HasPrefix(lower, p) {
|
||||
continue
|
||||
}
|
||||
rest := strings.TrimSpace(text[len(p):])
|
||||
i := 0
|
||||
for i < len(rest) && rest[i] >= '0' && rest[i] <= '9' {
|
||||
status = status*10 + int(rest[i]-'0')
|
||||
i++
|
||||
}
|
||||
if status <= 0 || i == 0 {
|
||||
return 0, "", false
|
||||
}
|
||||
rest = strings.TrimSpace(rest[i:])
|
||||
if strings.HasPrefix(rest, ":") {
|
||||
rest = strings.TrimSpace(rest[1:])
|
||||
}
|
||||
return status, rest, true
|
||||
}
|
||||
return 0, "", false
|
||||
}
|
||||
|
||||
func parseGrokUpstreamErrorJSON(errText string) (code, message string) {
|
||||
text := strings.TrimSpace(errText)
|
||||
if text == "" || text[0] != '{' {
|
||||
return "", ""
|
||||
}
|
||||
var payload map[string]any
|
||||
if json.Unmarshal([]byte(text), &payload) != nil {
|
||||
return "", ""
|
||||
}
|
||||
if v, ok := payload["code"].(string); ok {
|
||||
code = v
|
||||
}
|
||||
if v, ok := payload["message"].(string); ok {
|
||||
message = v
|
||||
}
|
||||
if errObj, ok := payload["error"].(map[string]any); ok {
|
||||
if v, ok := errObj["code"].(string); ok && code == "" {
|
||||
code = v
|
||||
}
|
||||
if v, ok := errObj["message"].(string); ok && message == "" {
|
||||
message = v
|
||||
}
|
||||
}
|
||||
if errStr, ok := payload["error"].(string); ok && message == "" {
|
||||
message = errStr
|
||||
}
|
||||
return strings.TrimSpace(code), strings.TrimSpace(message)
|
||||
}
|
||||
|
||||
func looksLikeGrokQuotaMessage(s string) bool {
|
||||
low := strings.ToLower(s)
|
||||
return strings.Contains(low, "quota") ||
|
||||
strings.Contains(low, "usage") ||
|
||||
strings.Contains(low, "credit") ||
|
||||
strings.Contains(low, "额度") ||
|
||||
strings.Contains(low, "free")
|
||||
}
|
||||
|
||||
func isGrokFreeUsageCode(code string) bool {
|
||||
c := strings.ToLower(strings.TrimSpace(code))
|
||||
if c == "" {
|
||||
return false
|
||||
}
|
||||
if strings.Contains(c, "subscription:free-usage-exhausted") ||
|
||||
strings.Contains(c, "free-usage-exhausted") ||
|
||||
strings.Contains(c, "free_usage_exhausted") ||
|
||||
strings.Contains(c, "usage-limit-exceeded") ||
|
||||
strings.Contains(c, "usage_limit_exceeded") {
|
||||
return true
|
||||
}
|
||||
return (strings.Contains(c, "free-usage") || strings.Contains(c, "free_usage")) &&
|
||||
(strings.Contains(c, "exhaust") || strings.Contains(c, "exceed") || strings.Contains(c, "limit"))
|
||||
}
|
||||
|
||||
func isGrokFreeUsageExhaustedText(low string) bool {
|
||||
if low == "" {
|
||||
return false
|
||||
}
|
||||
if strings.Contains(low, "free-usage-exhausted") ||
|
||||
strings.Contains(low, "free_usage_exhausted") ||
|
||||
strings.Contains(low, "subscription:free-usage") ||
|
||||
strings.Contains(low, "usage-limit-exceeded") ||
|
||||
strings.Contains(low, "usage_limit_exceeded") ||
|
||||
strings.Contains(low, "free-tier-limit") ||
|
||||
strings.Contains(low, "free_tier_limit") {
|
||||
return true
|
||||
}
|
||||
if strings.Contains(low, "free usage") ||
|
||||
strings.Contains(low, "included free usage") ||
|
||||
strings.Contains(low, "used all the included free") ||
|
||||
strings.Contains(low, "you've used all the included free") ||
|
||||
strings.Contains(low, "you have used all the included free") ||
|
||||
strings.Contains(low, "free quota") ||
|
||||
strings.Contains(low, "no remaining free") ||
|
||||
strings.Contains(low, "out of free") ||
|
||||
strings.Contains(low, "usage resets over a rolling") ||
|
||||
(strings.Contains(low, "free tier") && (strings.Contains(low, "exhaust") || strings.Contains(low, "limit") || strings.Contains(low, "exceed"))) {
|
||||
return true
|
||||
}
|
||||
for _, p := range []string{
|
||||
"额度耗尽", "额度用完", "额度不足", "额度已用尽", "额度已耗尽",
|
||||
"免费额度", "免费用量", "用量用完", "用量耗尽", "用量超限", "用量已用尽",
|
||||
"配额耗尽", "配额已用尽", "配额不足", "配额超限", "配额用完",
|
||||
"没有额度", "没额度", "无额度", "可用额度不足", "模型额度",
|
||||
"临时额度", "额度已满", "额度超限", "额度达到上限",
|
||||
"模型额度用完", "模型额度耗尽", "账号额度用完", "账号额度耗尽",
|
||||
"额度不够", "没额度了", "额度没了", "用完额度", "耗尽额度",
|
||||
} {
|
||||
if strings.Contains(low, p) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
if (strings.Contains(low, "quota") && (strings.Contains(low, "exhaust") || strings.Contains(low, "exceed") || strings.Contains(low, "limit"))) ||
|
||||
(strings.Contains(low, "usage") && (strings.Contains(low, "exhaust") || strings.Contains(low, "exceed")) && (strings.Contains(low, "limit") || strings.Contains(low, "free") || strings.Contains(low, "model"))) {
|
||||
if strings.Contains(low, "free") || strings.Contains(low, "rolling") ||
|
||||
strings.Contains(low, "24-hour") || strings.Contains(low, "24 hour") ||
|
||||
strings.Contains(low, "model") || strings.Contains(low, "subscription") ||
|
||||
strings.Contains(low, "included") || strings.Contains(low, "tokens") {
|
||||
return true
|
||||
}
|
||||
}
|
||||
if a, b, ok := parseGrokTokenPair(low); ok && b > 0 && a >= b {
|
||||
if strings.Contains(low, "free") || strings.Contains(low, "subscription") ||
|
||||
strings.Contains(low, "included") || strings.Contains(low, "model") ||
|
||||
strings.Contains(low, "usage") || strings.Contains(low, "quota") ||
|
||||
strings.Contains(low, "rolling") {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func isGrokBillingQuotaText(low string) bool {
|
||||
if low == "" {
|
||||
return false
|
||||
}
|
||||
if strings.Contains(low, "insufficient_quota") {
|
||||
return true
|
||||
}
|
||||
if strings.Contains(low, "billing") && strings.Contains(low, "quota") {
|
||||
return true
|
||||
}
|
||||
if strings.Contains(low, "payment") && (strings.Contains(low, "required") || strings.Contains(low, "fail")) {
|
||||
return true
|
||||
}
|
||||
if strings.Contains(low, "spending limit") || strings.Contains(low, "run out of credits") || strings.Contains(low, "out of credits") {
|
||||
return true
|
||||
}
|
||||
if strings.Contains(low, "余额不足") || strings.Contains(low, "欠费") || strings.Contains(low, "需要付费") {
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func isGrokModelCapacityText(low string) bool {
|
||||
return strings.Contains(low, "capacity") ||
|
||||
strings.Contains(low, "overloaded") ||
|
||||
strings.Contains(low, "server_busy") ||
|
||||
strings.Contains(low, "too many concurrent") ||
|
||||
strings.Contains(low, "engine_overloaded")
|
||||
}
|
||||
|
||||
func isGrokRateLimitText(low string) bool {
|
||||
return strings.Contains(low, "rate limit") ||
|
||||
strings.Contains(low, "rate_limit") ||
|
||||
strings.Contains(low, "too many requests") ||
|
||||
strings.Contains(low, "请求过于频繁") ||
|
||||
strings.Contains(low, "速率限制")
|
||||
}
|
||||
|
||||
func isGrokEmptyModelOutputText(low string) bool {
|
||||
if low == "" {
|
||||
return false
|
||||
}
|
||||
return strings.Contains(low, "empty model output") ||
|
||||
strings.Contains(low, "no content/tool_calls") ||
|
||||
strings.Contains(low, "no client-visible content") ||
|
||||
strings.Contains(low, "empty_upstream") ||
|
||||
strings.Contains(low, "empty upstream")
|
||||
}
|
||||
|
||||
func isGrokEmptyModelOutputCode(code string) bool {
|
||||
c := strings.ToLower(strings.TrimSpace(code))
|
||||
if c == "" {
|
||||
return false
|
||||
}
|
||||
return c == "empty_upstream" ||
|
||||
c == "empty-model-output" ||
|
||||
c == "empty_model_output" ||
|
||||
strings.Contains(c, "empty_upstream") ||
|
||||
strings.Contains(c, "empty-model-output")
|
||||
}
|
||||
|
||||
func grokFreeUsageCooldownDuration(low string) time.Duration {
|
||||
if m := reGrokResetsWindow.FindStringSubmatch(low); len(m) == 2 {
|
||||
hours, _ := strconv.Atoi(m[1])
|
||||
if hours > 0 {
|
||||
d := time.Duration(hours) * time.Hour / 12
|
||||
if d < 20*time.Minute {
|
||||
d = 20 * time.Minute
|
||||
}
|
||||
if d > 6*time.Hour {
|
||||
d = 6 * time.Hour
|
||||
}
|
||||
return d
|
||||
}
|
||||
}
|
||||
return 2 * time.Hour
|
||||
}
|
||||
|
||||
func parseGrokTokenPair(errText string) (actual, limit int64, ok bool) {
|
||||
m := reGrokTokenPair.FindStringSubmatch(errText)
|
||||
if len(m) != 3 {
|
||||
return 0, 0, false
|
||||
}
|
||||
a, errA := strconv.ParseInt(m[1], 10, 64)
|
||||
b, errB := strconv.ParseInt(m[2], 10, 64)
|
||||
if errA != nil || errB != nil {
|
||||
return 0, 0, false
|
||||
}
|
||||
return a, b, true
|
||||
}
|
||||
|
||||
func extractGrokFailureModel(text string, responseBody []byte, fallback string) string {
|
||||
if m := reGrokModelFor.FindStringSubmatch(text); len(m) == 2 {
|
||||
return strings.TrimSpace(m[1])
|
||||
}
|
||||
if len(responseBody) > 0 {
|
||||
if m := strings.TrimSpace(firstNonEmpty(
|
||||
gjson.GetBytes(responseBody, "error.model").String(),
|
||||
gjson.GetBytes(responseBody, "model").String(),
|
||||
)); m != "" {
|
||||
return m
|
||||
}
|
||||
}
|
||||
return strings.TrimSpace(fallback)
|
||||
}
|
||||
|
||||
// applyGrokUpstreamFailureDecision maps a classification onto existing account
|
||||
// health helpers. Returns true when the decision fully handled the error path
|
||||
// (caller should not apply the status-code switch defaults again).
|
||||
func (s *OpenAIGatewayService) applyGrokUpstreamFailureDecision(
|
||||
ctx context.Context,
|
||||
account *Account,
|
||||
decision GrokUpstreamFailureDecision,
|
||||
) bool {
|
||||
if s == nil || account == nil || !decision.ShouldCooldown || decision.Cooldown <= 0 {
|
||||
return false
|
||||
}
|
||||
// Keep reasons short and stable for ops UI / temp_unschedulable_reason.
|
||||
var reason string
|
||||
switch decision.Class {
|
||||
case GrokFailureFreeUsage:
|
||||
reason = "grok free usage exhausted"
|
||||
case GrokFailureBilling:
|
||||
low := strings.ToLower(decision.Reason)
|
||||
if strings.Contains(low, "spending") || strings.Contains(low, "credits") {
|
||||
reason = "grok spending limit"
|
||||
} else {
|
||||
// Keep the historical 402/payment reason for ops UI + regression tests.
|
||||
reason = "grok payment required"
|
||||
}
|
||||
case GrokFailureEmptyUpstream:
|
||||
reason = "grok empty model output"
|
||||
case GrokFailureModelCapacity:
|
||||
reason = "grok model capacity"
|
||||
case GrokFailureRateLimit:
|
||||
// Pure 429 without free-usage language keeps the existing rate-limit
|
||||
// snapshot path (Retry-After / quota headers). Body-only rate-limit
|
||||
// phrasing still cools here via ShouldCooldown from the classifier, but
|
||||
// the handler only invokes this for non-RateLimit classes.
|
||||
return false
|
||||
case GrokFailureServer:
|
||||
reason = "grok upstream temporary error"
|
||||
default:
|
||||
return false
|
||||
}
|
||||
s.tempUnscheduleGrok(ctx, account, decision.Cooldown, reason)
|
||||
return true
|
||||
}
|
||||
@@ -0,0 +1,167 @@
|
||||
//go:build unit
|
||||
|
||||
package service
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestClassifyGrokUpstreamFailure_FreeUsage(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
status int
|
||||
body string
|
||||
}{
|
||||
{
|
||||
name: "code free-usage-exhausted",
|
||||
status: http.StatusTooManyRequests,
|
||||
body: `{"error":{"code":"subscription:free-usage-exhausted","message":"You've used all the included free usage for model grok-4.5. Usage resets over a rolling 24-hour window."}}`,
|
||||
},
|
||||
{
|
||||
name: "chinese body without 429",
|
||||
status: http.StatusBadRequest,
|
||||
body: `{"error":{"message":"模型额度用完,请稍后再试"}}`,
|
||||
},
|
||||
{
|
||||
name: "token pair with free marker",
|
||||
status: http.StatusOK,
|
||||
body: `{"error":{"message":"free usage tokens (actual / limit): 2000000 / 2000000 for model grok-4.5"}}`,
|
||||
},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
d := classifyGrokUpstreamFailure(tc.status, []byte(tc.body), "grok-4.5")
|
||||
require.Equal(t, GrokFailureFreeUsage, d.Class)
|
||||
require.True(t, d.ShouldCooldown)
|
||||
require.True(t, d.ShouldFailover)
|
||||
require.False(t, d.BlockModel, "free-usage must not soft-block models")
|
||||
require.GreaterOrEqual(t, d.Cooldown, 20*time.Minute)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestClassifyGrokUpstreamFailure_EmptyUpstream(t *testing.T) {
|
||||
d := classifyGrokUpstreamFailure(http.StatusBadGateway, []byte(`empty model output: no content/tool_calls`), "grok-4.5")
|
||||
require.Equal(t, GrokFailureEmptyUpstream, d.Class)
|
||||
require.True(t, d.ShouldCooldown)
|
||||
require.True(t, d.ShouldFailover)
|
||||
require.True(t, d.BlockModel)
|
||||
require.Equal(t, 4*time.Minute, d.Cooldown)
|
||||
}
|
||||
|
||||
func TestClassifyGrokUpstreamFailure_Billing(t *testing.T) {
|
||||
d := classifyGrokUpstreamFailure(http.StatusForbidden, []byte(`{"code":"personal-team-blocked:spending-limit","error":"spending limit reached"}`), "")
|
||||
require.Equal(t, GrokFailureBilling, d.Class)
|
||||
require.True(t, d.ShouldCooldown)
|
||||
require.True(t, d.ShouldFailover)
|
||||
}
|
||||
|
||||
func TestClassifyGrokUpstreamFailure_ValidationNoCool(t *testing.T) {
|
||||
d := classifyGrokUpstreamFailure(http.StatusBadRequest, []byte(`{"error":{"message":"invalid tool schema"}}`), "")
|
||||
require.Equal(t, GrokFailureNone, d.Class)
|
||||
require.False(t, d.ShouldCooldown)
|
||||
require.False(t, d.ShouldFailover)
|
||||
}
|
||||
|
||||
func TestClassifyGrokUpstreamFailure_FreeUsageWinsOver5xx(t *testing.T) {
|
||||
// Proxy may rewrite free-usage into synthetic 502; body must win.
|
||||
d := classifyGrokUpstreamFailure(http.StatusBadGateway, []byte(`subscription:free-usage-exhausted for model grok-4.3`), "grok-4.3")
|
||||
require.Equal(t, GrokFailureFreeUsage, d.Class)
|
||||
require.NotEqual(t, GrokFailureServer, d.Class)
|
||||
}
|
||||
|
||||
func TestShouldFailoverGrokUpstreamError_FreeUsageBody(t *testing.T) {
|
||||
svc := &OpenAIGatewayService{}
|
||||
body := []byte(`{"error":{"code":"subscription:free-usage-exhausted","message":"free usage exhausted"}}`)
|
||||
require.True(t, svc.shouldFailoverGrokUpstreamError(http.StatusBadRequest, body))
|
||||
}
|
||||
|
||||
func TestShouldFailoverGrokUpstreamError_ContentPolicyStillNoFailover(t *testing.T) {
|
||||
svc := &OpenAIGatewayService{}
|
||||
body := []byte(`{"error":{"code":"new_sensitive","message":"text is sensitive"}}`)
|
||||
require.False(t, svc.shouldFailoverGrokUpstreamError(http.StatusForbidden, body))
|
||||
}
|
||||
|
||||
func TestHandleGrokAccountUpstreamError_FreeUsageBodyCoolsAccount(t *testing.T) {
|
||||
repo := &grokQuotaAccountRepo{}
|
||||
svc := &OpenAIGatewayService{accountRepo: repo}
|
||||
account := &Account{ID: 9101, Platform: PlatformGrok, Type: AccountTypeOAuth}
|
||||
before := time.Now()
|
||||
body := []byte(`{"error":{"code":"subscription:free-usage-exhausted","message":"You've used all the included free usage. Usage resets over a rolling 24-hour window."}}`)
|
||||
|
||||
svc.handleGrokAccountUpstreamError(context.Background(), account, http.StatusBadRequest, nil, body)
|
||||
|
||||
require.Equal(t, 1, repo.tempUnschedCalls)
|
||||
require.Equal(t, "grok free usage exhausted", repo.lastTempUnschedReason)
|
||||
// 24h rolling → 2h cool
|
||||
require.Greater(t, repo.lastTempUnschedUntil, before.Add(119*time.Minute))
|
||||
require.Less(t, repo.lastTempUnschedUntil, before.Add(121*time.Minute))
|
||||
}
|
||||
|
||||
func TestHandleGrokAccountUpstreamError_EmptyOutputCoolsAccount(t *testing.T) {
|
||||
repo := &grokQuotaAccountRepo{}
|
||||
svc := &OpenAIGatewayService{accountRepo: repo}
|
||||
account := &Account{ID: 9102, Platform: PlatformGrok, Type: AccountTypeOAuth}
|
||||
before := time.Now()
|
||||
|
||||
svc.handleGrokAccountUpstreamError(
|
||||
context.Background(), account, http.StatusBadGateway, nil,
|
||||
[]byte(`empty model output: no content/tool_calls`),
|
||||
)
|
||||
|
||||
require.Equal(t, 1, repo.tempUnschedCalls)
|
||||
require.Equal(t, "grok empty model output", repo.lastTempUnschedReason)
|
||||
require.WithinDuration(t, before.Add(4*time.Minute), repo.lastTempUnschedUntil, time.Second)
|
||||
}
|
||||
|
||||
func TestHandleGrokAccountUpstreamError_FreeUsageDoesNotCoolPoolMode(t *testing.T) {
|
||||
repo := &grokQuotaAccountRepo{}
|
||||
svc := &OpenAIGatewayService{accountRepo: repo}
|
||||
account := &Account{
|
||||
ID: 9103,
|
||||
Platform: PlatformGrok,
|
||||
Type: AccountTypeAPIKey,
|
||||
Credentials: map[string]any{
|
||||
"pool_mode": true,
|
||||
},
|
||||
}
|
||||
body := []byte(`{"error":{"code":"subscription:free-usage-exhausted","message":"free usage exhausted"}}`)
|
||||
|
||||
svc.handleGrokAccountUpstreamError(context.Background(), account, http.StatusBadRequest, nil, body)
|
||||
|
||||
require.Zero(t, repo.tempUnschedCalls)
|
||||
require.False(t, svc.isOpenAIAccountRuntimeBlocked(account))
|
||||
}
|
||||
|
||||
func TestHandleGrokAccountUpstreamError_ContentPolicyStillNoMutation(t *testing.T) {
|
||||
repo := &grokQuotaAccountRepo{}
|
||||
svc := &OpenAIGatewayService{accountRepo: repo}
|
||||
account := &Account{ID: 9104, Platform: PlatformGrok, Type: AccountTypeOAuth}
|
||||
body := []byte(`{"error":{"code":"new_sensitive","message":"text is sensitive"}}`)
|
||||
|
||||
svc.handleGrokAccountUpstreamError(context.Background(), account, http.StatusForbidden, nil, body)
|
||||
|
||||
require.Zero(t, repo.tempUnschedCalls)
|
||||
}
|
||||
|
||||
func TestHandleGrokAccountUpstreamError_Entitlement403Unchanged(t *testing.T) {
|
||||
repo := &grokQuotaAccountRepo{}
|
||||
svc := &OpenAIGatewayService{accountRepo: repo}
|
||||
account := &Account{ID: 9105, Platform: PlatformGrok, Type: AccountTypeOAuth}
|
||||
before := time.Now()
|
||||
|
||||
svc.handleGrokAccountUpstreamError(
|
||||
context.Background(), account, http.StatusForbidden, nil,
|
||||
[]byte(`{"error":{"message":"subscription required"}}`),
|
||||
)
|
||||
|
||||
require.Equal(t, 1, repo.tempUnschedCalls)
|
||||
require.Equal(t, "grok access or entitlement denied", repo.lastTempUnschedReason)
|
||||
require.Greater(t, repo.lastTempUnschedUntil, before.Add(29*time.Minute))
|
||||
require.Less(t, repo.lastTempUnschedUntil, before.Add(31*time.Minute))
|
||||
}
|
||||
@@ -1356,6 +1356,19 @@ func (s *OpenAIGatewayService) handleGrokAccountUpstreamError(ctx context.Contex
|
||||
}
|
||||
now := time.Now()
|
||||
s.updateGrokUsageSnapshot(ctx, account, parseGrokQuotaSnapshot(headers, statusCode, now))
|
||||
|
||||
// Body-first free-usage / empty / billing / capacity must run before the
|
||||
// status switch so non-429 free-usage bodies still cool the account.
|
||||
// Pool-mode still skips durable mutation unless an explicit temp rule matches.
|
||||
decision := classifyGrokUpstreamFailure(statusCode, responseBody, "")
|
||||
if decision.ShouldCooldown && decision.Class != GrokFailureNone && decision.Class != GrokFailureRateLimit {
|
||||
if account.IsPoolMode() {
|
||||
// Allow configured temp rules (403) below; skip default body cools.
|
||||
} else if s.applyGrokUpstreamFailureDecision(ctx, account, decision) {
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
if statusCode == http.StatusForbidden && s.applyGrokForbiddenPolicy(ctx, account, responseBody) {
|
||||
return
|
||||
}
|
||||
@@ -1367,11 +1380,10 @@ func (s *OpenAIGatewayService) handleGrokAccountUpstreamError(ctx context.Contex
|
||||
case http.StatusUnauthorized:
|
||||
s.tempUnscheduleGrok(ctx, account, 10*time.Minute, "grok credentials unauthorized")
|
||||
case http.StatusPaymentRequired:
|
||||
// 402: temporarily unschedulable with a clear payment-required reason.
|
||||
// 402 without a body-classified billing decision: keep the legacy 30m cool.
|
||||
s.tempUnscheduleGrok(ctx, account, 30*time.Minute, "grok payment required")
|
||||
case http.StatusForbidden:
|
||||
// Spending-limit 403 (personal-team-blocked:spending-limit) is billing exhaustion,
|
||||
// not a generic entitlement denial — still temp-unschedule with a distinct reason.
|
||||
// Spending-limit already handled by body classifier when phrasing matches.
|
||||
if isGrokSpendingLimitError(responseBody) {
|
||||
s.tempUnscheduleGrok(ctx, account, 30*time.Minute, "grok spending limit")
|
||||
return
|
||||
@@ -1379,6 +1391,7 @@ func (s *OpenAIGatewayService) handleGrokAccountUpstreamError(ctx context.Contex
|
||||
s.tempUnscheduleGrok(ctx, account, 30*time.Minute, "grok access or entitlement denied")
|
||||
case http.StatusTooManyRequests:
|
||||
// updateGrokUsageSnapshot installs rate-limit state for non-pool accounts.
|
||||
// Free-usage 429 was already cooled above via body classification.
|
||||
default:
|
||||
if statusCode >= 500 {
|
||||
s.tempUnscheduleGrok(ctx, account, 2*time.Minute, "grok upstream temporary error")
|
||||
|
||||
@@ -118,6 +118,12 @@ func explicitGrokCacheSeed(c *gin.Context, body []byte, explicitKey string) stri
|
||||
if seed == "" {
|
||||
seed = strings.TrimSpace(explicitKey)
|
||||
}
|
||||
// previous_response_id is last-resort: multi-turn Responses without an
|
||||
// explicit session still share one cache identity (model is already in the
|
||||
// isolated seed). Message ids are rejected by the seed helper.
|
||||
if seed == "" && len(body) > 0 {
|
||||
seed = grokPreviousResponseSessionSeed(body)
|
||||
}
|
||||
return seed
|
||||
}
|
||||
|
||||
|
||||
@@ -24,6 +24,36 @@ func newGrokCacheTestContext(apiKeyID int64) *gin.Context {
|
||||
return c
|
||||
}
|
||||
|
||||
func TestGrokPreviousResponseSessionSeed(t *testing.T) {
|
||||
require.Equal(t, "grok-prev-resp:resp_abc123", grokPreviousResponseSessionSeed([]byte(`{"previous_response_id":"resp_abc123"}`)))
|
||||
require.Empty(t, grokPreviousResponseSessionSeed([]byte(`{"previous_response_id":"msg_abc123"}`)))
|
||||
require.Empty(t, grokPreviousResponseSessionSeed([]byte(`{"previous_response_id":""}`)))
|
||||
require.Empty(t, grokPreviousResponseSessionSeed([]byte(`{}`)))
|
||||
}
|
||||
|
||||
func TestResolveGrokCacheIdentityUsesPreviousResponseIDWhenNoOtherSeed(t *testing.T) {
|
||||
gin.SetMode(gin.TestMode)
|
||||
c := newGrokCacheTestContext(301)
|
||||
// No prompt_cache_key / headers / reusable prefix — only previous_response_id.
|
||||
body := []byte(`{"model":"grok","input":[{"role":"user","content":"follow up"}],"previous_response_id":"resp_chain_001"}`)
|
||||
got := resolveGrokCacheIdentity(c, body, "", "grok-4.5")
|
||||
require.NotEmpty(t, got)
|
||||
|
||||
// Same previous_response_id → same identity (model already in isolated seed).
|
||||
again := resolveGrokCacheIdentity(c, body, "", "grok-4.5")
|
||||
require.Equal(t, got, again)
|
||||
|
||||
// Different model → different identity (model scope).
|
||||
otherModel := resolveGrokCacheIdentity(c, body, "", "grok-4.3")
|
||||
require.NotEqual(t, got, otherModel)
|
||||
|
||||
// prompt_cache_key still wins over previous_response_id.
|
||||
withCache := []byte(`{"model":"grok","prompt_cache_key":"client-session","previous_response_id":"resp_chain_001","input":[{"role":"user","content":"x"}]}`)
|
||||
cacheID := resolveGrokCacheIdentity(c, withCache, "", "grok-4.5")
|
||||
require.NotEmpty(t, cacheID)
|
||||
require.NotEqual(t, got, cacheID)
|
||||
}
|
||||
|
||||
func TestResolveGrokCacheIdentityStableAcrossAppendOnlyTurns(t *testing.T) {
|
||||
gin.SetMode(gin.TestMode)
|
||||
c := newGrokCacheTestContext(101)
|
||||
|
||||
@@ -75,6 +75,12 @@ func explicitOpenAISessionID(c *gin.Context, body []byte) string {
|
||||
// with Grok's native conversation header only for requests authenticated to a
|
||||
// Grok group. This keeps an unrelated x-grok-conv-id header from changing
|
||||
// scheduling or upstream session behavior for non-Grok groups.
|
||||
//
|
||||
// For Grok groups only, previous_response_id is a last-resort sticky seed so
|
||||
// multi-turn Responses chains stay on the same OAuth account when no explicit
|
||||
// session/conversation/prompt_cache_key is present. Non-Grok groups omit this
|
||||
// so HTTP OpenAI paths that delete previous_response_id before upstream are
|
||||
// unchanged.
|
||||
func explicitOpenAIRequestSessionID(c *gin.Context, body []byte) string {
|
||||
if c == nil {
|
||||
return ""
|
||||
@@ -87,9 +93,27 @@ func explicitOpenAIRequestSessionID(c *gin.Context, body []byte) string {
|
||||
if sessionID == "" && len(body) > 0 {
|
||||
sessionID = strings.TrimSpace(gjson.GetBytes(body, "prompt_cache_key").String())
|
||||
}
|
||||
if sessionID == "" && isGrokRequestContext(c) && len(body) > 0 {
|
||||
sessionID = grokPreviousResponseSessionSeed(body)
|
||||
}
|
||||
return sessionID
|
||||
}
|
||||
|
||||
// grokPreviousResponseSessionSeed returns a stable sticky seed from a Responses
|
||||
// previous_response_id. Only resp_* response ids are accepted; message ids and
|
||||
// unknown shapes must not pin sticky routing or prompt-cache identity.
|
||||
func grokPreviousResponseSessionSeed(body []byte) string {
|
||||
id := strings.TrimSpace(gjson.GetBytes(body, "previous_response_id").String())
|
||||
if id == "" {
|
||||
return ""
|
||||
}
|
||||
if ClassifyOpenAIPreviousResponseIDKind(id) != OpenAIPreviousResponseIDKindResponseID {
|
||||
return ""
|
||||
}
|
||||
// Namespace so content-derived seeds never collide with response ids.
|
||||
return "grok-prev-resp:" + id
|
||||
}
|
||||
|
||||
// GenerateExplicitSessionHash generates a sticky-session hash only from explicit
|
||||
// client session signals. It intentionally skips content-derived fallback and is
|
||||
// used by stateless endpoints such as /v1/images.
|
||||
|
||||
@@ -924,6 +924,11 @@ interface Props {
|
||||
showEmailPasswordOption?: boolean
|
||||
showManualOption?: boolean
|
||||
initialInputMethod?: AuthInputMethod
|
||||
/**
|
||||
* Prefill for Grok email----password reauth. Password is never stored;
|
||||
* pass only the email (or "email----") so the operator types the password.
|
||||
*/
|
||||
initialEmailPassword?: string
|
||||
platform?: AccountPlatform // Platform type for different UI/text
|
||||
showProjectId?: boolean // New prop to control project ID visibility
|
||||
}
|
||||
@@ -949,6 +954,7 @@ const props = withDefaults(defineProps<Props>(), {
|
||||
showEmailPasswordOption: false,
|
||||
showManualOption: true,
|
||||
initialInputMethod: 'manual',
|
||||
initialEmailPassword: '',
|
||||
platform: 'anthropic',
|
||||
showProjectId: true
|
||||
})
|
||||
@@ -1010,7 +1016,7 @@ const sessionTokenInput = ref('')
|
||||
const codexSessionInput = ref('')
|
||||
const codexPATInput = ref('')
|
||||
const ssoCookieInput = ref('')
|
||||
const emailPasswordInput = ref('')
|
||||
const emailPasswordInput = ref(props.initialEmailPassword || '')
|
||||
const showHelpDialog = ref(false)
|
||||
const oauthState = ref('')
|
||||
const projectId = ref('')
|
||||
@@ -1085,6 +1091,16 @@ watch(() => props.initialInputMethod, (newVal) => {
|
||||
inputMethod.value = newVal
|
||||
})
|
||||
|
||||
watch(
|
||||
() => props.initialEmailPassword,
|
||||
(newVal) => {
|
||||
// Only prefill when the field is empty so we never overwrite operator input.
|
||||
if (newVal && !emailPasswordInput.value.trim()) {
|
||||
emailPasswordInput.value = newVal
|
||||
}
|
||||
}
|
||||
)
|
||||
|
||||
watch(inputMethod, (newVal) => {
|
||||
emit('update:inputMethod', newVal)
|
||||
})
|
||||
|
||||
@@ -137,6 +137,8 @@
|
||||
:method-label="t('admin.accounts.inputMethod')"
|
||||
:platform="isOpenAI ? 'openai' : isGemini ? 'gemini' : isAntigravity ? 'antigravity' : isGrok ? 'grok' : 'anthropic'"
|
||||
:show-project-id="isGemini && geminiOAuthType === 'code_assist'"
|
||||
:initial-input-method="grokInitialInputMethod"
|
||||
:initial-email-password="grokPrefillEmailPassword"
|
||||
@generate-url="handleGenerateUrl"
|
||||
@cookie-auth="handleCookieAuth"
|
||||
@validate-refresh-token="handleGrokValidateRefreshToken"
|
||||
@@ -255,6 +257,40 @@ const isAnthropic = computed(() => props.account?.platform === 'anthropic')
|
||||
const isAntigravity = computed(() => props.account?.platform === 'antigravity')
|
||||
const isGrok = computed(() => props.account?.platform === 'grok')
|
||||
|
||||
/** Stored Grok email for reauth prefill (password is never stored). */
|
||||
const grokAccountEmail = computed(() => {
|
||||
if (!isGrok.value || !props.account) return ''
|
||||
const creds = (props.account.credentials || {}) as Record<string, unknown>
|
||||
const email = typeof creds.email === 'string' ? creds.email.trim() : ''
|
||||
return email
|
||||
})
|
||||
|
||||
/**
|
||||
* Prefill "email----" so the operator only types the password.
|
||||
* Empty when no email is known (full email----password required).
|
||||
*/
|
||||
const grokPrefillEmailPassword = computed(() => {
|
||||
const email = grokAccountEmail.value
|
||||
return email ? `${email}----` : ''
|
||||
})
|
||||
|
||||
/**
|
||||
* Grok reauth default tab:
|
||||
* - password first when we know the email (common reauth path)
|
||||
* - refresh_token when email unknown but RT may still work
|
||||
* - email_password otherwise
|
||||
*/
|
||||
const grokInitialInputMethod = computed<AuthInputMethod>(() => {
|
||||
if (!isGrok.value) return 'manual'
|
||||
if (grokAccountEmail.value) return 'email_password'
|
||||
const creds = (props.account?.credentials || {}) as Record<string, unknown>
|
||||
const hasRT =
|
||||
(typeof creds.refresh_token === 'string' && creds.refresh_token.trim() !== '') ||
|
||||
(typeof creds.has_refresh_token === 'boolean' && creds.has_refresh_token)
|
||||
if (hasRT) return 'refresh_token'
|
||||
return 'email_password'
|
||||
})
|
||||
|
||||
// Computed - current OAuth state based on platform
|
||||
const currentAuthUrl = computed(() => {
|
||||
if (isOpenAILike.value) return openaiOAuth.authUrl.value
|
||||
|
||||
@@ -32,4 +32,13 @@ describe('ReAuthAccountModal Grok re-auth paths', () => {
|
||||
expect(source).toContain("method === 'email_password'")
|
||||
expect(source).toContain("method === 'refresh_token'")
|
||||
})
|
||||
|
||||
it('prefills email---- and defaults to password method when email is known', () => {
|
||||
expect(source).toContain('grokPrefillEmailPassword')
|
||||
expect(source).toContain('grokInitialInputMethod')
|
||||
expect(source).toContain(':initial-email-password="grokPrefillEmailPassword"')
|
||||
expect(source).toContain(':initial-input-method="grokInitialInputMethod"')
|
||||
expect(source).toContain('email----')
|
||||
expect(source).toContain("return 'email_password'")
|
||||
})
|
||||
})
|
||||
|
||||
Reference in New Issue
Block a user