mirror of
https://github.com/Wei-Shaw/sub2api.git
synced 2026-10-07 12:57:57 +08:00
Merge pull request #5876 from wucm667/fix/issue-5872-exclude-model-not-found-sla
fix(ops): exclude model configuration errors from SLA
This commit is contained in:
@@ -106,7 +106,11 @@ func classifyNoAccountErrorFromGin(
|
||||
if c != nil && c.Request != nil {
|
||||
ctx = c.Request.Context()
|
||||
}
|
||||
return classifyNoAccountError(ctx, diag, apiKey, routingModel, displayModel, platform)
|
||||
classification := classifyNoAccountError(ctx, diag, apiKey, routingModel, displayModel, platform)
|
||||
if classification.ModelNotFound {
|
||||
service.MarkOpsClientBusinessLimited(c, service.OpsClientBusinessLimitedReasonLocalModelConfiguration)
|
||||
}
|
||||
return classification
|
||||
}
|
||||
|
||||
func classifyOpenAICompatibleNoAccountErrorFromGin(
|
||||
|
||||
@@ -113,6 +113,8 @@ func TestClassifyNoAccountError_ModelNotSupported_Returns404(t *testing.T) {
|
||||
require.Equal(t, service.PlatformOpenAI, fd.calls[0].Platform)
|
||||
require.NotNil(t, fd.calls[0].GroupID)
|
||||
require.Equal(t, int64(42), *fd.calls[0].GroupID)
|
||||
require.True(t, service.HasOpsClientBusinessLimited(c))
|
||||
require.Equal(t, service.OpsClientBusinessLimitedReasonLocalModelConfiguration, service.OpsClientBusinessLimitedReason(c))
|
||||
}
|
||||
|
||||
func TestClassifyOpenAICompatibleNoAccountError_GrokUsesGrokPlatform(t *testing.T) {
|
||||
@@ -134,6 +136,8 @@ func TestClassifyOpenAICompatibleNoAccountError_GrokUsesGrokPlatform(t *testing.
|
||||
require.True(t, cls.ModelNotFound)
|
||||
require.Len(t, fd.calls, 1)
|
||||
require.Equal(t, service.PlatformGrok, fd.calls[0].Platform)
|
||||
require.True(t, service.HasOpsClientBusinessLimited(c))
|
||||
require.Equal(t, service.OpsClientBusinessLimitedReasonLocalModelConfiguration, service.OpsClientBusinessLimitedReason(c))
|
||||
|
||||
logErr := openAICompatibleSelectionErrorForLog(
|
||||
fmt.Errorf("no available OpenAI accounts supporting model: grok-4.5"),
|
||||
@@ -142,6 +146,18 @@ func TestClassifyOpenAICompatibleNoAccountError_GrokUsesGrokPlatform(t *testing.
|
||||
require.EqualError(t, logErr, "no available Grok accounts supporting model: grok-4.5")
|
||||
}
|
||||
|
||||
func TestClassifyNoAccountError_PureClassifierDoesNotMarkGinContext(t *testing.T) {
|
||||
c := newTestGinContextWithRequest()
|
||||
fd := &fakeDiagnoser{resp: service.ModelAvailabilityDiagnosis{HasAccountsInPool: true, HasModelSupport: false}}
|
||||
apiKey := &service.APIKey{GroupID: ptrInt64(7)}
|
||||
|
||||
cls := classifyNoAccountError(c.Request.Context(), fd, apiKey, "gpt-5", "gpt-5", service.PlatformOpenAI)
|
||||
|
||||
require.True(t, cls.ModelNotFound)
|
||||
require.False(t, service.HasOpsClientBusinessLimited(c))
|
||||
require.Empty(t, service.OpsClientBusinessLimitedReason(c))
|
||||
}
|
||||
|
||||
func TestClassifyNoAccountError_HasModelSupport_KeepsRoutingMessageGenerationToCaller(t *testing.T) {
|
||||
c := newTestGinContextWithRequest()
|
||||
fd := &fakeDiagnoser{resp: service.ModelAvailabilityDiagnosis{HasAccountsInPool: true, HasModelSupport: true}}
|
||||
@@ -200,4 +216,6 @@ func TestClassifyNoAccountError_FromGin_NilContextStillSafe(t *testing.T) {
|
||||
|
||||
require.Equal(t, http.StatusNotFound, cls.Status, "even with a nil gin context the classifier must still run and yield a coherent response")
|
||||
require.True(t, cls.ModelNotFound)
|
||||
require.False(t, service.HasOpsClientBusinessLimited(nil))
|
||||
require.Empty(t, service.OpsClientBusinessLimitedReason(nil))
|
||||
}
|
||||
|
||||
@@ -1073,6 +1073,7 @@ func OpsErrorLoggerMiddleware(ops *service.OpsService) gin.HandlerFunc {
|
||||
}
|
||||
applyOpsLatencyFieldsFromContext(c, entry)
|
||||
applyOpsUpstreamFieldsFromContext(c, entry)
|
||||
suppressOpsUpstreamAttributionForLocalModelConfiguration(c, entry)
|
||||
|
||||
if apiKey != nil {
|
||||
entry.APIKeyID = &apiKey.ID
|
||||
@@ -1338,6 +1339,19 @@ func applyOpsUpstreamFieldsFromContext(c *gin.Context, entry *service.OpsInsertE
|
||||
}
|
||||
}
|
||||
|
||||
func suppressOpsUpstreamAttributionForLocalModelConfiguration(c *gin.Context, entry *service.OpsInsertErrorLogInput) {
|
||||
if entry == nil || !service.HasOpsClientBusinessLimited(c) || service.OpsClientBusinessLimitedReason(c) != service.OpsClientBusinessLimitedReasonLocalModelConfiguration {
|
||||
return
|
||||
}
|
||||
entry.AccountID = nil
|
||||
entry.UpstreamEndpoint = ""
|
||||
entry.UpstreamModel = ""
|
||||
entry.UpstreamStatusCode = nil
|
||||
entry.UpstreamErrorMessage = nil
|
||||
entry.UpstreamErrorDetail = nil
|
||||
entry.UpstreamErrors = nil
|
||||
}
|
||||
|
||||
func getContextLatencyMs(c *gin.Context, key string) *int64 {
|
||||
if c == nil || strings.TrimSpace(key) == "" {
|
||||
return nil
|
||||
@@ -1550,23 +1564,27 @@ func classifyOpsErrorLog(c *gin.Context, errType, message, code string, status i
|
||||
phase = classifyOpsPhase(errType, message, code)
|
||||
routingCapacityLimited := isOpsRoutingCapacityLimited(c)
|
||||
clientBusinessLimited := service.HasOpsClientBusinessLimited(c)
|
||||
localModelConfiguration := clientBusinessLimited && service.OpsClientBusinessLimitedReason(c) == service.OpsClientBusinessLimitedReasonLocalModelConfiguration
|
||||
upstreamError := hasOpsUpstreamErrorContext(c)
|
||||
accountAuthFailure := hasOpsAccountAuthFailure(c)
|
||||
if accountAuthFailure && !routingCapacityLimited {
|
||||
if localModelConfiguration {
|
||||
phase = "routing"
|
||||
} else if accountAuthFailure && !routingCapacityLimited {
|
||||
phase = "account_auth"
|
||||
} else if upstreamError && !routingCapacityLimited {
|
||||
phase = "upstream"
|
||||
}
|
||||
if clientBusinessLimited && !upstreamError && !routingCapacityLimited {
|
||||
if clientBusinessLimited && !upstreamError && !routingCapacityLimited && !localModelConfiguration {
|
||||
phase = "auth"
|
||||
}
|
||||
if routingCapacityLimited {
|
||||
phase = "routing"
|
||||
}
|
||||
msg := strings.ToLower(message)
|
||||
localClientAuthError := !upstreamError && phase == "auth" && isOpsClientAuthError(code, msg)
|
||||
localBusinessLimited := !upstreamError && classifyOpsIsBusinessLimited(errType, phase, code, status, message, localClientAuthError)
|
||||
isBusinessLimited = routingCapacityLimited || (clientBusinessLimited && !upstreamError) || localBusinessLimited
|
||||
effectiveUpstreamError := upstreamError && !localModelConfiguration
|
||||
localClientAuthError := !effectiveUpstreamError && phase == "auth" && isOpsClientAuthError(code, msg)
|
||||
localBusinessLimited := !effectiveUpstreamError && classifyOpsIsBusinessLimited(errType, phase, code, status, message, localClientAuthError)
|
||||
isBusinessLimited = localModelConfiguration || routingCapacityLimited || (clientBusinessLimited && !effectiveUpstreamError) || localBusinessLimited
|
||||
errorOwner = classifyOpsErrorOwner(phase, message)
|
||||
errorSource = classifyOpsErrorSource(phase, message)
|
||||
return phase, isBusinessLimited, errorOwner, errorSource
|
||||
|
||||
@@ -497,6 +497,107 @@ func TestClassifyOpsRoutingCapacityMarkerExcludesMaskedSelectionFailureFromSLA(t
|
||||
require.Equal(t, "gateway", errorSource)
|
||||
}
|
||||
|
||||
func TestClassifyOpsLocalModelConfigurationRejection(t *testing.T) {
|
||||
gin.SetMode(gin.TestMode)
|
||||
rec := httptest.NewRecorder()
|
||||
c, _ := gin.CreateTestContext(rec)
|
||||
service.MarkOpsClientBusinessLimited(c, service.OpsClientBusinessLimitedReasonLocalModelConfiguration)
|
||||
|
||||
phase, isBusinessLimited, errorOwner, errorSource := classifyOpsErrorLog(
|
||||
c,
|
||||
"model_not_found",
|
||||
"Model \"gpt-missing\" is not supported by any configured account in this group",
|
||||
"",
|
||||
http.StatusNotFound,
|
||||
)
|
||||
|
||||
require.Equal(t, "routing", phase)
|
||||
require.True(t, isBusinessLimited)
|
||||
require.Equal(t, "platform", errorOwner)
|
||||
require.Equal(t, "gateway", errorSource)
|
||||
}
|
||||
|
||||
func TestClassifyOpsLocalModelConfigurationOverridesStaleUpstreamMarkers(t *testing.T) {
|
||||
gin.SetMode(gin.TestMode)
|
||||
c, _ := gin.CreateTestContext(httptest.NewRecorder())
|
||||
service.MarkOpsClientBusinessLimited(c, service.OpsClientBusinessLimitedReasonLocalModelConfiguration)
|
||||
c.Set(service.OpsUpstreamStatusCodeKey, http.StatusUnauthorized)
|
||||
c.Set(service.OpsUpstreamErrorsKey, []*service.OpsUpstreamErrorEvent{{
|
||||
Stage: string(service.GatewayFailureStageAccountAuth),
|
||||
UpstreamStatusCode: http.StatusUnauthorized,
|
||||
}})
|
||||
|
||||
phase, limited, owner, source := classifyOpsErrorLog(c, "model_not_found", "unsupported configured model", "", http.StatusNotFound)
|
||||
|
||||
require.Equal(t, "routing", phase)
|
||||
require.True(t, limited)
|
||||
require.Equal(t, "platform", owner)
|
||||
require.Equal(t, "gateway", source)
|
||||
}
|
||||
|
||||
func TestClassifyOpsLocalModelConfigurationRequiresMarkerAndReason(t *testing.T) {
|
||||
gin.SetMode(gin.TestMode)
|
||||
c, _ := gin.CreateTestContext(httptest.NewRecorder())
|
||||
c.Set(service.OpsClientBusinessLimitedReasonKey, service.OpsClientBusinessLimitedReasonLocalModelConfiguration)
|
||||
c.Set(service.OpsUpstreamStatusCodeKey, http.StatusBadGateway)
|
||||
|
||||
phase, limited, owner, source := classifyOpsErrorLog(c, "upstream_error", "provider failed", "", http.StatusBadGateway)
|
||||
|
||||
require.Equal(t, "upstream", phase)
|
||||
require.False(t, limited)
|
||||
require.Equal(t, "provider", owner)
|
||||
require.Equal(t, "upstream_http", source)
|
||||
}
|
||||
|
||||
func TestOpsErrorLoggerMiddleware_LocalModelConfigurationFields(t *testing.T) {
|
||||
setupOpsErrorLogTestQueue(t, 1)
|
||||
gin.SetMode(gin.TestMode)
|
||||
|
||||
ops := service.NewOpsService(nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil)
|
||||
router := gin.New()
|
||||
router.Use(OpsErrorLoggerMiddleware(ops))
|
||||
router.POST("/v1/chat/completions", func(c *gin.Context) {
|
||||
service.MarkOpsClientBusinessLimited(c, service.OpsClientBusinessLimitedReasonLocalModelConfiguration)
|
||||
c.Set(opsAccountIDKey, int64(99))
|
||||
c.Set(opsUpstreamModelKey, "stale-upstream-model")
|
||||
setActualUpstreamEndpoint(c, "/v1/chat/completions")
|
||||
c.Set(service.OpsUpstreamStatusCodeKey, http.StatusUnauthorized)
|
||||
c.Set(service.OpsUpstreamErrorMessageKey, "stale upstream error")
|
||||
c.Set(service.OpsUpstreamErrorDetailKey, "stale upstream detail")
|
||||
c.Set(service.OpsUpstreamErrorsKey, []*service.OpsUpstreamErrorEvent{{
|
||||
Stage: string(service.GatewayFailureStageAccountAuth),
|
||||
UpstreamStatusCode: http.StatusUnauthorized,
|
||||
Message: "stale auth failure",
|
||||
}})
|
||||
c.JSON(http.StatusNotFound, gin.H{
|
||||
"error": gin.H{
|
||||
"type": "model_not_found",
|
||||
"message": "Model \"gpt-missing\" is not supported by any configured account in this group",
|
||||
},
|
||||
})
|
||||
})
|
||||
|
||||
w := httptest.NewRecorder()
|
||||
req := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", nil)
|
||||
router.ServeHTTP(w, req)
|
||||
|
||||
require.Equal(t, http.StatusNotFound, w.Code)
|
||||
require.JSONEq(t, `{"error":{"type":"model_not_found","message":"Model \"gpt-missing\" is not supported by any configured account in this group"}}`, w.Body.String())
|
||||
job := <-opsErrorLogQueue
|
||||
require.Equal(t, http.StatusNotFound, job.entry.StatusCode)
|
||||
require.Equal(t, "routing", job.entry.ErrorPhase)
|
||||
require.True(t, job.entry.IsBusinessLimited)
|
||||
require.Equal(t, "platform", job.entry.ErrorOwner)
|
||||
require.Equal(t, "gateway", job.entry.ErrorSource)
|
||||
require.Nil(t, job.entry.AccountID)
|
||||
require.Nil(t, job.entry.UpstreamStatusCode)
|
||||
require.Nil(t, job.entry.UpstreamErrors)
|
||||
require.Nil(t, job.entry.UpstreamErrorMessage)
|
||||
require.Nil(t, job.entry.UpstreamErrorDetail)
|
||||
require.Empty(t, job.entry.UpstreamModel)
|
||||
require.Empty(t, job.entry.UpstreamEndpoint)
|
||||
}
|
||||
|
||||
func TestClassifyOpsAuthClientErrorsExcludedFromSLA(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
|
||||
@@ -44,13 +44,14 @@ const (
|
||||
// ensureForwardErrorResponse 检查此 key,为 true 时跳过兜底写入,避免在已完成的 JSON 后追加 SSE。
|
||||
ResponseCommittedKey = "response_committed"
|
||||
|
||||
OpsClientBusinessLimitedKey = "ops_client_business_limited"
|
||||
OpsClientBusinessLimitedReasonKey = "ops_client_business_limited_reason"
|
||||
OpsClientBusinessLimitedReasonIPRestriction = "api_key_ip_restriction"
|
||||
OpsClientBusinessLimitedReasonAPIKeyGroupUnavailable = "api_key_group_unavailable"
|
||||
OpsClientBusinessLimitedReasonAPIKeyGroupUnassigned = "api_key_group_unassigned"
|
||||
OpsClientBusinessLimitedReasonLocalFeatureGate = "local_feature_gate"
|
||||
OpsClientBusinessLimitedReasonLocalPolicyDenied = "local_policy_denied"
|
||||
OpsClientBusinessLimitedKey = "ops_client_business_limited"
|
||||
OpsClientBusinessLimitedReasonKey = "ops_client_business_limited_reason"
|
||||
OpsClientBusinessLimitedReasonIPRestriction = "api_key_ip_restriction"
|
||||
OpsClientBusinessLimitedReasonAPIKeyGroupUnavailable = "api_key_group_unavailable"
|
||||
OpsClientBusinessLimitedReasonAPIKeyGroupUnassigned = "api_key_group_unassigned"
|
||||
OpsClientBusinessLimitedReasonLocalFeatureGate = "local_feature_gate"
|
||||
OpsClientBusinessLimitedReasonLocalPolicyDenied = "local_policy_denied"
|
||||
OpsClientBusinessLimitedReasonLocalModelConfiguration = "local_model_configuration"
|
||||
)
|
||||
|
||||
func MarkResponseCommitted(c *gin.Context) { c.Set(ResponseCommittedKey, true) }
|
||||
@@ -93,6 +94,18 @@ func HasOpsClientBusinessLimited(c *gin.Context) bool {
|
||||
return marked
|
||||
}
|
||||
|
||||
func OpsClientBusinessLimitedReason(c *gin.Context) string {
|
||||
if c == nil {
|
||||
return ""
|
||||
}
|
||||
v, ok := c.Get(OpsClientBusinessLimitedReasonKey)
|
||||
if !ok {
|
||||
return ""
|
||||
}
|
||||
reason, _ := v.(string)
|
||||
return strings.TrimSpace(reason)
|
||||
}
|
||||
|
||||
// OpsStreamError 描述网关在「响应状态已固化为 200」之后(keepalive ping 或部分数据
|
||||
// 已 flush)就地以 SSE error 帧形式返回的错误。由于 HTTP 状态码停留在 200,
|
||||
// 而 ops_error_logger 以 status>=400 为采集触发条件,这类流内失败
|
||||
|
||||
Reference in New Issue
Block a user