Merge pull request #5876 from wucm667/fix/issue-5872-exclude-model-not-found-sla

fix(ops): exclude model configuration errors from SLA
This commit is contained in:
Wesley Liddick
2026-08-20 14:45:08 +08:00
committed by GitHub
5 changed files with 167 additions and 13 deletions
+5 -1
View File
@@ -106,7 +106,11 @@ func classifyNoAccountErrorFromGin(
if c != nil && c.Request != nil {
ctx = c.Request.Context()
}
return classifyNoAccountError(ctx, diag, apiKey, routingModel, displayModel, platform)
classification := classifyNoAccountError(ctx, diag, apiKey, routingModel, displayModel, platform)
if classification.ModelNotFound {
service.MarkOpsClientBusinessLimited(c, service.OpsClientBusinessLimitedReasonLocalModelConfiguration)
}
return classification
}
func classifyOpenAICompatibleNoAccountErrorFromGin(
@@ -113,6 +113,8 @@ func TestClassifyNoAccountError_ModelNotSupported_Returns404(t *testing.T) {
require.Equal(t, service.PlatformOpenAI, fd.calls[0].Platform)
require.NotNil(t, fd.calls[0].GroupID)
require.Equal(t, int64(42), *fd.calls[0].GroupID)
require.True(t, service.HasOpsClientBusinessLimited(c))
require.Equal(t, service.OpsClientBusinessLimitedReasonLocalModelConfiguration, service.OpsClientBusinessLimitedReason(c))
}
func TestClassifyOpenAICompatibleNoAccountError_GrokUsesGrokPlatform(t *testing.T) {
@@ -134,6 +136,8 @@ func TestClassifyOpenAICompatibleNoAccountError_GrokUsesGrokPlatform(t *testing.
require.True(t, cls.ModelNotFound)
require.Len(t, fd.calls, 1)
require.Equal(t, service.PlatformGrok, fd.calls[0].Platform)
require.True(t, service.HasOpsClientBusinessLimited(c))
require.Equal(t, service.OpsClientBusinessLimitedReasonLocalModelConfiguration, service.OpsClientBusinessLimitedReason(c))
logErr := openAICompatibleSelectionErrorForLog(
fmt.Errorf("no available OpenAI accounts supporting model: grok-4.5"),
@@ -142,6 +146,18 @@ func TestClassifyOpenAICompatibleNoAccountError_GrokUsesGrokPlatform(t *testing.
require.EqualError(t, logErr, "no available Grok accounts supporting model: grok-4.5")
}
func TestClassifyNoAccountError_PureClassifierDoesNotMarkGinContext(t *testing.T) {
c := newTestGinContextWithRequest()
fd := &fakeDiagnoser{resp: service.ModelAvailabilityDiagnosis{HasAccountsInPool: true, HasModelSupport: false}}
apiKey := &service.APIKey{GroupID: ptrInt64(7)}
cls := classifyNoAccountError(c.Request.Context(), fd, apiKey, "gpt-5", "gpt-5", service.PlatformOpenAI)
require.True(t, cls.ModelNotFound)
require.False(t, service.HasOpsClientBusinessLimited(c))
require.Empty(t, service.OpsClientBusinessLimitedReason(c))
}
func TestClassifyNoAccountError_HasModelSupport_KeepsRoutingMessageGenerationToCaller(t *testing.T) {
c := newTestGinContextWithRequest()
fd := &fakeDiagnoser{resp: service.ModelAvailabilityDiagnosis{HasAccountsInPool: true, HasModelSupport: true}}
@@ -200,4 +216,6 @@ func TestClassifyNoAccountError_FromGin_NilContextStillSafe(t *testing.T) {
require.Equal(t, http.StatusNotFound, cls.Status, "even with a nil gin context the classifier must still run and yield a coherent response")
require.True(t, cls.ModelNotFound)
require.False(t, service.HasOpsClientBusinessLimited(nil))
require.Empty(t, service.OpsClientBusinessLimitedReason(nil))
}
+23 -5
View File
@@ -1073,6 +1073,7 @@ func OpsErrorLoggerMiddleware(ops *service.OpsService) gin.HandlerFunc {
}
applyOpsLatencyFieldsFromContext(c, entry)
applyOpsUpstreamFieldsFromContext(c, entry)
suppressOpsUpstreamAttributionForLocalModelConfiguration(c, entry)
if apiKey != nil {
entry.APIKeyID = &apiKey.ID
@@ -1338,6 +1339,19 @@ func applyOpsUpstreamFieldsFromContext(c *gin.Context, entry *service.OpsInsertE
}
}
func suppressOpsUpstreamAttributionForLocalModelConfiguration(c *gin.Context, entry *service.OpsInsertErrorLogInput) {
if entry == nil || !service.HasOpsClientBusinessLimited(c) || service.OpsClientBusinessLimitedReason(c) != service.OpsClientBusinessLimitedReasonLocalModelConfiguration {
return
}
entry.AccountID = nil
entry.UpstreamEndpoint = ""
entry.UpstreamModel = ""
entry.UpstreamStatusCode = nil
entry.UpstreamErrorMessage = nil
entry.UpstreamErrorDetail = nil
entry.UpstreamErrors = nil
}
func getContextLatencyMs(c *gin.Context, key string) *int64 {
if c == nil || strings.TrimSpace(key) == "" {
return nil
@@ -1550,23 +1564,27 @@ func classifyOpsErrorLog(c *gin.Context, errType, message, code string, status i
phase = classifyOpsPhase(errType, message, code)
routingCapacityLimited := isOpsRoutingCapacityLimited(c)
clientBusinessLimited := service.HasOpsClientBusinessLimited(c)
localModelConfiguration := clientBusinessLimited && service.OpsClientBusinessLimitedReason(c) == service.OpsClientBusinessLimitedReasonLocalModelConfiguration
upstreamError := hasOpsUpstreamErrorContext(c)
accountAuthFailure := hasOpsAccountAuthFailure(c)
if accountAuthFailure && !routingCapacityLimited {
if localModelConfiguration {
phase = "routing"
} else if accountAuthFailure && !routingCapacityLimited {
phase = "account_auth"
} else if upstreamError && !routingCapacityLimited {
phase = "upstream"
}
if clientBusinessLimited && !upstreamError && !routingCapacityLimited {
if clientBusinessLimited && !upstreamError && !routingCapacityLimited && !localModelConfiguration {
phase = "auth"
}
if routingCapacityLimited {
phase = "routing"
}
msg := strings.ToLower(message)
localClientAuthError := !upstreamError && phase == "auth" && isOpsClientAuthError(code, msg)
localBusinessLimited := !upstreamError && classifyOpsIsBusinessLimited(errType, phase, code, status, message, localClientAuthError)
isBusinessLimited = routingCapacityLimited || (clientBusinessLimited && !upstreamError) || localBusinessLimited
effectiveUpstreamError := upstreamError && !localModelConfiguration
localClientAuthError := !effectiveUpstreamError && phase == "auth" && isOpsClientAuthError(code, msg)
localBusinessLimited := !effectiveUpstreamError && classifyOpsIsBusinessLimited(errType, phase, code, status, message, localClientAuthError)
isBusinessLimited = localModelConfiguration || routingCapacityLimited || (clientBusinessLimited && !effectiveUpstreamError) || localBusinessLimited
errorOwner = classifyOpsErrorOwner(phase, message)
errorSource = classifyOpsErrorSource(phase, message)
return phase, isBusinessLimited, errorOwner, errorSource
@@ -497,6 +497,107 @@ func TestClassifyOpsRoutingCapacityMarkerExcludesMaskedSelectionFailureFromSLA(t
require.Equal(t, "gateway", errorSource)
}
func TestClassifyOpsLocalModelConfigurationRejection(t *testing.T) {
gin.SetMode(gin.TestMode)
rec := httptest.NewRecorder()
c, _ := gin.CreateTestContext(rec)
service.MarkOpsClientBusinessLimited(c, service.OpsClientBusinessLimitedReasonLocalModelConfiguration)
phase, isBusinessLimited, errorOwner, errorSource := classifyOpsErrorLog(
c,
"model_not_found",
"Model \"gpt-missing\" is not supported by any configured account in this group",
"",
http.StatusNotFound,
)
require.Equal(t, "routing", phase)
require.True(t, isBusinessLimited)
require.Equal(t, "platform", errorOwner)
require.Equal(t, "gateway", errorSource)
}
func TestClassifyOpsLocalModelConfigurationOverridesStaleUpstreamMarkers(t *testing.T) {
gin.SetMode(gin.TestMode)
c, _ := gin.CreateTestContext(httptest.NewRecorder())
service.MarkOpsClientBusinessLimited(c, service.OpsClientBusinessLimitedReasonLocalModelConfiguration)
c.Set(service.OpsUpstreamStatusCodeKey, http.StatusUnauthorized)
c.Set(service.OpsUpstreamErrorsKey, []*service.OpsUpstreamErrorEvent{{
Stage: string(service.GatewayFailureStageAccountAuth),
UpstreamStatusCode: http.StatusUnauthorized,
}})
phase, limited, owner, source := classifyOpsErrorLog(c, "model_not_found", "unsupported configured model", "", http.StatusNotFound)
require.Equal(t, "routing", phase)
require.True(t, limited)
require.Equal(t, "platform", owner)
require.Equal(t, "gateway", source)
}
func TestClassifyOpsLocalModelConfigurationRequiresMarkerAndReason(t *testing.T) {
gin.SetMode(gin.TestMode)
c, _ := gin.CreateTestContext(httptest.NewRecorder())
c.Set(service.OpsClientBusinessLimitedReasonKey, service.OpsClientBusinessLimitedReasonLocalModelConfiguration)
c.Set(service.OpsUpstreamStatusCodeKey, http.StatusBadGateway)
phase, limited, owner, source := classifyOpsErrorLog(c, "upstream_error", "provider failed", "", http.StatusBadGateway)
require.Equal(t, "upstream", phase)
require.False(t, limited)
require.Equal(t, "provider", owner)
require.Equal(t, "upstream_http", source)
}
func TestOpsErrorLoggerMiddleware_LocalModelConfigurationFields(t *testing.T) {
setupOpsErrorLogTestQueue(t, 1)
gin.SetMode(gin.TestMode)
ops := service.NewOpsService(nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil)
router := gin.New()
router.Use(OpsErrorLoggerMiddleware(ops))
router.POST("/v1/chat/completions", func(c *gin.Context) {
service.MarkOpsClientBusinessLimited(c, service.OpsClientBusinessLimitedReasonLocalModelConfiguration)
c.Set(opsAccountIDKey, int64(99))
c.Set(opsUpstreamModelKey, "stale-upstream-model")
setActualUpstreamEndpoint(c, "/v1/chat/completions")
c.Set(service.OpsUpstreamStatusCodeKey, http.StatusUnauthorized)
c.Set(service.OpsUpstreamErrorMessageKey, "stale upstream error")
c.Set(service.OpsUpstreamErrorDetailKey, "stale upstream detail")
c.Set(service.OpsUpstreamErrorsKey, []*service.OpsUpstreamErrorEvent{{
Stage: string(service.GatewayFailureStageAccountAuth),
UpstreamStatusCode: http.StatusUnauthorized,
Message: "stale auth failure",
}})
c.JSON(http.StatusNotFound, gin.H{
"error": gin.H{
"type": "model_not_found",
"message": "Model \"gpt-missing\" is not supported by any configured account in this group",
},
})
})
w := httptest.NewRecorder()
req := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", nil)
router.ServeHTTP(w, req)
require.Equal(t, http.StatusNotFound, w.Code)
require.JSONEq(t, `{"error":{"type":"model_not_found","message":"Model \"gpt-missing\" is not supported by any configured account in this group"}}`, w.Body.String())
job := <-opsErrorLogQueue
require.Equal(t, http.StatusNotFound, job.entry.StatusCode)
require.Equal(t, "routing", job.entry.ErrorPhase)
require.True(t, job.entry.IsBusinessLimited)
require.Equal(t, "platform", job.entry.ErrorOwner)
require.Equal(t, "gateway", job.entry.ErrorSource)
require.Nil(t, job.entry.AccountID)
require.Nil(t, job.entry.UpstreamStatusCode)
require.Nil(t, job.entry.UpstreamErrors)
require.Nil(t, job.entry.UpstreamErrorMessage)
require.Nil(t, job.entry.UpstreamErrorDetail)
require.Empty(t, job.entry.UpstreamModel)
require.Empty(t, job.entry.UpstreamEndpoint)
}
func TestClassifyOpsAuthClientErrorsExcludedFromSLA(t *testing.T) {
tests := []struct {
name string
@@ -44,13 +44,14 @@ const (
// ensureForwardErrorResponse 检查此 key,为 true 时跳过兜底写入,避免在已完成的 JSON 后追加 SSE。
ResponseCommittedKey = "response_committed"
OpsClientBusinessLimitedKey = "ops_client_business_limited"
OpsClientBusinessLimitedReasonKey = "ops_client_business_limited_reason"
OpsClientBusinessLimitedReasonIPRestriction = "api_key_ip_restriction"
OpsClientBusinessLimitedReasonAPIKeyGroupUnavailable = "api_key_group_unavailable"
OpsClientBusinessLimitedReasonAPIKeyGroupUnassigned = "api_key_group_unassigned"
OpsClientBusinessLimitedReasonLocalFeatureGate = "local_feature_gate"
OpsClientBusinessLimitedReasonLocalPolicyDenied = "local_policy_denied"
OpsClientBusinessLimitedKey = "ops_client_business_limited"
OpsClientBusinessLimitedReasonKey = "ops_client_business_limited_reason"
OpsClientBusinessLimitedReasonIPRestriction = "api_key_ip_restriction"
OpsClientBusinessLimitedReasonAPIKeyGroupUnavailable = "api_key_group_unavailable"
OpsClientBusinessLimitedReasonAPIKeyGroupUnassigned = "api_key_group_unassigned"
OpsClientBusinessLimitedReasonLocalFeatureGate = "local_feature_gate"
OpsClientBusinessLimitedReasonLocalPolicyDenied = "local_policy_denied"
OpsClientBusinessLimitedReasonLocalModelConfiguration = "local_model_configuration"
)
func MarkResponseCommitted(c *gin.Context) { c.Set(ResponseCommittedKey, true) }
@@ -93,6 +94,18 @@ func HasOpsClientBusinessLimited(c *gin.Context) bool {
return marked
}
func OpsClientBusinessLimitedReason(c *gin.Context) string {
if c == nil {
return ""
}
v, ok := c.Get(OpsClientBusinessLimitedReasonKey)
if !ok {
return ""
}
reason, _ := v.(string)
return strings.TrimSpace(reason)
}
// OpsStreamError 描述网关在「响应状态已固化为 200」之后(keepalive ping 或部分数据
// 已 flush)就地以 SSE error 帧形式返回的错误。由于 HTTP 状态码停留在 200,
// 而 ops_error_logger 以 status>=400 为采集触发条件,这类流内失败