feat: 异步图片任务结果落对象存储

为异步生图任务增加 S3 兼容对象存储支持,任务结果不再把大图内联存进 Redis:

- 新增可插拔接口 service.ImageStorage(Save -> url),适配别的厂商只需实现它
- S3 实现 S3ImageStorage(AWS S3 / R2 / 阿里云 OSS / MinIO),与备份共用 S3 客户端构造
- 新增 image_storage 配置(config.yaml + IMAGE_STORAGE_* 环境变量),默认关闭
- enabled 同时作为总开关:关闭或未配置对象存储时,异步生图接口返回 404 且不写
  Redis,从根上避免几 MB 的 b64_json 结果撑爆 Redis
- 完成时把图片上传对象存储并把结果改写为短链接(公开直链或 presigned),
  上传失败则任务标记为失败

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01SM1tf3CFVRzC7guuhBXvMd
This commit is contained in:
haruka
2026-07-15 19:57:37 -07:00
co-authored by Claude Opus 4.8
parent e5e94d1eb1
commit 0eb6e21aaa
14 changed files with 710 additions and 37 deletions
+5 -1
View File
@@ -263,7 +263,11 @@ func initializeApplication(buildInfo handler.BuildInfo) (*Application, error) {
paymentWebhookHandler := handler.NewPaymentWebhookHandler(paymentService, registry)
availableChannelHandler := handler.NewAvailableChannelHandler(channelService, apiKeyService, settingService)
imageTaskStore := repository.NewImageTaskStore(redisClient)
imageTaskService := service.NewImageTaskService(imageTaskStore)
imageStorage, err := repository.ProvideImageStorage(configConfig)
if err != nil {
return nil, err
}
imageTaskService := service.ProvideImageTaskService(imageTaskStore, imageStorage, configConfig)
asyncImageHandler := handler.NewAsyncImageHandler(imageTaskService, openAIGatewayHandler)
batchImageRepository := repository.NewBatchImageRepository(db)
batchImageQueue := repository.NewBatchImageQueue(redisClient, configConfig)
+36
View File
@@ -95,6 +95,7 @@ type Config struct {
Update UpdateConfig `mapstructure:"update"`
Idempotency IdempotencyConfig `mapstructure:"idempotency"`
BatchImage BatchImageConfig `mapstructure:"batch_image"`
ImageStorage ImageStorageConfig `mapstructure:"image_storage"`
}
type LogConfig struct {
@@ -227,6 +228,33 @@ type BatchImageConfig struct {
VertexGCSBaseURL string `mapstructure:"vertex_gcs_base_url"`
}
// ImageStorageConfig 配置异步图片任务结果上传的 S3 兼容对象存储。
// Enabled 同时作为异步图片任务功能的总开关:未启用或未配置完整凭证时,
// 异步生图接口整体禁用,避免把上游返回的大 base64 结果塞进 Redis。
type ImageStorageConfig struct {
Enabled bool `mapstructure:"enabled"`
Endpoint string `mapstructure:"endpoint"` // e.g. https://<account_id>.r2.cloudflarestorage.com
Region string `mapstructure:"region"` // R2 用 "auto"
Bucket string `mapstructure:"bucket"`
AccessKeyID string `mapstructure:"access_key_id"`
SecretAccessKey string `mapstructure:"secret_access_key"`
Prefix string `mapstructure:"prefix"` // S3 key 前缀,如 "images/"
ForcePathStyle bool `mapstructure:"force_path_style"` // MinIO/路径风格桶
PublicBaseURL string `mapstructure:"public_base_url"` // 配了则返回 public_base_url/key 直链;否则 presigned
PresignExpiry int `mapstructure:"presign_expiry_hours"` // public_base_url 为空时的 presigned 过期时长(小时)
MaxDownloadByte int64 `mapstructure:"max_download_bytes"` // 下载上游 url 图片的字节上限
}
// IsConfigured 检查对象存储必要字段是否已配置
func (c *ImageStorageConfig) IsConfigured() bool {
return c.Bucket != "" && c.AccessKeyID != "" && c.SecretAccessKey != ""
}
// Active 返回异步图片任务是否可用:开关打开且凭证齐全
func (c *ImageStorageConfig) Active() bool {
return c.Enabled && c.IsConfigured()
}
type LinuxDoConnectConfig struct {
Enabled bool `mapstructure:"enabled"`
ClientID string `mapstructure:"client_id"`
@@ -1891,6 +1919,14 @@ func setDefaults() {
viper.SetDefault("batch_image.vertex_batch_prediction_base_url", "")
viper.SetDefault("batch_image.vertex_gcs_base_url", "")
// Image storage (async image task result offload to S3-compatible object storage)
viper.SetDefault("image_storage.enabled", false)
viper.SetDefault("image_storage.region", "auto")
viper.SetDefault("image_storage.prefix", "images/")
viper.SetDefault("image_storage.force_path_style", false)
viper.SetDefault("image_storage.presign_expiry_hours", 24)
viper.SetDefault("image_storage.max_download_bytes", 33554432)
// Ops (vNext)
viper.SetDefault("ops.enabled", true)
viper.SetDefault("ops.use_preaggregated_tables", true)
+15 -4
View File
@@ -32,9 +32,20 @@ func NewAsyncImageHandler(tasks *service.ImageTaskService, openAI *OpenAIGateway
return h
}
// enabled reports whether the async image task feature is available. Object
// storage is the enablement gate: without it the endpoints are fully disabled
// so that large base64 results never land in Redis.
func (h *AsyncImageHandler) enabled() bool {
return h != nil && h.tasks != nil && h.tasks.Enabled()
}
// Submit accepts the same payload as the synchronous Images endpoint and
// returns before the upstream image generation begins.
func (h *AsyncImageHandler) Submit(c *gin.Context) {
if !h.enabled() {
imageTaskJSONError(c, http.StatusNotFound, "not_found_error", "async image tasks are not enabled")
return
}
apiKey, ok := middleware2.GetAPIKeyFromContext(c)
if !ok || apiKey == nil || apiKey.UserID <= 0 || apiKey.ID <= 0 {
imageTaskError(c, service.ErrImageTaskForbidden)
@@ -105,15 +116,15 @@ func (h *AsyncImageHandler) Submit(c *gin.Context) {
}
func (h *AsyncImageHandler) Get(c *gin.Context) {
if !h.enabled() {
imageTaskJSONError(c, http.StatusNotFound, "not_found_error", "async image tasks are not enabled")
return
}
apiKey, ok := middleware2.GetAPIKeyFromContext(c)
if !ok || apiKey == nil || apiKey.UserID <= 0 || apiKey.ID <= 0 {
imageTaskError(c, service.ErrImageTaskForbidden)
return
}
if h == nil || h.tasks == nil {
imageTaskError(c, service.ErrImageTaskUnavailable)
return
}
task, err := h.tasks.Get(c.Request.Context(), service.ImageTaskOwner{UserID: apiKey.UserID, APIKeyID: apiKey.ID}, c.Param("task_id"))
if err != nil {
imageTaskError(c, err)
@@ -47,7 +47,7 @@ func (s *asyncImageMemoryStore) Get(_ context.Context, id string) (*service.Imag
func TestAsyncImageHandlerSubmitAndPoll(t *testing.T) {
gin.SetMode(gin.TestMode)
store := &asyncImageMemoryStore{tasks: make(map[string]*service.ImageTaskRecord)}
tasks := service.NewImageTaskServiceWithOptions(store, time.Hour, time.Minute)
tasks := service.NewImageTaskServiceWithUploader(store, nil, time.Hour, time.Minute)
release := make(chan struct{})
h := &AsyncImageHandler{tasks: tasks}
h.execute = func(_ string, c *gin.Context) {
@@ -105,3 +105,41 @@ func TestAsyncImageHandlerSubmitAndPoll(t *testing.T) {
require.Empty(t, pollWriter.Header().Get("Retry-After"))
require.Contains(t, pollWriter.Body.String(), "https://example.test/image.png")
}
// When object storage is not configured the feature is fully disabled: the
// endpoints must return 404 without creating a task or writing to Redis.
func TestAsyncImageHandlerDisabledReturns404(t *testing.T) {
gin.SetMode(gin.TestMode)
store := &asyncImageMemoryStore{tasks: make(map[string]*service.ImageTaskRecord)}
tasks := service.NewImageTaskServiceWithOptions(store, time.Hour, time.Minute) // enabled == false
h := &AsyncImageHandler{tasks: tasks}
router := gin.New()
router.Use(func(c *gin.Context) {
groupID := int64(3)
c.Set(string(middleware2.ContextKeyAPIKey), &service.APIKey{
ID: 9,
UserID: 7,
GroupID: &groupID,
Group: &service.Group{ID: groupID, Platform: service.PlatformOpenAI, AllowImageGeneration: true},
})
c.Next()
})
router.POST("/v1/images/generations/async", h.Submit)
router.GET("/v1/images/tasks/:task_id", h.Get)
req := httptest.NewRequest(http.MethodPost, "/v1/images/generations/async", strings.NewReader(`{"model":"gpt-image-1","prompt":"cat"}`))
req.Header.Set("Content-Type", "application/json")
w := httptest.NewRecorder()
router.ServeHTTP(w, req)
require.Equal(t, http.StatusNotFound, w.Code)
require.Contains(t, w.Body.String(), "not enabled")
pollReq := httptest.NewRequest(http.MethodGet, "/v1/images/tasks/imgtask_missing", nil)
pollWriter := httptest.NewRecorder()
router.ServeHTTP(pollWriter, pollReq)
require.Equal(t, http.StatusNotFound, pollWriter.Code)
// No task was created / persisted.
require.Empty(t, store.tasks)
}
+9 -29
View File
@@ -7,10 +7,6 @@ import (
"io"
"time"
"github.com/aws/aws-sdk-go-v2/aws"
v4 "github.com/aws/aws-sdk-go-v2/aws/signer/v4"
awsconfig "github.com/aws/aws-sdk-go-v2/config"
"github.com/aws/aws-sdk-go-v2/credentials"
"github.com/aws/aws-sdk-go-v2/service/s3"
"github.com/Wei-Shaw/sub2api/internal/pkg/servertiming"
@@ -26,32 +22,16 @@ type S3BackupStore struct {
// NewS3BackupStoreFactory returns a BackupObjectStoreFactory that creates S3-backed stores
func NewS3BackupStoreFactory() service.BackupObjectStoreFactory {
return func(ctx context.Context, cfg *service.BackupS3Config) (service.BackupObjectStore, error) {
region := cfg.Region
if region == "" {
region = "auto" // Cloudflare R2 默认 region
}
awsCfg, err := awsconfig.LoadDefaultConfig(ctx,
awsconfig.WithRegion(region),
awsconfig.WithCredentialsProvider(
credentials.NewStaticCredentialsProvider(cfg.AccessKeyID, cfg.SecretAccessKey, ""),
),
)
if err != nil {
return nil, fmt.Errorf("load aws config: %w", err)
}
client := s3.NewFromConfig(awsCfg, func(o *s3.Options) {
if cfg.Endpoint != "" {
o.BaseEndpoint = &cfg.Endpoint
}
if cfg.ForcePathStyle {
o.UsePathStyle = true
}
o.APIOptions = append(o.APIOptions, v4.SwapComputePayloadSHA256ForUnsignedPayloadMiddleware)
o.RequestChecksumCalculation = aws.RequestChecksumCalculationWhenRequired
client, err := newS3Client(ctx, s3ClientParams{
Endpoint: cfg.Endpoint,
Region: cfg.Region,
AccessKeyID: cfg.AccessKeyID,
SecretAccessKey: cfg.SecretAccessKey,
ForcePathStyle: cfg.ForcePathStyle,
})
if err != nil {
return nil, err
}
return &S3BackupStore{client: client, bucket: cfg.Bucket}, nil
}
}
@@ -0,0 +1,80 @@
package repository
import (
"bytes"
"context"
"fmt"
"strings"
"time"
"github.com/aws/aws-sdk-go-v2/service/s3"
"github.com/Wei-Shaw/sub2api/internal/config"
"github.com/Wei-Shaw/sub2api/internal/pkg/servertiming"
"github.com/Wei-Shaw/sub2api/internal/service"
)
// S3ImageStorage 用 S3 兼容对象存储实现 service.ImageStorage。
type S3ImageStorage struct {
client *s3.Client
bucket string
publicBaseURL string
presignExpiry time.Duration
}
var _ service.ImageStorage = (*S3ImageStorage)(nil)
// NewS3ImageStorage 依据配置构造 S3 图片存储(调用方应先确认 cfg.Active())。
func NewS3ImageStorage(ctx context.Context, cfg *config.ImageStorageConfig) (*S3ImageStorage, error) {
client, err := newS3Client(ctx, s3ClientParams{
Endpoint: cfg.Endpoint,
Region: cfg.Region,
AccessKeyID: cfg.AccessKeyID,
SecretAccessKey: cfg.SecretAccessKey,
ForcePathStyle: cfg.ForcePathStyle,
})
if err != nil {
return nil, err
}
expiry := time.Duration(cfg.PresignExpiry) * time.Hour
if expiry <= 0 {
expiry = 24 * time.Hour
}
return &S3ImageStorage{
client: client,
bucket: cfg.Bucket,
publicBaseURL: strings.TrimRight(cfg.PublicBaseURL, "/"),
presignExpiry: expiry,
}, nil
}
// Save 上传图片字节,返回可访问 URL:配了 public_base_url 则返回公开直链,否则返回 presigned 临时链接。
func (s *S3ImageStorage) Save(ctx context.Context, key, contentType string, data []byte) (string, error) {
finish := servertiming.ObserveDependency(ctx, "s3")
_, err := s.client.PutObject(ctx, &s3.PutObjectInput{
Bucket: &s.bucket,
Key: &key,
Body: bytes.NewReader(data),
ContentType: &contentType,
})
finish()
if err != nil {
return "", fmt.Errorf("S3 PutObject: %w", err)
}
if s.publicBaseURL != "" {
return s.publicBaseURL + "/" + strings.TrimLeft(key, "/"), nil
}
presignClient := s3.NewPresignClient(s.client)
result, err := presignClient.PresignGetObject(ctx, &s3.GetObjectInput{
Bucket: &s.bucket,
Key: &key,
}, s3.WithPresignExpires(s.presignExpiry))
if err != nil {
return "", fmt.Errorf("presign url: %w", err)
}
return result.URL, nil
}
+53
View File
@@ -0,0 +1,53 @@
package repository
import (
"context"
"fmt"
"github.com/aws/aws-sdk-go-v2/aws"
v4 "github.com/aws/aws-sdk-go-v2/aws/signer/v4"
awsconfig "github.com/aws/aws-sdk-go-v2/config"
"github.com/aws/aws-sdk-go-v2/credentials"
"github.com/aws/aws-sdk-go-v2/service/s3"
)
// s3ClientParams 描述构造 S3 兼容客户端所需的参数。
type s3ClientParams struct {
Endpoint string
Region string
AccessKeyID string
SecretAccessKey string
ForcePathStyle bool
}
// newS3Client 构造一个 S3 兼容客户端,兼容 AWS S3 / Cloudflare R2 / 阿里云 OSS / MinIO。
//
// 通过 SwapComputePayloadSHA256ForUnsignedPayloadMiddleware + RequestChecksumCalculationWhenRequired
// 规避阿里云 OSS 不兼容 s3manager 分片签名的问题(backup 与 image storage 共用此构造)。
func newS3Client(ctx context.Context, p s3ClientParams) (*s3.Client, error) {
region := p.Region
if region == "" {
region = "auto" // Cloudflare R2 默认 region
}
awsCfg, err := awsconfig.LoadDefaultConfig(ctx,
awsconfig.WithRegion(region),
awsconfig.WithCredentialsProvider(
credentials.NewStaticCredentialsProvider(p.AccessKeyID, p.SecretAccessKey, ""),
),
)
if err != nil {
return nil, fmt.Errorf("load aws config: %w", err)
}
return s3.NewFromConfig(awsCfg, func(o *s3.Options) {
if p.Endpoint != "" {
o.BaseEndpoint = &p.Endpoint
}
if p.ForcePathStyle {
o.UsePathStyle = true
}
o.APIOptions = append(o.APIOptions, v4.SwapComputePayloadSHA256ForUnsignedPayloadMiddleware)
o.RequestChecksumCalculation = aws.RequestChecksumCalculationWhenRequired
}), nil
}
+17
View File
@@ -1,6 +1,7 @@
package repository
import (
"context"
"database/sql"
"errors"
@@ -137,6 +138,9 @@ var ProviderSet = wire.NewSet(
NewPgDumper,
NewS3BackupStoreFactory,
// Image storage (async image task result offload)
ProvideImageStorage,
// HTTP service ports (DI Strategy A: return interface directly)
NewTurnstileVerifier,
ProvidePricingRemoteClient,
@@ -168,6 +172,19 @@ func ProvideEnt(cfg *config.Config) (*ent.Client, error) {
return client, err
}
// ProvideImageStorage 提供异步图片任务结果转存所用的对象存储实现。
// 仅当开关打开且 S3 凭证齐全时返回具体实现,否则返回 nil(功能整体禁用)。
func ProvideImageStorage(cfg *config.Config) (service.ImageStorage, error) {
if !cfg.ImageStorage.Active() {
return nil, nil
}
store, err := NewS3ImageStorage(context.Background(), &cfg.ImageStorage)
if err != nil {
return nil, err
}
return store, nil
}
// ProvideSQLDB 从 Ent 客户端提取底层的 *sql.DB 连接。
//
// 某些 Repository 需要直接执行原生 SQL(如复杂的批量更新、聚合查询),
+192
View File
@@ -0,0 +1,192 @@
package service
import (
"context"
"encoding/base64"
"encoding/json"
"errors"
"fmt"
"io"
"net/http"
"strconv"
"strings"
"time"
)
const defaultImageMaxDownloadBytes int64 = 32 << 20 // 32 MiB
// ImageStorage 把图片字节写入对象存储并返回可访问 URL。
//
// 这是对象存储的可插拔抽象:适配一个新的对象存储厂商,只需实现本接口
// (例如包一个厂商 SDK),无需改动任务/网关逻辑。仓库内自带一个 S3 兼容实现
// (repository.S3ImageStorage),适用于 AWS S3 / Cloudflare R2 / 阿里云 OSS / MinIO 等。
type ImageStorage interface {
// Save 把 data 以 key 存入对象存储,返回可下载的 URL(公开直链或 presigned 临时链接)。
// contentType 为图片 MIME 类型,如 "image/png"。
Save(ctx context.Context, key, contentType string, data []byte) (url string, err error)
}
// ImageResultUploader 是 ImageStorage 的上层编排器(与具体厂商无关):
// 把上游生图响应里的每张图片(b64_json 解码 / url 下载)转存到对象存储,
// 并把响应结果改写为只含短链接的紧凑 JSON,从而避免大 base64 落 Redis。
type ImageResultUploader struct {
storage ImageStorage
httpClient *http.Client
prefix string
maxDownloadBytes int64
}
// NewImageResultUploader 构造一个 uploader;storage 为 nil 时 Rewrite 直接透传。
func NewImageResultUploader(storage ImageStorage, prefix string, maxDownloadBytes int64, httpClient *http.Client) *ImageResultUploader {
if httpClient == nil {
httpClient = defaultImageDownloadHTTPClient()
}
if maxDownloadBytes <= 0 {
maxDownloadBytes = defaultImageMaxDownloadBytes
}
return &ImageResultUploader{
storage: storage,
httpClient: httpClient,
prefix: prefix,
maxDownloadBytes: maxDownloadBytes,
}
}
func defaultImageDownloadHTTPClient() *http.Client {
return &http.Client{Timeout: 60 * time.Second}
}
// Rewrite 将 result(上游生图响应 JSON)里的每张图片转存到对象存储,
// 返回改写后的紧凑结果(data[i].url 指向对象存储,b64_json 被移除)。
// 任一图片转存失败即返回 error(调用方据此将任务标记为失败,绝不把大 blob 落 Redis)。
func (u *ImageResultUploader) Rewrite(ctx context.Context, taskID string, result json.RawMessage) (json.RawMessage, error) {
if u == nil || u.storage == nil {
return result, nil
}
var top map[string]json.RawMessage
if err := json.Unmarshal(result, &top); err != nil {
return nil, fmt.Errorf("parse image response: %w", err)
}
rawData, ok := top["data"]
if !ok {
// 没有 data 数组(结构不符合预期),保持原样返回,交由上层决定。
return result, nil
}
var items []map[string]json.RawMessage
if err := json.Unmarshal(rawData, &items); err != nil {
return nil, fmt.Errorf("parse image response data: %w", err)
}
if len(items) == 0 {
return result, nil
}
for i, item := range items {
data, contentType, err := u.fetchImageBytes(ctx, item)
if err != nil {
return nil, fmt.Errorf("image %d: %w", i, err)
}
key := u.buildKey(taskID, i, contentType)
url, err := u.storage.Save(ctx, key, contentType, data)
if err != nil {
return nil, fmt.Errorf("image %d: upload to object storage: %w", i, err)
}
urlRaw, err := json.Marshal(url)
if err != nil {
return nil, fmt.Errorf("image %d: encode url: %w", i, err)
}
item["url"] = urlRaw
delete(item, "b64_json")
items[i] = item
}
newData, err := json.Marshal(items)
if err != nil {
return nil, fmt.Errorf("encode image response data: %w", err)
}
top["data"] = newData
out, err := json.Marshal(top)
if err != nil {
return nil, fmt.Errorf("encode image response: %w", err)
}
return out, nil
}
func (u *ImageResultUploader) fetchImageBytes(ctx context.Context, item map[string]json.RawMessage) ([]byte, string, error) {
if raw, ok := item["b64_json"]; ok {
var b64 string
if err := json.Unmarshal(raw, &b64); err == nil {
if b64 = strings.TrimSpace(b64); b64 != "" {
data, err := base64.StdEncoding.DecodeString(b64)
if err != nil {
return nil, "", fmt.Errorf("decode b64_json: %w", err)
}
return data, detectImageContentType(data), nil
}
}
}
if raw, ok := item["url"]; ok {
var rawURL string
if err := json.Unmarshal(raw, &rawURL); err == nil {
if rawURL = strings.TrimSpace(rawURL); rawURL != "" {
return u.download(ctx, rawURL)
}
}
}
return nil, "", errors.New("image item has neither b64_json nor url")
}
func (u *ImageResultUploader) download(ctx context.Context, rawURL string) ([]byte, string, error) {
req, err := http.NewRequestWithContext(ctx, http.MethodGet, rawURL, nil)
if err != nil {
return nil, "", fmt.Errorf("build download request: %w", err)
}
resp, err := u.httpClient.Do(req)
if err != nil {
return nil, "", fmt.Errorf("download image: %w", err)
}
defer func() { _ = resp.Body.Close() }()
if resp.StatusCode < http.StatusOK || resp.StatusCode >= http.StatusMultipleChoices {
return nil, "", fmt.Errorf("download image: unexpected status %d", resp.StatusCode)
}
limit := u.maxDownloadBytes
if limit <= 0 {
limit = defaultImageMaxDownloadBytes
}
data, err := io.ReadAll(io.LimitReader(resp.Body, limit+1))
if err != nil {
return nil, "", fmt.Errorf("read image body: %w", err)
}
if int64(len(data)) > limit {
return nil, "", fmt.Errorf("downloaded image exceeds %d bytes", limit)
}
contentType := strings.TrimSpace(strings.Split(resp.Header.Get("Content-Type"), ";")[0])
if !strings.HasPrefix(contentType, "image/") {
contentType = detectImageContentType(data)
}
return data, contentType, nil
}
func (u *ImageResultUploader) buildKey(taskID string, index int, contentType string) string {
return u.prefix + taskID + "-" + strconv.Itoa(index) + extensionForContentType(contentType)
}
func detectImageContentType(data []byte) string {
ct := strings.TrimSpace(strings.Split(http.DetectContentType(data), ";")[0])
if strings.HasPrefix(ct, "image/") {
return ct
}
return "image/png"
}
func extensionForContentType(ct string) string {
switch {
case strings.Contains(ct, "png"):
return ".png"
case strings.Contains(ct, "jpeg"), strings.Contains(ct, "jpg"):
return ".jpg"
case strings.Contains(ct, "webp"):
return ".webp"
case strings.Contains(ct, "gif"):
return ".gif"
default:
return ".png"
}
}
@@ -0,0 +1,157 @@
package service
import (
"context"
"encoding/base64"
"encoding/json"
"errors"
"net/http"
"net/http/httptest"
"testing"
"time"
"github.com/stretchr/testify/require"
)
// pngBytes is a minimal payload whose signature makes http.DetectContentType
// report image/png.
var pngBytes = []byte("\x89PNG\r\n\x1a\nfake-png-payload")
type savedImage struct {
key string
contentType string
data []byte
}
type fakeImageStorage struct {
saved []savedImage
url string
err error
}
func (f *fakeImageStorage) Save(_ context.Context, key, contentType string, data []byte) (string, error) {
if f.err != nil {
return "", f.err
}
f.saved = append(f.saved, savedImage{key: key, contentType: contentType, data: append([]byte(nil), data...)})
if f.url != "" {
return f.url, nil
}
return "https://cdn.test/" + key, nil
}
func TestImageResultUploaderRewritesB64JSON(t *testing.T) {
storage := &fakeImageStorage{}
uploader := NewImageResultUploader(storage, "images/", 0, nil)
b64 := base64.StdEncoding.EncodeToString(pngBytes)
result := json.RawMessage(`{"created":1,"data":[{"b64_json":"` + b64 + `","revised_prompt":"a cat"}]}`)
out, err := uploader.Rewrite(context.Background(), "imgtask_abc", result)
require.NoError(t, err)
require.Len(t, storage.saved, 1)
require.Equal(t, "images/imgtask_abc-0.png", storage.saved[0].key)
require.Equal(t, "image/png", storage.saved[0].contentType)
require.Equal(t, pngBytes, storage.saved[0].data)
var parsed struct {
Data []map[string]json.RawMessage `json:"data"`
}
require.NoError(t, json.Unmarshal(out, &parsed))
require.Len(t, parsed.Data, 1)
require.JSONEq(t, `"https://cdn.test/images/imgtask_abc-0.png"`, string(parsed.Data[0]["url"]))
_, hasB64 := parsed.Data[0]["b64_json"]
require.False(t, hasB64, "b64_json must be stripped after offload")
require.JSONEq(t, `"a cat"`, string(parsed.Data[0]["revised_prompt"]), "unrelated fields preserved")
}
func TestImageResultUploaderRewritesURL(t *testing.T) {
upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
w.Header().Set("Content-Type", "image/png")
_, _ = w.Write(pngBytes)
}))
defer upstream.Close()
storage := &fakeImageStorage{}
uploader := NewImageResultUploader(storage, "images/", 0, nil)
result := json.RawMessage(`{"created":1,"data":[{"url":"` + upstream.URL + `/pic.png"}]}`)
out, err := uploader.Rewrite(context.Background(), "imgtask_xyz", result)
require.NoError(t, err)
require.Len(t, storage.saved, 1)
require.Equal(t, pngBytes, storage.saved[0].data)
require.Equal(t, "image/png", storage.saved[0].contentType)
var parsed struct {
Data []map[string]json.RawMessage `json:"data"`
}
require.NoError(t, json.Unmarshal(out, &parsed))
require.JSONEq(t, `"https://cdn.test/images/imgtask_xyz-0.png"`, string(parsed.Data[0]["url"]))
}
func TestImageResultUploaderPropagatesStorageError(t *testing.T) {
storage := &fakeImageStorage{err: errors.New("bucket unreachable")}
uploader := NewImageResultUploader(storage, "images/", 0, nil)
b64 := base64.StdEncoding.EncodeToString(pngBytes)
result := json.RawMessage(`{"data":[{"b64_json":"` + b64 + `"}]}`)
_, err := uploader.Rewrite(context.Background(), "imgtask_err", result)
require.Error(t, err)
require.Contains(t, err.Error(), "bucket unreachable")
}
func TestImageResultUploaderNilStoragePassthrough(t *testing.T) {
var uploader *ImageResultUploader
result := json.RawMessage(`{"data":[{"url":"https://example.test/x.png"}]}`)
out, err := uploader.Rewrite(context.Background(), "imgtask_nil", result)
require.NoError(t, err)
require.JSONEq(t, string(result), string(out))
}
func TestImageTaskServiceCompleteOffloadsToStorage(t *testing.T) {
store := &imageTaskMemoryStore{}
storage := &fakeImageStorage{}
uploader := NewImageResultUploader(storage, "images/", 0, nil)
svc := NewImageTaskServiceWithUploader(store, uploader, time.Hour, time.Minute)
require.True(t, svc.Enabled())
owner := ImageTaskOwner{UserID: 1, APIKeyID: 2}
created, err := svc.Create(context.Background(), owner)
require.NoError(t, err)
b64 := base64.StdEncoding.EncodeToString(pngBytes)
result := json.RawMessage(`{"created":1,"data":[{"b64_json":"` + b64 + `"}]}`)
require.NoError(t, svc.Complete(context.Background(), created.ID, http.StatusOK, result))
got, err := svc.Get(context.Background(), owner, created.ID)
require.NoError(t, err)
require.Equal(t, ImageTaskStatusCompleted, got.Status)
require.Equal(t, "https://cdn.test/images/"+created.ID+"-0.png", got.ImageURL)
require.NotContains(t, string(got.Result), "b64_json", "large base64 must not be persisted to Redis")
require.Len(t, storage.saved, 1)
}
func TestImageTaskServiceCompleteOffloadFailureMarksFailed(t *testing.T) {
store := &imageTaskMemoryStore{}
storage := &fakeImageStorage{err: errors.New("bucket unreachable")}
uploader := NewImageResultUploader(storage, "images/", 0, nil)
svc := NewImageTaskServiceWithUploader(store, uploader, time.Hour, time.Minute)
owner := ImageTaskOwner{UserID: 1, APIKeyID: 2}
created, err := svc.Create(context.Background(), owner)
require.NoError(t, err)
b64 := base64.StdEncoding.EncodeToString(pngBytes)
result := json.RawMessage(`{"data":[{"b64_json":"` + b64 + `"}]}`)
require.NoError(t, svc.Complete(context.Background(), created.ID, http.StatusOK, result))
got, err := svc.Get(context.Background(), owner, created.ID)
require.NoError(t, err)
require.Equal(t, ImageTaskStatusFailed, got.Status)
require.Equal(t, http.StatusBadGateway, got.HTTPStatus)
require.Contains(t, string(got.Error), "object storage")
require.NotContains(t, string(got.Result), "b64_json", "failed offload must not persist base64 to Redis")
}
+28
View File
@@ -9,7 +9,9 @@ import (
"time"
infraerrors "github.com/Wei-Shaw/sub2api/internal/pkg/errors"
"github.com/Wei-Shaw/sub2api/internal/pkg/logger"
"github.com/google/uuid"
"go.uber.org/zap"
)
const (
@@ -69,6 +71,8 @@ type ImageTaskStore interface {
type ImageTaskService struct {
store ImageTaskStore
uploader *ImageResultUploader
enabled bool
ttl time.Duration
executionTimeout time.Duration
}
@@ -87,6 +91,21 @@ func NewImageTaskServiceWithOptions(store ImageTaskStore, ttl, executionTimeout
return &ImageTaskService{store: store, ttl: ttl, executionTimeout: executionTimeout}
}
// NewImageTaskServiceWithUploader 构造一个已启用的图片任务服务:结果会先经 uploader
// 转存到对象存储再落 Redis。uploader 为 nil 时不做转存(仅用于测试)。
func NewImageTaskServiceWithUploader(store ImageTaskStore, uploader *ImageResultUploader, ttl, executionTimeout time.Duration) *ImageTaskService {
s := NewImageTaskServiceWithOptions(store, ttl, executionTimeout)
s.uploader = uploader
s.enabled = true
return s
}
// Enabled 表示异步图片任务功能是否可用(总开关 + 凭证齐全)。
// 关闭时 handler 直接返回 404,不创建任务、不写 Redis。
func (s *ImageTaskService) Enabled() bool {
return s != nil && s.enabled && s.store != nil
}
func (s *ImageTaskService) ExecutionTimeout() time.Duration {
if s == nil || s.executionTimeout <= 0 {
return defaultImageTaskExecutionTimeout
@@ -135,6 +154,15 @@ func (s *ImageTaskService) Complete(ctx context.Context, id string, statusCode i
if !json.Valid(result) {
return s.Fail(ctx, id, http.StatusBadGateway, imageTaskErrorJSON("api_error", "upstream returned a non-JSON image response"))
}
if s.uploader != nil {
rewritten, err := s.uploader.Rewrite(ctx, id, result)
if err != nil {
// 转存失败不回退存 base64,避免大 blob 撑爆 Redis:直接把任务标记为失败。
logger.L().Error("image_task.offload_failed", zap.String("task_id", id), zap.Error(err))
return s.Fail(ctx, id, http.StatusBadGateway, imageTaskErrorJSON("api_error", "failed to store generated image to object storage"))
}
result = rewritten
}
return s.finish(ctx, id, ImageTaskStatusCompleted, statusCode, result, nil)
}
+17 -1
View File
@@ -513,6 +513,22 @@ func ProvideAPIKeyAuthCacheInvalidator(apiKeyService *APIKeyService) APIKeyAuthC
return apiKeyService
}
// ProvideImageTaskService 构造异步图片任务服务。
//
// 对象存储是异步图片任务的启用前提:仅当 image_storage 开关打开且凭证齐全时,
// 服务才启用,并挂上把结果转存到对象存储的 uploader;否则功能整体禁用
// (handler 返回 404,不创建任务、不写 Redis),从而避免大 base64 结果撑爆 Redis。
func ProvideImageTaskService(store ImageTaskStore, storage ImageStorage, cfg *config.Config) *ImageTaskService {
if !cfg.ImageStorage.Active() {
if cfg.ImageStorage.Enabled {
logger.L().Warn("image_storage.enabled is true but object storage is not fully configured; async image tasks are disabled")
}
return NewImageTaskService(store)
}
uploader := NewImageResultUploader(storage, cfg.ImageStorage.Prefix, cfg.ImageStorage.MaxDownloadByte, nil)
return NewImageTaskServiceWithUploader(store, uploader, defaultImageTaskTTL, defaultImageTaskExecutionTimeout)
}
// ProvideBackupService creates and starts BackupService
func ProvideBackupService(
settingRepo SettingRepository,
@@ -637,7 +653,7 @@ var ProviderSet = wire.NewSet(
NewAdminService,
NewGatewayService,
NewOpenAIGatewayService,
NewImageTaskService,
ProvideImageTaskService,
ProvideBatchImageModelPricingResolver,
NewBatchImagePublicService,
NewBatchImageDownloadService,
+36
View File
@@ -1175,3 +1175,39 @@ update:
# Leave empty for direct connection (recommended for overseas servers)
# 留空表示直连(适用于海外服务器)
proxy_url: ""
# =============================================================================
# Image Storage (异步图片任务结果对象存储)
# =============================================================================
# 长耗时生图套 Cloudflare 会 524 超时,异步图片任务接口(/v1/images/generations/async
# 等)先返回 task_id,再由客户端轮询 /v1/images/tasks/{task_id} 获取结果。
#
# 这里配置一个 S3 兼容对象存储(AWS S3 / Cloudflare R2 / 阿里云 OSS / MinIO 等),
# 任务完成后把生成的图片上传到对象存储,只在 Redis 存一个短链接,避免 gpt-image-1 等
# 返回的大 base64 结果把 Redis 撑爆。
#
# enabled 同时是异步图片任务功能的总开关:为 false 或凭证未配全时,异步生图接口
# 整体返回 404、不创建任务、不写 Redis。
#
# 换其它厂商对象存储:只要它兼容 S3 API 即可直接用;如需完全自定义,实现
# service.ImageStorage 接口(Save(ctx, key, contentType, data) -> url)即可。
image_storage:
enabled: false
# S3 兼容端点。AWS 官方可留空;R2 形如 https://<account_id>.r2.cloudflarestorage.com
endpoint: ""
# 区域。Cloudflare R2 用 "auto"
region: "auto"
bucket: ""
access_key_id: ""
secret_access_key: ""
# 对象 key 前缀
prefix: "images/"
# MinIO / 需要路径风格(path-style)访问的桶设为 true
force_path_style: false
# 若填写公开桶 / CDN 域名,则返回 public_base_url/key 永久直链;
# 留空则返回带过期时间的 presigned 临时链接
public_base_url: ""
# public_base_url 为空时,presigned 链接的有效时长(小时)
presign_expiry_hours: 24
# 当上游返回的是图片 url 时,下载该图片再转存的字节上限(默认 32MB)
max_download_bytes: 33554432
+26 -1
View File
@@ -16,6 +16,31 @@ The aliases are `/images/generations/async`, `/images/edits/async`, and `/images
Only OpenAI and Grok groups are supported. Requests use the same JSON or multipart payload as the corresponding synchronous endpoint. Streaming image requests are rejected because a polled task returns one final JSON result.
## Enabling the feature (object storage)
Asynchronous image tasks are **disabled by default** and gated on object storage. When the switch is off — or the S3 credentials are incomplete — the async endpoints return `404` and never create a task or write to Redis. This is deliberate: without offloading, large `b64_json` results (several MB each, e.g. `gpt-image-1`) would accumulate in Redis and exhaust its memory.
Configure an S3-compatible object store (AWS S3, Cloudflare R2, Aliyun OSS, MinIO, …) in `config.yaml` (all keys also accept the `IMAGE_STORAGE_*` environment overrides):
```yaml
image_storage:
enabled: true
endpoint: "https://<account_id>.r2.cloudflarestorage.com" # AWS 官方可留空
region: "auto"
bucket: "my-images"
access_key_id: "..."
secret_access_key: "..."
prefix: "images/"
force_path_style: false # MinIO/path-style buckets set true
public_base_url: "" # set to return public_base_url/key直链; empty → presigned URL
presign_expiry_hours: 24 # presigned link TTL when public_base_url is empty
max_download_bytes: 33554432 # cap when re-hosting an upstream image URL (32MB)
```
When a task completes, each generated image is uploaded to the bucket and the result is rewritten to a compact form: `data[].url` points at the stored object (a permanent `public_base_url/key` link, or a time-limited presigned URL) and `b64_json` is removed. Only this small JSON is stored in Redis. If an upload fails, the task is marked `failed` rather than persisting the raw base64.
To support a different vendor beyond the S3-compatible client, implement the `service.ImageStorage` interface (`Save(ctx, key, contentType, data) (url, error)`) and provide it in place of the S3 implementation.
## Submit a task
```bash
@@ -67,7 +92,7 @@ While work is in progress:
}
```
On success, `result` is the unmodified JSON body from the synchronous image API, so URL and base64 response formats both remain supported:
On success, `result` mirrors the synchronous image API body, except each image has been offloaded to object storage: `data[].url` points at the stored object and `b64_json` is stripped (so both URL and base64 upstream formats end up as compact stored links):
```json
{