From a5a0383f10af82df8154ced0eb6d3e15e05e1fee Mon Sep 17 00:00:00 2001 From: YunaiV Date: Sun, 24 Aug 2025 20:32:19 +0800 Subject: [PATCH] =?UTF-8?q?feat=EF=BC=9A=E3=80=90ai=20=E5=A4=A7=E6=A8=A1?= =?UTF-8?q?=E5=9E=8B=E3=80=91=E5=AF=B9=E8=AF=9D=E5=A2=9E=E5=8A=A0=E9=99=84?= =?UTF-8?q?=E4=BB=B6=E7=9A=84=E6=94=AF=E6=8C=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../admin/chat/AiChatMessageController.http | 24 +++++ .../chat/vo/message/AiChatMessageRespVO.java | 3 + .../vo/message/AiChatMessageSendReqVO.java | 7 +- .../dal/dataobject/chat/AiChatMessageDO.java | 7 ++ .../chat/AiChatMessageServiceImpl.java | 93 ++++++++++++++++--- .../file/core/utils/FileTypeUtils.java | 18 +++- 6 files changed, 133 insertions(+), 19 deletions(-) diff --git a/yudao-module-ai/src/main/java/cn/iocoder/yudao/module/ai/controller/admin/chat/AiChatMessageController.http b/yudao-module-ai/src/main/java/cn/iocoder/yudao/module/ai/controller/admin/chat/AiChatMessageController.http index 4c4c8c0891..5cc54fc8e0 100644 --- a/yudao-module-ai/src/main/java/cn/iocoder/yudao/module/ai/controller/admin/chat/AiChatMessageController.http +++ b/yudao-module-ai/src/main/java/cn/iocoder/yudao/module/ai/controller/admin/chat/AiChatMessageController.http @@ -20,6 +20,30 @@ tenant-id: {{adminTenantId}} "content": "1+1=?" } +### 发送消息（流式）【带文件】 +POST {{baseUrl}}/ai/chat/message/send-stream +Content-Type: application/json +Authorization: {{token}} +tenant-id: {{adminTenantId}} + +{ + "conversationId": "1781604279872581797", + "content": "图片里有什么？", + "attachmentUrls": ["http://test.yudao.iocoder.cn/1755531278.jpeg"] +} + +### 发送消息（流式）【追问带文件】 +POST {{baseUrl}}/ai/chat/message/send-stream +Content-Type: application/json +Authorization: {{token}} +tenant-id: {{adminTenantId}} + +{ + "conversationId": "1781604279872581797", + "content": "说下图片里，有哪些字？", + "useContext": true +} + ### 获得指定对话的消息列表 GET {{baseUrl}}/ai/chat/message/list-by-conversation-id?conversationId=1781604279872581649 Authorization: {{token}} diff --git a/yudao-module-ai/src/main/java/cn/iocoder/yudao/module/ai/controller/admin/chat/vo/message/AiChatMessageRespVO.java b/yudao-module-ai/src/main/java/cn/iocoder/yudao/module/ai/controller/admin/chat/vo/message/AiChatMessageRespVO.java index 79d41d463d..b0ff0b8001 100644 --- a/yudao-module-ai/src/main/java/cn/iocoder/yudao/module/ai/controller/admin/chat/vo/message/AiChatMessageRespVO.java +++ b/yudao-module-ai/src/main/java/cn/iocoder/yudao/module/ai/controller/admin/chat/vo/message/AiChatMessageRespVO.java @@ -49,6 +49,9 @@ public class AiChatMessageRespVO { @Schema(description = "知识库段落数组") private List segments; + @Schema(description = "附件 URL 数组", example = "https://www.iocoder.cn/1.png") + private List attachmentUrls; + @Schema(description = "创建时间", requiredMode = Schema.RequiredMode.REQUIRED, example = "2024-05-12 12:51") private LocalDateTime createTime; diff --git a/yudao-module-ai/src/main/java/cn/iocoder/yudao/module/ai/controller/admin/chat/vo/message/AiChatMessageSendReqVO.java b/yudao-module-ai/src/main/java/cn/iocoder/yudao/module/ai/controller/admin/chat/vo/message/AiChatMessageSendReqVO.java index 89a84bcbd2..8ba3ebf1da 100644 --- a/yudao-module-ai/src/main/java/cn/iocoder/yudao/module/ai/controller/admin/chat/vo/message/AiChatMessageSendReqVO.java +++ b/yudao-module-ai/src/main/java/cn/iocoder/yudao/module/ai/controller/admin/chat/vo/message/AiChatMessageSendReqVO.java @@ -3,9 +3,9 @@ package cn.iocoder.yudao.module.ai.controller.admin.chat.vo.message; import io.swagger.v3.oas.annotations.media.Schema; import jakarta.validation.constraints.NotEmpty; import jakarta.validation.constraints.NotNull; -import jakarta.validation.constraints.Size; import lombok.Data; -import lombok.experimental.Accessors; + +import java.util.List; @Schema(description = "管理后台 - AI 聊天消息发送 Request VO") @Data @@ -22,4 +22,7 @@ public class AiChatMessageSendReqVO { @Schema(description = "是否携带上下文", example = "true") private Boolean useContext; + @Schema(description = "附件 URL 数组", example = "https://www.iocoder.cn/1.png") + private List attachmentUrls; + } diff --git a/yudao-module-ai/src/main/java/cn/iocoder/yudao/module/ai/dal/dataobject/chat/AiChatMessageDO.java b/yudao-module-ai/src/main/java/cn/iocoder/yudao/module/ai/dal/dataobject/chat/AiChatMessageDO.java index 60c413c470..7a339ee354 100644 --- a/yudao-module-ai/src/main/java/cn/iocoder/yudao/module/ai/dal/dataobject/chat/AiChatMessageDO.java +++ b/yudao-module-ai/src/main/java/cn/iocoder/yudao/module/ai/dal/dataobject/chat/AiChatMessageDO.java @@ -2,6 +2,7 @@ package cn.iocoder.yudao.module.ai.dal.dataobject.chat; import cn.iocoder.yudao.framework.mybatis.core.dataobject.BaseDO; import cn.iocoder.yudao.framework.mybatis.core.type.LongListTypeHandler; +import cn.iocoder.yudao.framework.mybatis.core.type.StringListTypeHandler; import cn.iocoder.yudao.module.ai.dal.dataobject.knowledge.AiKnowledgeSegmentDO; import cn.iocoder.yudao.module.ai.dal.dataobject.model.AiChatRoleDO; import cn.iocoder.yudao.module.ai.dal.dataobject.model.AiModelDO; @@ -105,4 +106,10 @@ public class AiChatMessageDO extends BaseDO { @TableField(typeHandler = LongListTypeHandler.class) private List segmentIds; + /** + * 附件 URL 数组 + */ + @TableField(typeHandler = StringListTypeHandler.class) + private List attachmentUrls; + } diff --git a/yudao-module-ai/src/main/java/cn/iocoder/yudao/module/ai/service/chat/AiChatMessageServiceImpl.java b/yudao-module-ai/src/main/java/cn/iocoder/yudao/module/ai/service/chat/AiChatMessageServiceImpl.java index adbd377efa..af970e28ef 100644 --- a/yudao-module-ai/src/main/java/cn/iocoder/yudao/module/ai/service/chat/AiChatMessageServiceImpl.java +++ b/yudao-module-ai/src/main/java/cn/iocoder/yudao/module/ai/service/chat/AiChatMessageServiceImpl.java @@ -1,8 +1,11 @@ package cn.iocoder.yudao.module.ai.service.chat; +import cn.hutool.core.codec.Base64; import cn.hutool.core.collection.CollUtil; +import cn.hutool.core.io.file.FileNameUtil; import cn.hutool.core.util.ObjUtil; import cn.hutool.core.util.StrUtil; +import cn.hutool.http.HttpUtil; import cn.iocoder.yudao.framework.common.pojo.CommonResult; import cn.iocoder.yudao.framework.common.pojo.PageResult; import cn.iocoder.yudao.framework.common.util.object.BeanUtils; @@ -28,6 +31,8 @@ import cn.iocoder.yudao.module.ai.service.model.AiChatRoleService; import cn.iocoder.yudao.module.ai.service.model.AiModelService; import cn.iocoder.yudao.module.ai.service.model.AiToolService; import cn.iocoder.yudao.module.ai.util.AiUtils; +import cn.iocoder.yudao.module.infra.framework.file.core.utils.FileTypeUtils; +import com.google.common.collect.Maps; import jakarta.annotation.Resource; import lombok.extern.slf4j.Slf4j; import org.springframework.ai.chat.messages.Message; @@ -64,6 +69,8 @@ import static cn.iocoder.yudao.module.ai.enums.ErrorCodeConstants.CHAT_MESSAGE_N @Slf4j public class AiChatMessageServiceImpl implements AiChatMessageService { + // TODO @芋艿：后续优化下对话的 Prompt 整体结构 + /** * 知识库转 {@link UserMessage} 的内容模版 */ @@ -71,6 +78,14 @@ public class AiChatMessageServiceImpl implements AiChatMessageService { "%s\n\n" + // 多个 的拼接 "回答要求：\n- 避免提及你是从 获取的知识。"; + /** + * 附件转 ${@link UserMessage} 的内容模版 + */ + @SuppressWarnings("TextBlockMigration") + private static final String Attachment_USER_MESSAGE_TEMPLATE = "使用 标记用户对话上传的附件内容:\n\n" + + "%s\n\n" + // 多个 的拼接 + "回答要求：\n- 避免提及 附件的编码格式。"; + @Resource private AiChatMessageMapper chatMessageMapper; @@ -101,17 +116,18 @@ public class AiChatMessageServiceImpl implements AiChatMessageService { ChatModel chatModel = modalService.getChatModel(model.getId()); // 2. 知识库找回 - List knowledgeSegments = recallKnowledgeSegment(sendReqVO.getContent(), conversation); + List knowledgeSegments = recallKnowledgeSegment( + sendReqVO.getContent(), conversation); // 3. 插入 user 发送消息 AiChatMessageDO userMessage = createChatMessage(conversation.getId(), null, model, userId, conversation.getRoleId(), MessageType.USER, sendReqVO.getContent(), sendReqVO.getUseContext(), - null); + null, sendReqVO.getAttachmentUrls()); // 3.1 插入 assistant 接收消息 AiChatMessageDO assistantMessage = createChatMessage(conversation.getId(), userMessage.getId(), model, userId, conversation.getRoleId(), MessageType.ASSISTANT, "", sendReqVO.getUseContext(), - knowledgeSegments); + knowledgeSegments, null); // 3.2 创建 chat 需要的 Prompt Prompt prompt = buildPrompt(conversation, historyMessages, knowledgeSegments, model, sendReqVO); @@ -151,18 +167,18 @@ public class AiChatMessageServiceImpl implements AiChatMessageService { StreamingChatModel chatModel = modalService.getChatModel(model.getId()); // 2. 知识库找回 - List knowledgeSegments = recallKnowledgeSegment(sendReqVO.getContent(), - conversation); + List knowledgeSegments = recallKnowledgeSegment( + sendReqVO.getContent(), conversation); // 3. 插入 user 发送消息 AiChatMessageDO userMessage = createChatMessage(conversation.getId(), null, model, userId, conversation.getRoleId(), MessageType.USER, sendReqVO.getContent(), sendReqVO.getUseContext(), - null); + null, sendReqVO.getAttachmentUrls()); // 4.1 插入 assistant 接收消息 AiChatMessageDO assistantMessage = createChatMessage(conversation.getId(), userMessage.getId(), model, userId, conversation.getRoleId(), MessageType.ASSISTANT, "", sendReqVO.getUseContext(), - knowledgeSegments); + knowledgeSegments, null); // 4.2 构建 Prompt，并进行调用 Prompt prompt = buildPrompt(conversation, historyMessages, knowledgeSegments, model, sendReqVO); @@ -243,8 +259,13 @@ public class AiChatMessageServiceImpl implements AiChatMessageService { // 1.2 历史 history message 历史消息 List contextMessages = filterContextMessages(messages, conversation, sendReqVO); - contextMessages - .forEach(message -> chatMessages.add(AiUtils.buildMessage(message.getType(), message.getContent()))); + contextMessages.forEach(message -> { + chatMessages.add(AiUtils.buildMessage(message.getType(), message.getContent())); + UserMessage attachmentUserMessage = buildAttachmentUserMessage(message.getAttachmentUrls()); + if (attachmentUserMessage != null) { + chatMessages.add(attachmentUserMessage); + } + }); // 1.3 当前 user message 新发送消息 chatMessages.add(new UserMessage(sendReqVO.getContent())); @@ -257,6 +278,14 @@ public class AiChatMessageServiceImpl implements AiChatMessageService { chatMessages.add(new UserMessage(String.format(KNOWLEDGE_USER_MESSAGE_TEMPLATE, reference))); } + // 1.5 附件，通过 UserMessage 实现 + if (CollUtil.isNotEmpty(sendReqVO.getAttachmentUrls())) { + UserMessage attachmentUserMessage = buildAttachmentUserMessage(sendReqVO.getAttachmentUrls()); + if (attachmentUserMessage != null) { + chatMessages.add(attachmentUserMessage); + } + } + // 2.1 查询 tool 工具 Set toolNames = null; Map toolContext = Map.of(); @@ -314,14 +343,52 @@ public class AiChatMessageServiceImpl implements AiChatMessageService { return contextMessages; } + private UserMessage buildAttachmentUserMessage(List attachmentUrls) { + if (CollUtil.isEmpty(attachmentUrls)) { + return null; + } + // 读取文件内容 + Map attachmentContents = Maps.newLinkedHashMapWithExpectedSize(attachmentUrls.size()); + for (String attachmentUrl : attachmentUrls) { + try { + String name = FileNameUtil.getName(attachmentUrl); + String mineType = FileTypeUtils.getMineType(name); + String content; + if (FileTypeUtils.isImage(mineType)) { + // 特殊：图片则转为 Base64 + byte[] bytes = HttpUtil.downloadBytes(attachmentUrl); + content = Base64.encode(bytes); + } else { + content = knowledgeDocumentService.readUrl(attachmentUrl); + } + if (StrUtil.isNotEmpty(content)) { + attachmentContents.put(name, content); + } + } catch (Exception e) { + log.error("[buildAttachmentUserMessage][读取附件({}) 发生异常]", attachmentUrl, e); + } + } + if (CollUtil.isEmpty(attachmentContents)) { + return null; + } + + // 拼接 UserMessage 消息 + String attachment = attachmentContents.entrySet().stream() + .map(entry -> "" + entry.getValue() + "") + .collect(Collectors.joining("\n\n")); + return new UserMessage(String.format(Attachment_USER_MESSAGE_TEMPLATE, attachment)); + } + private AiChatMessageDO createChatMessage(Long conversationId, Long replyId, - AiModelDO model, Long userId, Long roleId, - MessageType messageType, String content, Boolean useContext, - List knowledgeSegments) { + AiModelDO model, Long userId, Long roleId, + MessageType messageType, String content, Boolean useContext, + List knowledgeSegments, + List attachmentUrls) { AiChatMessageDO message = new AiChatMessageDO().setConversationId(conversationId).setReplyId(replyId) .setModel(model.getModel()).setModelId(model.getId()).setUserId(userId).setRoleId(roleId) .setType(messageType.getValue()).setContent(content).setUseContext(useContext) - .setSegmentIds(convertList(knowledgeSegments, AiKnowledgeSegmentSearchRespBO::getId)); + .setSegmentIds(convertList(knowledgeSegments, AiKnowledgeSegmentSearchRespBO::getId)) + .setAttachmentUrls(attachmentUrls); message.setCreateTime(LocalDateTime.now()); chatMessageMapper.insert(message); return message; diff --git a/yudao-module-infra/src/main/java/cn/iocoder/yudao/module/infra/framework/file/core/utils/FileTypeUtils.java b/yudao-module-infra/src/main/java/cn/iocoder/yudao/module/infra/framework/file/core/utils/FileTypeUtils.java index d8a13e9530..28d6a9fa16 100644 --- a/yudao-module-infra/src/main/java/cn/iocoder/yudao/module/infra/framework/file/core/utils/FileTypeUtils.java +++ b/yudao-module-infra/src/main/java/cn/iocoder/yudao/module/infra/framework/file/core/utils/FileTypeUtils.java @@ -80,17 +80,17 @@ public class FileTypeUtils { */ public static void writeAttachment(HttpServletResponse response, String filename, byte[] content) throws IOException { // 设置 header 和 contentType - String contentType = getMineType(content, filename); - response.setContentType(contentType); + String mineType = getMineType(content, filename); + response.setContentType(mineType); // 设置内容显示、下载文件名：https://www.cnblogs.com/wq-9/articles/12165056.html - if (StrUtil.containsIgnoreCase(contentType, "image/")) { + if (isImage(mineType)) { // 参见 https://github.com/YunaiV/ruoyi-vue-pro/issues/692 讨论 response.setHeader("Content-Disposition", "inline;filename=" + HttpUtils.encodeUtf8(filename)); } else { response.setHeader("Content-Disposition", "attachment;filename=" + HttpUtils.encodeUtf8(filename)); } // 针对 video 的特殊处理，解决视频地址在移动端播放的兼容性问题 - if (StrUtil.containsIgnoreCase(contentType, "video")) { + if (StrUtil.containsIgnoreCase(mineType, "video")) { response.setHeader("Content-Length", String.valueOf(content.length)); response.setHeader("Content-Range", "bytes 0-" + (content.length - 1) + "/" + content.length); response.setHeader("Accept-Ranges", "bytes"); @@ -99,4 +99,14 @@ public class FileTypeUtils { IoUtil.write(response.getOutputStream(), false, content); } + /** + * 判断是否是图片 + * + * @param mineType 类型 + * @return 是否是图片 + */ + public static boolean isImage(String mineType) { + return StrUtil.startWith(mineType, "image/"); + } + }