diff --git a/yudao-framework/yudao-spring-boot-starter-biz-tenant/src/main/java/cn/iocoder/yudao/framework/tenant/core/redis/TenantRedisCacheManager.java b/yudao-framework/yudao-spring-boot-starter-biz-tenant/src/main/java/cn/iocoder/yudao/framework/tenant/core/redis/TenantRedisCacheManager.java index aeea4b589c..86577388b5 100644 --- a/yudao-framework/yudao-spring-boot-starter-biz-tenant/src/main/java/cn/iocoder/yudao/framework/tenant/core/redis/TenantRedisCacheManager.java +++ b/yudao-framework/yudao-spring-boot-starter-biz-tenant/src/main/java/cn/iocoder/yudao/framework/tenant/core/redis/TenantRedisCacheManager.java @@ -1,6 +1,7 @@ package cn.iocoder.yudao.framework.tenant.core.redis; import cn.hutool.core.collection.CollUtil; +import cn.hutool.core.util.StrUtil; import cn.iocoder.yudao.framework.redis.core.TimeoutRedisCacheManager; import cn.iocoder.yudao.framework.tenant.core.context.TenantContextHolder; import lombok.extern.slf4j.Slf4j; @@ -21,6 +22,8 @@ import java.util.Set; @Slf4j public class TenantRedisCacheManager extends TimeoutRedisCacheManager { + private static final String SPLIT = "#"; + private final Set ignoreCaches; public TenantRedisCacheManager(RedisCacheWriter cacheWriter, @@ -32,10 +35,11 @@ public class TenantRedisCacheManager extends TimeoutRedisCacheManager { @Override public Cache getCache(String name) { + String[] names = StrUtil.splitToArray(name, SPLIT); // 如果开启多租户,则 name 拼接租户后缀 if (!TenantContextHolder.isIgnore() - && TenantContextHolder.getTenantId() != null - && !CollUtil.contains(ignoreCaches, name)) { + && TenantContextHolder.getTenantId() != null + && !CollUtil.contains(ignoreCaches, names[0])) { name = name + ":" + TenantContextHolder.getTenantId(); } @@ -43,4 +47,4 @@ public class TenantRedisCacheManager extends TimeoutRedisCacheManager { return super.getCache(name); } -} +} \ No newline at end of file diff --git a/yudao-module-ai/pom.xml b/yudao-module-ai/pom.xml index 65017da345..d295f72ff6 100644 --- a/yudao-module-ai/pom.xml +++ b/yudao-module-ai/pom.xml @@ -20,7 +20,8 @@ 1.1.0 - 1.1.0.0-M5 + + 1.1.0.0-RC1 1.2.6 @@ -226,6 +227,11 @@ com.agentsflex agents-flex-store-elasticsearch + + + com.agentsflex + agents-flex-search-engine-es + org.codehaus.groovy diff --git a/yudao-module-ai/src/main/java/cn/iocoder/yudao/module/ai/controller/admin/knowledge/vo/segment/AiKnowledgeSegmentPageReqVO.java b/yudao-module-ai/src/main/java/cn/iocoder/yudao/module/ai/controller/admin/knowledge/vo/segment/AiKnowledgeSegmentPageReqVO.java index f53d5be076..dd3b90300b 100644 --- a/yudao-module-ai/src/main/java/cn/iocoder/yudao/module/ai/controller/admin/knowledge/vo/segment/AiKnowledgeSegmentPageReqVO.java +++ b/yudao-module-ai/src/main/java/cn/iocoder/yudao/module/ai/controller/admin/knowledge/vo/segment/AiKnowledgeSegmentPageReqVO.java @@ -11,7 +11,7 @@ import lombok.Data; public class AiKnowledgeSegmentPageReqVO extends PageParam { @Schema(description = "文档编号", example = "1") - private Integer documentId; + private Long documentId; @Schema(description = "分段内容关键字", example = "Java 开发") private String content; diff --git a/yudao-module-ai/src/main/java/cn/iocoder/yudao/module/ai/enums/AiDocumentSplitStrategyEnum.java b/yudao-module-ai/src/main/java/cn/iocoder/yudao/module/ai/enums/AiDocumentSplitStrategyEnum.java new file mode 100644 index 0000000000..f0a9cd21d6 --- /dev/null +++ b/yudao-module-ai/src/main/java/cn/iocoder/yudao/module/ai/enums/AiDocumentSplitStrategyEnum.java @@ -0,0 +1,53 @@ +package cn.iocoder.yudao.module.ai.enums; + +import lombok.AllArgsConstructor; +import lombok.Getter; + +/** + * AI 知识库文档切片策略枚举 + * + * @author runzhen + */ +@AllArgsConstructor +@Getter +public enum AiDocumentSplitStrategyEnum { + + /** + * 自动识别文档类型并选择最佳切片策略 + */ + AUTO("auto", "自动识别"), + + /** + * 基于 Token 数量机械切分(默认策略) + */ + TOKEN("token", "Token 切分"), + + /** + * 按段落切分(以双换行符为分隔) + */ + PARAGRAPH("paragraph", "段落切分"), + + /** + * Markdown QA 格式专用切片器 + * 识别二级标题作为问题,保持问答对完整性 + * 长答案智能切分但保留问题作为上下文 + */ + MARKDOWN_QA("markdown_qa", "Markdown QA 切分"), + + /** + * 语义化切分,保留句子完整性 + * 在段落和句子边界处切分,避免截断 + */ + SEMANTIC("semantic", "语义切分"); + + /** + * 策略代码 + */ + private final String code; + + /** + * 策略名称 + */ + private final String name; + +} diff --git a/yudao-module-ai/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/AiKnowledgeSegmentServiceImpl.java b/yudao-module-ai/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/AiKnowledgeSegmentServiceImpl.java index 43c7e9cefe..9d64fcce9f 100644 --- a/yudao-module-ai/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/AiKnowledgeSegmentServiceImpl.java +++ b/yudao-module-ai/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/AiKnowledgeSegmentServiceImpl.java @@ -4,6 +4,7 @@ import cn.hutool.core.collection.CollUtil; import cn.hutool.core.collection.ListUtil; import cn.hutool.core.util.ObjUtil; import cn.hutool.core.util.StrUtil; + import cn.iocoder.yudao.framework.common.enums.CommonStatusEnum; import cn.iocoder.yudao.framework.common.pojo.PageResult; import cn.iocoder.yudao.framework.common.util.object.BeanUtils; @@ -15,8 +16,11 @@ import cn.iocoder.yudao.module.ai.dal.dataobject.knowledge.AiKnowledgeDO; import cn.iocoder.yudao.module.ai.dal.dataobject.knowledge.AiKnowledgeDocumentDO; import cn.iocoder.yudao.module.ai.dal.dataobject.knowledge.AiKnowledgeSegmentDO; import cn.iocoder.yudao.module.ai.dal.mysql.knowledge.AiKnowledgeSegmentMapper; +import cn.iocoder.yudao.module.ai.enums.AiDocumentSplitStrategyEnum; import cn.iocoder.yudao.module.ai.service.knowledge.bo.AiKnowledgeSegmentSearchReqBO; import cn.iocoder.yudao.module.ai.service.knowledge.bo.AiKnowledgeSegmentSearchRespBO; +import cn.iocoder.yudao.module.ai.service.knowledge.splitter.MarkdownQaSplitter; +import cn.iocoder.yudao.module.ai.service.knowledge.splitter.SemanticTextSplitter; import cn.iocoder.yudao.module.ai.service.model.AiModelService; import com.alibaba.cloud.ai.dashscope.rerank.DashScopeRerankOptions; import com.alibaba.cloud.ai.model.RerankModel; @@ -39,8 +43,7 @@ import java.util.*; import static cn.iocoder.yudao.framework.common.exception.util.ServiceExceptionUtil.exception; import static cn.iocoder.yudao.framework.common.util.collection.CollectionUtils.convertList; -import static cn.iocoder.yudao.module.ai.enums.ErrorCodeConstants.KNOWLEDGE_SEGMENT_CONTENT_TOO_LONG; -import static cn.iocoder.yudao.module.ai.enums.ErrorCodeConstants.KNOWLEDGE_SEGMENT_NOT_EXISTS; +import static cn.iocoder.yudao.module.ai.enums.ErrorCodeConstants.*; import static org.springframework.ai.vectorstore.SearchRequest.SIMILARITY_THRESHOLD_ACCEPT_ALL; /** @@ -95,8 +98,9 @@ public class AiKnowledgeSegmentServiceImpl implements AiKnowledgeSegmentService AiKnowledgeDO knowledgeDO = knowledgeService.validateKnowledgeExists(documentDO.getKnowledgeId()); VectorStore vectorStore = getVectorStoreById(knowledgeDO); - // 2. 文档切片 - List documentSegments = splitContentByToken(content, documentDO.getSegmentMaxTokens()); + // 2. 文档切片(使用自动检测策略) + List documentSegments = splitContentByStrategy(content, documentDO.getSegmentMaxTokens(), + AiDocumentSplitStrategyEnum.AUTO, documentDO.getUrl()); // 3.1 存储切片 List segmentDOs = convertList(documentSegments, segment -> { @@ -295,8 +299,10 @@ public class AiKnowledgeSegmentServiceImpl implements AiKnowledgeSegmentService // 1. 读取 URL 内容 String content = knowledgeDocumentService.readUrl(url); - // 2. 文档切片 - List documentSegments = splitContentByToken(content, segmentMaxTokens); + // 2.1 自动检测文档类型并选择策略 + AiDocumentSplitStrategyEnum strategy = detectDocumentStrategy(content, url); + // 2.2 文档切片 + List documentSegments = splitContentByStrategy(content, segmentMaxTokens, strategy, url); // 3. 转换为段落对象 return convertList(documentSegments, segment -> { @@ -333,11 +339,103 @@ public class AiKnowledgeSegmentServiceImpl implements AiKnowledgeSegmentService return getVectorStoreById(knowledge); } - private static List splitContentByToken(String content, Integer segmentMaxTokens) { - TextSplitter textSplitter = buildTokenTextSplitter(segmentMaxTokens); + /** + * 根据策略切分内容 + * + * @param content 文档内容 + * @param segmentMaxTokens 分段的最大 Token 数 + * @param strategy 切片策略 + * @param url 文档 URL(用于自动检测文件类型) + * @return 切片后的文档列表 + */ + @SuppressWarnings("EnhancedSwitchMigration") + private List splitContentByStrategy(String content, Integer segmentMaxTokens, + AiDocumentSplitStrategyEnum strategy, String url) { + // 自动检测策略 + if (strategy == AiDocumentSplitStrategyEnum.AUTO) { + strategy = detectDocumentStrategy(content, url); + log.info("[splitContentByStrategy][自动检测到文档策略: {}]", strategy.getName()); + } + // 根据策略切分 + TextSplitter textSplitter; + switch (strategy) { + case MARKDOWN_QA: + textSplitter = new MarkdownQaSplitter(segmentMaxTokens); + break; + case SEMANTIC: + textSplitter = new SemanticTextSplitter(segmentMaxTokens); + break; + case PARAGRAPH: + textSplitter = new SemanticTextSplitter(segmentMaxTokens, 0); // 段落切分,无重叠 + break; + case TOKEN: + default: + textSplitter = buildTokenTextSplitter(segmentMaxTokens); + break; + } + // 执行切分 return textSplitter.apply(Collections.singletonList(new Document(content))); } + /** + * 自动检测文档类型并选择切片策略 + * + * @param content 文档内容 + * @param url 文档 URL + * @return 推荐的切片策略 + */ + private AiDocumentSplitStrategyEnum detectDocumentStrategy(String content, String url) { + if (StrUtil.isEmpty(content)) { + return AiDocumentSplitStrategyEnum.TOKEN; + } + // 1. 检测 Markdown QA 格式 + if (isMarkdownQaFormat(content, url)) { + return AiDocumentSplitStrategyEnum.MARKDOWN_QA; + } + // 2. 检测普通 Markdown 文档 + if (isMarkdownDocument(url)) { + return AiDocumentSplitStrategyEnum.SEMANTIC; + } + // 3. 默认使用语义切分(比 Token 切分更智能) + return AiDocumentSplitStrategyEnum.SEMANTIC; + } + + /** + * 检测是否为 Markdown QA 格式 + * 特征:包含多个二级标题(## )且标题后紧跟答案内容 + */ + private boolean isMarkdownQaFormat(String content, String url) { + // 文件扩展名判断 + if (StrUtil.isNotEmpty(url) && !url.toLowerCase().endsWith(".md")) { + return false; + } + + // 统计二级标题数量 + long h2Count = content.lines() + .filter(line -> line.trim().startsWith("## ")) + .count(); + + // 要求一:至少包含 2 个二级标题才认为是 QA 格式 + if (h2Count < 2) { + return false; + } + + // 要求二:检查标题占比(QA 文档标题行数相对较多),如果二级标题占比超过 10%,认为是 QA 格式 + long totalLines = content.lines().count(); + double h2Ratio = (double) h2Count / totalLines; + return h2Ratio > 0.1; + } + + /** + * 检测是否为 Markdown 文档 + */ + private boolean isMarkdownDocument(String url) { + return StrUtil.endWithAnyIgnoreCase(url, ".md", ".markdown"); + } + + /** + * 构建基于 Token 的文本切片器(原有逻辑保留) + */ private static TextSplitter buildTokenTextSplitter(Integer segmentMaxTokens) { return TokenTextSplitter.builder() .withChunkSize(segmentMaxTokens) diff --git a/yudao-module-ai/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/splitter/MarkdownQaSplitter.java b/yudao-module-ai/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/splitter/MarkdownQaSplitter.java new file mode 100644 index 0000000000..1fbf4f2429 --- /dev/null +++ b/yudao-module-ai/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/splitter/MarkdownQaSplitter.java @@ -0,0 +1,342 @@ +package cn.iocoder.yudao.module.ai.service.knowledge.splitter; + +import cn.hutool.core.collection.CollUtil; +import cn.hutool.core.util.StrUtil; +import lombok.AllArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.ai.transformer.splitter.TextSplitter; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** + * Markdown QA 格式专用切片器 + * + *

功能特点: + *

    + *
  • 识别二级标题(## )作为问题标记
  • + *
  • 短 QA 对保持完整(不超过 Token 限制)
  • + *
  • 长答案智能切分,每个片段保留完整问题作为上下文
  • + *
  • 支持自定义 Token 估算器
  • + *
+ * + * @author runzhen + */ +@Slf4j +@SuppressWarnings("SizeReplaceableByIsEmpty") +public class MarkdownQaSplitter extends TextSplitter { + + /** + * 二级标题正则:匹配 "## " 开头的行 + */ + private static final Pattern H2_PATTERN = Pattern.compile("^##\\s+(.+)$", Pattern.MULTILINE); + + /** + * 段落分隔符:双换行 + */ + private static final String PARAGRAPH_SEPARATOR = "\n\n"; + + /** + * 句子分隔符 + */ + private static final Pattern SENTENCE_PATTERN = Pattern.compile("[。!?.!?]\\s*"); + + /** + * 分段的最大 Token 数 + */ + private final int chunkSize; + + /** + * Token 估算器(简单实现:中文按字符数,英文按单词数的 1.3 倍) + */ + private final TokenEstimator tokenEstimator; + + public MarkdownQaSplitter(int chunkSize) { + this.chunkSize = chunkSize; + this.tokenEstimator = new SimpleTokenEstimator(); + } + + @Override + protected List splitText(String text) { + if (StrUtil.isEmpty(text)) { + return Collections.emptyList(); + } + + // 解析 QA 对 + List qaPairs = parseQaPairs(text); + if (CollUtil.isEmpty(qaPairs)) { + // 如果没有识别到 QA 格式,按段落切分 + return fallbackSplit(text); + } + + // 处理每个 QA 对 + List result = new ArrayList<>(); + for (QaPair qaPair : qaPairs) { + result.addAll(splitQaPair(qaPair)); + } + return result; + } + + /** + * 解析 Markdown QA 对 + * + * @param content 文本内容 + * @return QA 对列表 + */ + private List parseQaPairs(String content) { + // 找到所有二级标题位置 + List qaPairs = new ArrayList<>(); + List headingPositions = new ArrayList<>(); + List questions = new ArrayList<>(); + Matcher matcher = H2_PATTERN.matcher(content); + while (matcher.find()) { + headingPositions.add(matcher.start()); + questions.add(matcher.group(1).trim()); + } + if (CollUtil.isEmpty(headingPositions)) { + return qaPairs; + } + + // 提取每个 QA 对 + for (int i = 0; i < headingPositions.size(); i++) { + int start = headingPositions.get(i); + int end = (i + 1 < headingPositions.size()) + ? headingPositions.get(i + 1) + : content.length(); + String qaText = content.substring(start, end).trim(); + String question = questions.get(i); + // 提取答案部分(去掉问题标题) + String answer = qaText.substring(qaText.indexOf('\n') + 1).trim(); + qaPairs.add(new QaPair(question, answer, qaText)); + } + return qaPairs; + } + + /** + * 切分单个 QA 对 + * + * @param qaPair QA 对 + * @return 切分后的文本片段列表 + */ + private List splitQaPair(QaPair qaPair) { + // 如果整个 QA 对不超过限制,保持完整 + List chunks = new ArrayList<>(); + String fullQa = qaPair.fullText; + int qaTokens = tokenEstimator.estimate(fullQa); + if (qaTokens <= chunkSize) { + chunks.add(fullQa); + return chunks; + } + + // 长答案需要切分 + log.debug("QA 对超过 Token 限制 ({} > {}),开始智能切分: {}", qaTokens, chunkSize, qaPair.question); + List answerChunks = splitLongAnswer(qaPair.answer, qaPair.question); + for (String answerChunk : answerChunks) { + // 每个片段都包含完整问题 + String chunkText = "## " + qaPair.question + "\n" + answerChunk; + chunks.add(chunkText); + } + return chunks; + } + + /** + * 切分长答案 + * + * @param answer 答案文本 + * @param question 问题文本 + * @return 切分后的答案片段列表 + */ + private List splitLongAnswer(String answer, String question) { + List chunks = new ArrayList<>(); + // 预留问题的 Token 空间 + String questionHeader = "## " + question + "\n"; + int questionTokens = tokenEstimator.estimate(questionHeader); + int availableTokens = chunkSize - questionTokens - 10; // 预留 10 个 Token 的缓冲 + + // 先按段落切分 + String[] paragraphs = answer.split(PARAGRAPH_SEPARATOR); + StringBuilder currentChunk = new StringBuilder(); + int currentTokens = 0; + for (String paragraph : paragraphs) { + if (StrUtil.isEmpty(paragraph)) { + continue; + } + int paragraphTokens = tokenEstimator.estimate(paragraph); + // 如果单个段落就超过限制,需要按句子切分 + if (paragraphTokens > availableTokens) { + // 先保存当前块 + if (currentChunk.length() > 0) { + chunks.add(currentChunk.toString().trim()); + currentChunk = new StringBuilder(); + currentTokens = 0; + } + // 按句子切分长段落 + chunks.addAll(splitLongParagraph(paragraph, availableTokens)); + continue; + } + // 如果加上这个段落会超过限制 + if (currentTokens + paragraphTokens > availableTokens && currentChunk.length() > 0) { + chunks.add(currentChunk.toString().trim()); + currentChunk = new StringBuilder(); + currentTokens = 0; + } + if (currentChunk.length() > 0) { + currentChunk.append("\n\n"); + } + // 添加段落 + currentChunk.append(paragraph); + currentTokens += paragraphTokens; + } + + // 添加最后一块 + if (currentChunk.length() > 0) { + chunks.add(currentChunk.toString().trim()); + } + return CollUtil.isEmpty(chunks) ? Collections.singletonList(answer) : chunks; + } + + /** + * 切分长段落(按句子) + * + * @param paragraph 段落文本 + * @param availableTokens 可用的 Token 数 + * @return 切分后的文本片段列表 + */ + private List splitLongParagraph(String paragraph, int availableTokens) { + // 按句子切分 + List chunks = new ArrayList<>(); + String[] sentences = SENTENCE_PATTERN.split(paragraph); + + // 按句子累积切分 + StringBuilder currentChunk = new StringBuilder(); + int currentTokens = 0; + for (String sentence : sentences) { + if (StrUtil.isEmpty(sentence)) { + continue; + } + int sentenceTokens = tokenEstimator.estimate(sentence); + // 如果单个句子就超过限制,强制切分 + if (sentenceTokens > availableTokens) { + if (currentChunk.length() > 0) { + chunks.add(currentChunk.toString().trim()); + currentChunk = new StringBuilder(); + currentTokens = 0; + } + chunks.add(sentence.trim()); + continue; + } + // 如果加上这个句子会超过限制 + if (currentTokens + sentenceTokens > availableTokens && currentChunk.length() > 0) { + chunks.add(currentChunk.toString().trim()); + currentChunk = new StringBuilder(); + currentTokens = 0; + } + // 添加句子 + currentChunk.append(sentence); + currentTokens += sentenceTokens; + } + + // 添加最后一块 + if (currentChunk.length() > 0) { + chunks.add(currentChunk.toString().trim()); + } + return chunks.isEmpty() ? Collections.singletonList(paragraph) : chunks; + } + + /** + * 降级切分策略(当未识别到 QA 格式时) + * + * @param content 文本内容 + * @return 切分后的文本片段列表 + */ + private List fallbackSplit(String content) { + // 按段落切分 + List chunks = new ArrayList<>(); + String[] paragraphs = content.split(PARAGRAPH_SEPARATOR); + + // 按段落累积切分 + StringBuilder currentChunk = new StringBuilder(); + int currentTokens = 0; + for (String paragraph : paragraphs) { + if (StrUtil.isEmpty(paragraph)) { + continue; + } + int paragraphTokens = tokenEstimator.estimate(paragraph); + // 如果加上这个段落会超过限制 + if (currentTokens + paragraphTokens > chunkSize && currentChunk.length() > 0) { + chunks.add(currentChunk.toString().trim()); + currentChunk = new StringBuilder(); + currentTokens = 0; + } + // 添加段落 + if (currentChunk.length() > 0) { + currentChunk.append("\n\n"); + } + currentChunk.append(paragraph); + currentTokens += paragraphTokens; + } + + // 添加最后一块 + if (currentChunk.length() > 0) { + chunks.add(currentChunk.toString().trim()); + } + return chunks.isEmpty() ? Collections.singletonList(content) : chunks; + } + + /** + * QA 对数据结构 + */ + @AllArgsConstructor + private static class QaPair { + + String question; + String answer; + String fullText; + + } + + /** + * Token 估算器接口 + */ + public interface TokenEstimator { + + int estimate(String text); + + } + + /** + * 简单的 Token 估算器实现 + * 中文:1 字符 ≈ 1 Token + * 英文:1 单词 ≈ 1.3 Token + */ + private static class SimpleTokenEstimator implements TokenEstimator { + + @Override + public int estimate(String text) { + if (StrUtil.isEmpty(text)) { + return 0; + } + + int chineseChars = 0; + int englishWords = 0; + // 简单统计中英文 + for (char c : text.toCharArray()) { + if (c >= 0x4E00 && c <= 0x9FA5) { + chineseChars++; + } + } + // 英文单词估算 + String[] words = text.split("\\s+"); + for (String word : words) { + if (word.matches(".*[a-zA-Z].*")) { + englishWords++; + } + } + return chineseChars + (int) (englishWords * 1.3); + } + } + +} diff --git a/yudao-module-ai/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/splitter/SemanticTextSplitter.java b/yudao-module-ai/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/splitter/SemanticTextSplitter.java new file mode 100644 index 0000000000..4c7112e9ad --- /dev/null +++ b/yudao-module-ai/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/splitter/SemanticTextSplitter.java @@ -0,0 +1,301 @@ +package cn.iocoder.yudao.module.ai.service.knowledge.splitter; + +import cn.hutool.core.util.StrUtil; +import lombok.extern.slf4j.Slf4j; +import org.springframework.ai.transformer.splitter.TextSplitter; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** + * 语义化文本切片器 + * + *

功能特点: + *

    + *
  • 优先在段落边界(双换行)处切分
  • + *
  • 其次在句子边界(句号、问号、感叹号)处切分
  • + *
  • 避免在句子中间截断,保持语义完整性
  • + *
  • 支持中英文标点符号识别
  • + *
+ * + * @author runzhen + */ +@Slf4j +public class SemanticTextSplitter extends TextSplitter { + + /** + * 分段的最大 Token 数 + */ + private final int chunkSize; + + /** + * 段落重叠大小(用于保持上下文连贯性) + */ + private final int chunkOverlap; + + /** + * 段落分隔符(按优先级排序) + */ + private static final List PARAGRAPH_SEPARATORS = Arrays.asList( + "\n\n\n", // 三个换行 + "\n\n", // 双换行 + "\n" // 单换行 + ); + + /** + * 句子结束标记(中英文标点) + */ + private static final Pattern SENTENCE_END_PATTERN = Pattern.compile( + "[。!?.!?]+[\\s\"'))】\\]]*" + ); + + /** + * Token 估算器 + */ + private final MarkdownQaSplitter.TokenEstimator tokenEstimator; + + public SemanticTextSplitter(int chunkSize, int chunkOverlap) { + this.chunkSize = chunkSize; + this.chunkOverlap = Math.min(chunkOverlap, chunkSize / 2); // 重叠不超过一半 + this.tokenEstimator = new SimpleTokenEstimator(); + } + + public SemanticTextSplitter(int chunkSize) { + this(chunkSize, 50); // 默认重叠 50 个 Token + } + + @Override + protected List splitText(String text) { + if (StrUtil.isEmpty(text)) { + return Collections.emptyList(); + } + return splitTextRecursive(text); + } + + /** + * 切分文本(递归策略) + * + * @param text 待切分文本 + * @return 切分后的文本块列表 + */ + private List splitTextRecursive(String text) { + List chunks = new ArrayList<>(); + + // 如果文本不超过限制,直接返回 + int textTokens = tokenEstimator.estimate(text); + if (textTokens <= chunkSize) { + chunks.add(text.trim()); + return chunks; + } + + // 尝试按不同分隔符切分 + List splits = null; + String usedSeparator = null; + for (String separator : PARAGRAPH_SEPARATORS) { + if (text.contains(separator)) { + splits = Arrays.asList(text.split(Pattern.quote(separator))); + usedSeparator = separator; + break; + } + } + + // 如果没有找到段落分隔符,按句子切分 + if (splits == null || splits.size() == 1) { + splits = splitBySentences(text); + usedSeparator = ""; // 句子切分不需要分隔符 + } + + // 合并小片段 + chunks = mergeSplits(splits, usedSeparator); + return chunks; + } + + /** + * 按句子切分 + * + * @param text 待切分文本 + * @return 句子列表 + */ + private List splitBySentences(String text) { + // 使用正则表达式匹配句子结束位置 + List sentences = new ArrayList<>(); + int lastEnd = 0; + Matcher matcher = SENTENCE_END_PATTERN.matcher(text); + while (matcher.find()) { + String sentence = text.substring(lastEnd, matcher.end()).trim(); + if (StrUtil.isNotEmpty(sentence)) { + sentences.add(sentence); + } + lastEnd = matcher.end(); + } + + // 添加剩余部分 + if (lastEnd < text.length()) { + String remaining = text.substring(lastEnd).trim(); + if (StrUtil.isNotEmpty(remaining)) { + sentences.add(remaining); + } + } + return sentences.isEmpty() ? Collections.singletonList(text) : sentences; + } + + /** + * 合并切分后的小片段 + * + * @param splits 切分后的片段列表 + * @param separator 片段间的分隔符 + * @return 合并后的文本块列表 + */ + private List mergeSplits(List splits, String separator) { + List chunks = new ArrayList<>(); + List currentChunks = new ArrayList<>(); + int currentLength = 0; + + for (String split : splits) { + if (StrUtil.isEmpty(split)) { + continue; + } + int splitTokens = tokenEstimator.estimate(split); + // 如果单个片段就超过限制,进一步递归切分 + if (splitTokens > chunkSize) { + // 先保存当前累积的块 + if (!currentChunks.isEmpty()) { + String chunkText = String.join(separator, currentChunks); + chunks.add(chunkText.trim()); + currentChunks.clear(); + currentLength = 0; + } + // 递归切分大片段 + if (!separator.isEmpty()) { + // 如果是段落分隔符,尝试按句子切分 + chunks.addAll(splitTextRecursive(split)); + } else { + // 如果已经是句子级别,强制按字符切分 + chunks.addAll(forceSplitLongText(split)); + } + continue; + } + // 计算加上分隔符的 Token 数 + int separatorTokens = StrUtil.isEmpty(separator) ? 0 : tokenEstimator.estimate(separator); + // 如果加上这个片段会超过限制 + if (!currentChunks.isEmpty() && currentLength + splitTokens + separatorTokens > chunkSize) { + // 保存当前块 + String chunkText = String.join(separator, currentChunks); + chunks.add(chunkText.trim()); + + // 处理重叠:保留最后几个片段 + currentChunks = getOverlappingChunks(currentChunks, separator); + currentLength = estimateTokens(currentChunks, separator); + } + // 添加当前片段 + currentChunks.add(split); + currentLength += splitTokens + separatorTokens; + } + + // 添加最后一块 + if (!currentChunks.isEmpty()) { + String chunkText = String.join(separator, currentChunks); + chunks.add(chunkText.trim()); + } + return chunks; + } + + /** + * 获取重叠的片段(用于保持上下文) + * + * @param chunks 当前片段列表 + * @param separator 片段间的分隔符 + * @return 重叠的片段列表 + */ + private List getOverlappingChunks(List chunks, String separator) { + if (chunkOverlap == 0 || chunks.isEmpty()) { + return new ArrayList<>(); + } + + // 从后往前取片段,直到达到重叠大小 + List overlapping = new ArrayList<>(); + int tokens = 0; + for (int i = chunks.size() - 1; i >= 0; i--) { + String chunk = chunks.get(i); + int chunkTokens = tokenEstimator.estimate(chunk); + if (tokens + chunkTokens > chunkOverlap) { + break; + } + // 添加到重叠列表前端 + overlapping.add(0, chunk); + tokens += chunkTokens + (StrUtil.isEmpty(separator) ? 0 : tokenEstimator.estimate(separator)); + } + return overlapping; + } + + /** + * 估算片段列表的总 Token 数 + * + * @param chunks 片段列表 + * @param separator 片段间的分隔符 + * @return 总 Token 数 + */ + private int estimateTokens(List chunks, String separator) { + int total = 0; + for (int i = 0; i < chunks.size(); i++) { + total += tokenEstimator.estimate(chunks.get(i)); + if (i < chunks.size() - 1 && StrUtil.isNotEmpty(separator)) { + total += tokenEstimator.estimate(separator); + } + } + return total; + } + + /** + * 强制切分长文本(当语义切分失败时) + * + * @param text 待切分文本 + * @return 切分后的文本块列表 + */ + private List forceSplitLongText(String text) { + List chunks = new ArrayList<>(); + int charsPerChunk = (int) (chunkSize * 0.8); // 保守估计 + for (int i = 0; i < text.length(); i += charsPerChunk) { + int end = Math.min(i + charsPerChunk, text.length()); + String chunk = text.substring(i, end); + chunks.add(chunk.trim()); + } + log.warn("文本过长,已强制按字符切分,可能影响语义完整性"); + return chunks; + } + + /** + * 简单的 Token 估算器实现 + */ + private static class SimpleTokenEstimator implements MarkdownQaSplitter.TokenEstimator { + + @Override + public int estimate(String text) { + if (StrUtil.isEmpty(text)) { + return 0; + } + + int chineseChars = 0; + int englishWords = 0; + // 简单统计中英文 + for (char c : text.toCharArray()) { + if (c >= 0x4E00 && c <= 0x9FA5) { + chineseChars++; + } + } + // 英文单词估算 + String[] words = text.split("\\s+"); + for (String word : words) { + if (word.matches(".*[a-zA-Z].*")) { + englishWords++; + } + } + return chineseChars + (int) (englishWords * 1.3); + } + } + +} diff --git a/yudao-module-infra/src/main/java/cn/iocoder/yudao/module/infra/controller/admin/file/FileController.java b/yudao-module-infra/src/main/java/cn/iocoder/yudao/module/infra/controller/admin/file/FileController.java index 2789242c27..8081aa04a3 100644 --- a/yudao-module-infra/src/main/java/cn/iocoder/yudao/module/infra/controller/admin/file/FileController.java +++ b/yudao-module-infra/src/main/java/cn/iocoder/yudao/module/infra/controller/admin/file/FileController.java @@ -13,6 +13,7 @@ import cn.iocoder.yudao.module.infra.service.file.FileService; import io.swagger.v3.oas.annotations.Operation; import io.swagger.v3.oas.annotations.Parameter; import io.swagger.v3.oas.annotations.Parameters; +import io.swagger.v3.oas.annotations.media.Schema; import io.swagger.v3.oas.annotations.tags.Tag; import lombok.extern.slf4j.Slf4j; import org.springframework.http.HttpStatus; @@ -44,6 +45,8 @@ public class FileController { @PostMapping("/upload") @Operation(summary = "上传文件", description = "模式一:后端上传文件") + @Parameter(name = "file", description = "文件附件", required = true, + schema = @Schema(type = "string", format = "binary")) public CommonResult uploadFile(@Valid FileUploadReqVO uploadReqVO) throws Exception { MultipartFile file = uploadReqVO.getFile(); byte[] content = IoUtil.readBytes(file.getInputStream()); diff --git a/yudao-module-infra/src/main/java/cn/iocoder/yudao/module/infra/controller/admin/file/vo/file/FileUploadReqVO.java b/yudao-module-infra/src/main/java/cn/iocoder/yudao/module/infra/controller/admin/file/vo/file/FileUploadReqVO.java index 3ffd6f658b..be23401d18 100644 --- a/yudao-module-infra/src/main/java/cn/iocoder/yudao/module/infra/controller/admin/file/vo/file/FileUploadReqVO.java +++ b/yudao-module-infra/src/main/java/cn/iocoder/yudao/module/infra/controller/admin/file/vo/file/FileUploadReqVO.java @@ -23,7 +23,14 @@ public class FileUploadReqVO { @AssertTrue(message = "文件目录不正确") @JsonIgnore public boolean isDirectoryValid() { - return !StrUtil.containsAny(directory, "..", "/", "\\"); + return isDirectoryValid(directory); + } + + public static boolean isDirectoryValid(String directory) { + // 1. 不能包含 .. 防止目录穿越 + // 2. 不能以 / 或 \ 开头,防止上传到根目录 + return !StrUtil.contains(directory, "..") + && !StrUtil.startWithAny(directory, "/", "\\"); } } diff --git a/yudao-module-infra/src/main/java/cn/iocoder/yudao/module/infra/controller/admin/job/vo/log/JobLogRespVO.java b/yudao-module-infra/src/main/java/cn/iocoder/yudao/module/infra/controller/admin/job/vo/log/JobLogRespVO.java index 3574e98d59..1d33cdf278 100644 --- a/yudao-module-infra/src/main/java/cn/iocoder/yudao/module/infra/controller/admin/job/vo/log/JobLogRespVO.java +++ b/yudao-module-infra/src/main/java/cn/iocoder/yudao/module/infra/controller/admin/job/vo/log/JobLogRespVO.java @@ -49,7 +49,7 @@ public class JobLogRespVO { @Schema(description = "任务状态,参见 JobLogStatusEnum 枚举", requiredMode = Schema.RequiredMode.REQUIRED, example = "1") @ExcelProperty(value = "任务状态", converter = DictConvert.class) - @DictFormat(DictTypeConstants.JOB_STATUS) + @DictFormat(DictTypeConstants.JOB_LOG_STATUS) private Integer status; @Schema(description = "结果数据", example = "执行成功") diff --git a/yudao-module-infra/src/main/java/cn/iocoder/yudao/module/infra/controller/admin/logger/ApiAccessLogController.java b/yudao-module-infra/src/main/java/cn/iocoder/yudao/module/infra/controller/admin/logger/ApiAccessLogController.java index 81b0ee9d84..db86fed4f5 100644 --- a/yudao-module-infra/src/main/java/cn/iocoder/yudao/module/infra/controller/admin/logger/ApiAccessLogController.java +++ b/yudao-module-infra/src/main/java/cn/iocoder/yudao/module/infra/controller/admin/logger/ApiAccessLogController.java @@ -11,11 +11,13 @@ import cn.iocoder.yudao.module.infra.controller.admin.logger.vo.apiaccesslog.Api import cn.iocoder.yudao.module.infra.dal.dataobject.logger.ApiAccessLogDO; import cn.iocoder.yudao.module.infra.service.logger.ApiAccessLogService; import io.swagger.v3.oas.annotations.Operation; +import io.swagger.v3.oas.annotations.Parameter; import io.swagger.v3.oas.annotations.tags.Tag; import org.springframework.security.access.prepost.PreAuthorize; import org.springframework.validation.annotation.Validated; import org.springframework.web.bind.annotation.GetMapping; import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestParam; import org.springframework.web.bind.annotation.RestController; import javax.annotation.Resource; @@ -36,6 +38,15 @@ public class ApiAccessLogController { @Resource private ApiAccessLogService apiAccessLogService; + @GetMapping("/get") + @Operation(summary = "获得 API 访问日志") + @Parameter(name = "id", description = "编号", required = true, example = "1024") + @PreAuthorize("@ss.hasPermission('infra:api-access-log:query')") + public CommonResult getApiAccessLog(@RequestParam("id") Long id) { + ApiAccessLogDO apiAccessLog = apiAccessLogService.getApiAccessLog(id); + return success(BeanUtils.toBean(apiAccessLog, ApiAccessLogRespVO.class)); + } + @GetMapping("/page") @Operation(summary = "获得API 访问日志分页") @PreAuthorize("@ss.hasPermission('infra:api-access-log:query')") diff --git a/yudao-module-infra/src/main/java/cn/iocoder/yudao/module/infra/controller/admin/logger/ApiErrorLogController.java b/yudao-module-infra/src/main/java/cn/iocoder/yudao/module/infra/controller/admin/logger/ApiErrorLogController.java index 50e22df61c..05945f5c28 100644 --- a/yudao-module-infra/src/main/java/cn/iocoder/yudao/module/infra/controller/admin/logger/ApiErrorLogController.java +++ b/yudao-module-infra/src/main/java/cn/iocoder/yudao/module/infra/controller/admin/logger/ApiErrorLogController.java @@ -50,6 +50,15 @@ public class ApiErrorLogController { return success(true); } + @GetMapping("/get") + @Operation(summary = "获得 API 错误日志") + @Parameter(name = "id", description = "编号", required = true, example = "1024") + @PreAuthorize("@ss.hasPermission('infra:api-error-log:query')") + public CommonResult getApiErrorLog(@RequestParam("id") Long id) { + ApiErrorLogDO apiErrorLog = apiErrorLogService.getApiErrorLog(id); + return success(BeanUtils.toBean(apiErrorLog, ApiErrorLogRespVO.class)); + } + @GetMapping("/page") @Operation(summary = "获得 API 错误日志分页") @PreAuthorize("@ss.hasPermission('infra:api-error-log:query')") diff --git a/yudao-module-infra/src/main/java/cn/iocoder/yudao/module/infra/controller/app/file/AppFileController.java b/yudao-module-infra/src/main/java/cn/iocoder/yudao/module/infra/controller/app/file/AppFileController.java index 7e354fa05b..5805a0fcd0 100644 --- a/yudao-module-infra/src/main/java/cn/iocoder/yudao/module/infra/controller/app/file/AppFileController.java +++ b/yudao-module-infra/src/main/java/cn/iocoder/yudao/module/infra/controller/app/file/AppFileController.java @@ -9,6 +9,7 @@ import cn.iocoder.yudao.module.infra.service.file.FileService; import io.swagger.v3.oas.annotations.Operation; import io.swagger.v3.oas.annotations.Parameter; import io.swagger.v3.oas.annotations.Parameters; +import io.swagger.v3.oas.annotations.media.Schema; import io.swagger.v3.oas.annotations.tags.Tag; import lombok.extern.slf4j.Slf4j; import org.springframework.validation.annotation.Validated; @@ -33,6 +34,8 @@ public class AppFileController { @PostMapping("/upload") @Operation(summary = "上传文件") + @Parameter(name = "file", description = "文件附件", required = true, + schema = @Schema(type = "string", format = "binary")) @PermitAll public CommonResult uploadFile(AppFileUploadReqVO uploadReqVO) throws Exception { MultipartFile file = uploadReqVO.getFile(); diff --git a/yudao-module-infra/src/main/java/cn/iocoder/yudao/module/infra/controller/app/file/vo/AppFileUploadReqVO.java b/yudao-module-infra/src/main/java/cn/iocoder/yudao/module/infra/controller/app/file/vo/AppFileUploadReqVO.java index ec7764e4ca..349b444628 100644 --- a/yudao-module-infra/src/main/java/cn/iocoder/yudao/module/infra/controller/app/file/vo/AppFileUploadReqVO.java +++ b/yudao-module-infra/src/main/java/cn/iocoder/yudao/module/infra/controller/app/file/vo/AppFileUploadReqVO.java @@ -1,6 +1,6 @@ package cn.iocoder.yudao.module.infra.controller.app.file.vo; -import cn.hutool.core.util.StrUtil; +import cn.iocoder.yudao.module.infra.controller.admin.file.vo.file.FileUploadReqVO; import com.fasterxml.jackson.annotation.JsonIgnore; import io.swagger.v3.oas.annotations.media.Schema; import lombok.Data; @@ -23,7 +23,7 @@ public class AppFileUploadReqVO { @AssertTrue(message = "文件目录不正确") @JsonIgnore public boolean isDirectoryValid() { - return !StrUtil.containsAny(directory, "..", "/", "\\"); + return FileUploadReqVO.isDirectoryValid(directory); } } diff --git a/yudao-module-infra/src/main/java/cn/iocoder/yudao/module/infra/service/logger/ApiAccessLogService.java b/yudao-module-infra/src/main/java/cn/iocoder/yudao/module/infra/service/logger/ApiAccessLogService.java index 65faa3f333..cc4cda5c52 100644 --- a/yudao-module-infra/src/main/java/cn/iocoder/yudao/module/infra/service/logger/ApiAccessLogService.java +++ b/yudao-module-infra/src/main/java/cn/iocoder/yudao/module/infra/service/logger/ApiAccessLogService.java @@ -19,6 +19,14 @@ public interface ApiAccessLogService { */ void createApiAccessLog(ApiAccessLogCreateReqDTO createReqDTO); + /** + * 获得 API 访问日志 + * + * @param id 编号 + * @return API 访问日志 + */ + ApiAccessLogDO getApiAccessLog(Long id); + /** * 获得 API 访问日志分页 * diff --git a/yudao-module-infra/src/main/java/cn/iocoder/yudao/module/infra/service/logger/ApiAccessLogServiceImpl.java b/yudao-module-infra/src/main/java/cn/iocoder/yudao/module/infra/service/logger/ApiAccessLogServiceImpl.java index 84ceebdbaa..396068f345 100644 --- a/yudao-module-infra/src/main/java/cn/iocoder/yudao/module/infra/service/logger/ApiAccessLogServiceImpl.java +++ b/yudao-module-infra/src/main/java/cn/iocoder/yudao/module/infra/service/logger/ApiAccessLogServiceImpl.java @@ -45,6 +45,11 @@ public class ApiAccessLogServiceImpl implements ApiAccessLogService { } } + @Override + public ApiAccessLogDO getApiAccessLog(Long id) { + return apiAccessLogMapper.selectById(id); + } + @Override public PageResult getApiAccessLogPage(ApiAccessLogPageReqVO pageReqVO) { return apiAccessLogMapper.selectPage(pageReqVO); diff --git a/yudao-module-infra/src/main/java/cn/iocoder/yudao/module/infra/service/logger/ApiErrorLogService.java b/yudao-module-infra/src/main/java/cn/iocoder/yudao/module/infra/service/logger/ApiErrorLogService.java index b05ccf3d89..5c45797e35 100644 --- a/yudao-module-infra/src/main/java/cn/iocoder/yudao/module/infra/service/logger/ApiErrorLogService.java +++ b/yudao-module-infra/src/main/java/cn/iocoder/yudao/module/infra/service/logger/ApiErrorLogService.java @@ -19,6 +19,14 @@ public interface ApiErrorLogService { */ void createApiErrorLog(ApiErrorLogCreateReqDTO createReqDTO); + /** + * 获得 API 错误日志 + * + * @param id 编号 + * @return API 错误日志 + */ + ApiErrorLogDO getApiErrorLog(Long id); + /** * 获得 API 错误日志分页 * diff --git a/yudao-module-infra/src/main/java/cn/iocoder/yudao/module/infra/service/logger/ApiErrorLogServiceImpl.java b/yudao-module-infra/src/main/java/cn/iocoder/yudao/module/infra/service/logger/ApiErrorLogServiceImpl.java index bde4ee54e9..d1bb5ddecd 100644 --- a/yudao-module-infra/src/main/java/cn/iocoder/yudao/module/infra/service/logger/ApiErrorLogServiceImpl.java +++ b/yudao-module-infra/src/main/java/cn/iocoder/yudao/module/infra/service/logger/ApiErrorLogServiceImpl.java @@ -58,6 +58,11 @@ public class ApiErrorLogServiceImpl implements ApiErrorLogService { return apiErrorLogMapper.selectPage(pageReqVO); } + @Override + public ApiErrorLogDO getApiErrorLog(Long id) { + return apiErrorLogMapper.selectById(id); + } + @Override public void updateApiErrorLogProcess(Long id, Integer processStatus, Long processUserId) { ApiErrorLogDO errorLog = apiErrorLogMapper.selectById(id); diff --git a/yudao-module-mall/yudao-module-product/src/main/java/cn/iocoder/yudao/module/product/dal/mysql/sku/ProductSkuMapper.java b/yudao-module-mall/yudao-module-product/src/main/java/cn/iocoder/yudao/module/product/dal/mysql/sku/ProductSkuMapper.java index da920a5b82..21f58f75e4 100755 --- a/yudao-module-mall/yudao-module-product/src/main/java/cn/iocoder/yudao/module/product/dal/mysql/sku/ProductSkuMapper.java +++ b/yudao-module-mall/yudao-module-product/src/main/java/cn/iocoder/yudao/module/product/dal/mysql/sku/ProductSkuMapper.java @@ -5,9 +5,8 @@ import cn.iocoder.yudao.framework.mybatis.core.mapper.BaseMapperX; import cn.iocoder.yudao.framework.mybatis.core.query.LambdaQueryWrapperX; import cn.iocoder.yudao.module.product.dal.dataobject.sku.ProductSkuDO; import com.baomidou.mybatisplus.core.conditions.update.LambdaUpdateWrapper; -import org.apache.ibatis.annotations.Mapper; -import org.apache.ibatis.annotations.Param; -import org.apache.ibatis.annotations.Select; +import com.baomidou.mybatisplus.extension.handlers.JacksonTypeHandler; +import org.apache.ibatis.annotations.*; import java.util.Collection; import java.util.List; @@ -15,7 +14,14 @@ import java.util.List; @Mapper public interface ProductSkuMapper extends BaseMapperX { + /** + * 查询商品 SKU(包含已删除) + * 注意:使用 @Results 手动指定 typeHandler,否则 @Select 不会应用 autoResultMap,properties 字段无法解析 JSON + */ @Select("SELECT * FROM product_sku WHERE id = #{id}") + @Results({ + @Result(column = "properties", property = "properties", typeHandler = JacksonTypeHandler.class), + }) ProductSkuDO selectByIdIncludeDeleted(@Param("id") Long id); default List selectListBySpuId(Long spuId) { diff --git a/yudao-module-mall/yudao-module-product/src/main/java/cn/iocoder/yudao/module/product/dal/mysql/spu/ProductSpuMapper.java b/yudao-module-mall/yudao-module-product/src/main/java/cn/iocoder/yudao/module/product/dal/mysql/spu/ProductSpuMapper.java index fc00ae78d4..24ebd6864e 100755 --- a/yudao-module-mall/yudao-module-product/src/main/java/cn/iocoder/yudao/module/product/dal/mysql/spu/ProductSpuMapper.java +++ b/yudao-module-mall/yudao-module-product/src/main/java/cn/iocoder/yudao/module/product/dal/mysql/spu/ProductSpuMapper.java @@ -4,15 +4,15 @@ import cn.hutool.core.util.ObjectUtil; import cn.iocoder.yudao.framework.common.pojo.PageResult; import cn.iocoder.yudao.framework.mybatis.core.mapper.BaseMapperX; import cn.iocoder.yudao.framework.mybatis.core.query.LambdaQueryWrapperX; +import cn.iocoder.yudao.framework.mybatis.core.type.IntegerListTypeHandler; import cn.iocoder.yudao.module.product.controller.admin.spu.vo.ProductSpuPageReqVO; import cn.iocoder.yudao.module.product.controller.app.spu.vo.AppProductSpuPageReqVO; import cn.iocoder.yudao.module.product.dal.dataobject.spu.ProductSpuDO; import cn.iocoder.yudao.module.product.enums.ProductConstants; import cn.iocoder.yudao.module.product.enums.spu.ProductSpuStatusEnum; import com.baomidou.mybatisplus.core.conditions.update.LambdaUpdateWrapper; -import org.apache.ibatis.annotations.Mapper; -import org.apache.ibatis.annotations.Param; -import org.apache.ibatis.annotations.Select; +import com.baomidou.mybatisplus.extension.handlers.JacksonTypeHandler; +import org.apache.ibatis.annotations.*; import java.util.Objects; import java.util.Set; @@ -20,7 +20,15 @@ import java.util.Set; @Mapper public interface ProductSpuMapper extends BaseMapperX { + /** + * 查询商品 SPU(包含已删除) + * 注意:使用 @Results 手动指定 typeHandler,否则 @Select 不会应用 autoResultMap,sliderPicUrls,deliveryTypes 字段无法解析 JSON + */ @Select("SELECT * FROM product_spu WHERE id = #{id}") + @Results({ + @Result(column = "slider_pic_urls", property = "sliderPicUrls", typeHandler = JacksonTypeHandler.class), + @Result(column = "delivery_types", property = "deliveryTypes", typeHandler = IntegerListTypeHandler.class), + }) ProductSpuDO selectByIdIncludeDeleted(@Param("id") Long id); /** diff --git a/yudao-module-mall/yudao-module-trade/src/main/java/cn/iocoder/yudao/module/trade/framework/aftersale/core/aop/AfterSaleLogAspect.java b/yudao-module-mall/yudao-module-trade/src/main/java/cn/iocoder/yudao/module/trade/framework/aftersale/core/aop/AfterSaleLogAspect.java index dc0d538631..a6f1fa5965 100644 --- a/yudao-module-mall/yudao-module-trade/src/main/java/cn/iocoder/yudao/module/trade/framework/aftersale/core/aop/AfterSaleLogAspect.java +++ b/yudao-module-mall/yudao-module-trade/src/main/java/cn/iocoder/yudao/module/trade/framework/aftersale/core/aop/AfterSaleLogAspect.java @@ -113,6 +113,9 @@ public class AfterSaleLogAspect { * @return 用户类型 */ private static Long getUserId() { + if (USER_ID.get() != null) { + return USER_ID.get(); + } return ObjectUtil.defaultIfNull(WebFrameworkUtils.getLoginUserId(), TradeOrderLogDO.USER_ID_SYSTEM); } diff --git a/yudao-module-mall/yudao-module-trade/src/main/java/cn/iocoder/yudao/module/trade/framework/order/core/aop/TradeOrderLogAspect.java b/yudao-module-mall/yudao-module-trade/src/main/java/cn/iocoder/yudao/module/trade/framework/order/core/aop/TradeOrderLogAspect.java index ccdf91c617..6a22091214 100644 --- a/yudao-module-mall/yudao-module-trade/src/main/java/cn/iocoder/yudao/module/trade/framework/order/core/aop/TradeOrderLogAspect.java +++ b/yudao-module-mall/yudao-module-trade/src/main/java/cn/iocoder/yudao/module/trade/framework/order/core/aop/TradeOrderLogAspect.java @@ -109,6 +109,9 @@ public class TradeOrderLogAspect { * @return 用户类型 */ private static Long getUserId() { + if (USER_ID.get() != null) { + return USER_ID.get(); + } return ObjectUtil.defaultIfNull(WebFrameworkUtils.getLoginUserId(), TradeOrderLogDO.USER_ID_SYSTEM); } diff --git a/yudao-module-pay/src/main/java/cn/iocoder/yudao/module/pay/controller/admin/wallet/PayWalletRechargeController.java b/yudao-module-pay/src/main/java/cn/iocoder/yudao/module/pay/controller/admin/wallet/PayWalletRechargeController.java index d2f3afe2fc..339c9bb94f 100644 --- a/yudao-module-pay/src/main/java/cn/iocoder/yudao/module/pay/controller/admin/wallet/PayWalletRechargeController.java +++ b/yudao-module-pay/src/main/java/cn/iocoder/yudao/module/pay/controller/admin/wallet/PayWalletRechargeController.java @@ -51,7 +51,7 @@ public class PayWalletRechargeController { public CommonResult updateWalletRechargeRefunded(@RequestBody PayRefundNotifyReqDTO notifyReqDTO) { walletRechargeService.updateWalletRechargeRefunded( Long.valueOf(notifyReqDTO.getMerchantOrderId()), - Long.valueOf(notifyReqDTO.getMerchantRefundId()), + notifyReqDTO.getMerchantRefundId(), notifyReqDTO.getPayRefundId()); return success(true); } diff --git a/yudao-module-pay/src/main/java/cn/iocoder/yudao/module/pay/service/wallet/PayWalletRechargeService.java b/yudao-module-pay/src/main/java/cn/iocoder/yudao/module/pay/service/wallet/PayWalletRechargeService.java index f2ab50ed68..d837b9acd6 100644 --- a/yudao-module-pay/src/main/java/cn/iocoder/yudao/module/pay/service/wallet/PayWalletRechargeService.java +++ b/yudao-module-pay/src/main/java/cn/iocoder/yudao/module/pay/service/wallet/PayWalletRechargeService.java @@ -56,9 +56,9 @@ public interface PayWalletRechargeService { * 更新钱包充值记录为已退款 * * @param id 钱包充值记录编号 - * @param refundId 钱包充值退款编号(实际和 id 相同) + * @param refundId 钱包充值退款编号(格式:{id}-refund) * @param payRefundId 退款单id */ - void updateWalletRechargeRefunded(Long id, Long refundId, Long payRefundId); + void updateWalletRechargeRefunded(Long id, String refundId, Long payRefundId); } diff --git a/yudao-module-pay/src/main/java/cn/iocoder/yudao/module/pay/service/wallet/PayWalletRechargeServiceImpl.java b/yudao-module-pay/src/main/java/cn/iocoder/yudao/module/pay/service/wallet/PayWalletRechargeServiceImpl.java index 7d9192d876..401ae6e8f2 100644 --- a/yudao-module-pay/src/main/java/cn/iocoder/yudao/module/pay/service/wallet/PayWalletRechargeServiceImpl.java +++ b/yudao-module-pay/src/main/java/cn/iocoder/yudao/module/pay/service/wallet/PayWalletRechargeServiceImpl.java @@ -220,9 +220,8 @@ public class PayWalletRechargeServiceImpl implements PayWalletRechargeService { @Override @Transactional(rollbackFor = Exception.class) - public void updateWalletRechargeRefunded(Long id, Long refundId, Long payRefundId) { + public void updateWalletRechargeRefunded(Long id, String refundId, Long payRefundId) { // 1.1 获取钱包充值记录 - // 说明:因为 id 和 refundId 是相同的,所以直接使用 id 查询即可! PayWalletRechargeDO walletRecharge = walletRechargeMapper.selectById(id); if (walletRecharge == null) { log.error("[updateWalletRechargerPaid][钱包充值记录不存在,钱包充值记录 id({})]", id); @@ -274,8 +273,8 @@ public class PayWalletRechargeServiceImpl implements PayWalletRechargeService { walletRecharge.getId(), payRefundId, toJsonString(walletRecharge), toJsonString(payRefund)); throw exception(WALLET_RECHARGE_REFUND_FAIL_REFUND_PRICE_NOT_MATCH); } - // 2.3 校验退款订单商户订单是否匹配 - if (notEqual(payRefund.getMerchantRefundId(), walletRecharge.getId().toString())) { + // 2.3 校验退款订单商户退款单是否匹配 + if (notEqual(payRefund.getMerchantRefundId(), walletRecharge.getId() + "-refund")) { log.error("[validateWalletRechargeCanRefunded][钱包({}) 退款单不匹配({}),请进行处理!payRefund 数据是:{}]", walletRecharge.getId(), payRefundId, toJsonString(payRefund)); throw exception(WALLET_RECHARGE_REFUND_FAIL_REFUND_ORDER_ID_ERROR); diff --git a/yudao-module-pay/src/main/java/cn/iocoder/yudao/module/pay/service/wallet/PayWalletServiceImpl.java b/yudao-module-pay/src/main/java/cn/iocoder/yudao/module/pay/service/wallet/PayWalletServiceImpl.java index 024cecd5ff..776aad13a4 100644 --- a/yudao-module-pay/src/main/java/cn/iocoder/yudao/module/pay/service/wallet/PayWalletServiceImpl.java +++ b/yudao-module-pay/src/main/java/cn/iocoder/yudao/module/pay/service/wallet/PayWalletServiceImpl.java @@ -58,13 +58,22 @@ public class PayWalletServiceImpl implements PayWalletService { private PayRefundService refundService; @Override + @SneakyThrows public PayWalletDO getOrCreateWallet(Long userId, Integer userType) { PayWalletDO wallet = walletMapper.selectByUserIdAndType(userId, userType); if (wallet == null) { - wallet = new PayWalletDO().setUserId(userId).setUserType(userType) - .setBalance(0).setTotalExpense(0).setTotalRecharge(0); - wallet.setCreateTime(LocalDateTime.now()); - walletMapper.insert(wallet); + // 使用双重检查锁,保证钱包创建并发问题 + // https://gitee.com/zhijiantianya/ruoyi-vue-pro/pulls/1475/files + wallet = lockRedisDAO.lock(userId, UPDATE_TIMEOUT_MILLIS, () -> { + PayWalletDO newWallet = walletMapper.selectByUserIdAndType(userId, userType); + if (newWallet == null) { + newWallet = new PayWalletDO().setUserId(userId).setUserType(userType) + .setBalance(0).setTotalExpense(0).setTotalRecharge(0); + newWallet.setCreateTime(LocalDateTime.now()); + walletMapper.insert(newWallet); + } + return newWallet; + }); } return wallet; } diff --git a/yudao-module-system/src/main/java/cn/iocoder/yudao/module/system/controller/admin/dept/PostController.java b/yudao-module-system/src/main/java/cn/iocoder/yudao/module/system/controller/admin/dept/PostController.java index 86815b7a6a..817273bbb4 100644 --- a/yudao-module-system/src/main/java/cn/iocoder/yudao/module/system/controller/admin/dept/PostController.java +++ b/yudao-module-system/src/main/java/cn/iocoder/yudao/module/system/controller/admin/dept/PostController.java @@ -64,6 +64,14 @@ public class PostController { return success(true); } + @DeleteMapping("delete-list") + @Operation(summary = "批量删除岗位") + @PreAuthorize("@ss.hasPermission('system:post:delete')") + public CommonResult deletePostList(@RequestParam("ids") List ids) { + postService.deletePostList(ids); + return success(true); + } + @GetMapping(value = "/get") @Operation(summary = "获得岗位信息") @Parameter(name = "id", description = "岗位编号", required = true, example = "1024") diff --git a/yudao-module-system/src/main/java/cn/iocoder/yudao/module/system/controller/admin/logger/OperateLogController.java b/yudao-module-system/src/main/java/cn/iocoder/yudao/module/system/controller/admin/logger/OperateLogController.java index 0fb7326b76..07ded4c692 100644 --- a/yudao-module-system/src/main/java/cn/iocoder/yudao/module/system/controller/admin/logger/OperateLogController.java +++ b/yudao-module-system/src/main/java/cn/iocoder/yudao/module/system/controller/admin/logger/OperateLogController.java @@ -13,11 +13,13 @@ import cn.iocoder.yudao.module.system.dal.dataobject.logger.OperateLogDO; import cn.iocoder.yudao.module.system.service.logger.OperateLogService; import com.fhs.core.trans.anno.TransMethodResult; import io.swagger.v3.oas.annotations.Operation; +import io.swagger.v3.oas.annotations.Parameter; import io.swagger.v3.oas.annotations.tags.Tag; import org.springframework.security.access.prepost.PreAuthorize; import org.springframework.validation.annotation.Validated; import org.springframework.web.bind.annotation.GetMapping; import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestParam; import org.springframework.web.bind.annotation.RestController; import javax.annotation.Resource; @@ -38,6 +40,15 @@ public class OperateLogController { @Resource private OperateLogService operateLogService; + @GetMapping("/get") + @Operation(summary = "查看操作日志") + @Parameter(name = "id", description = "编号", required = true, example = "1024") + @PreAuthorize("@ss.hasPermission('system:operate-log:query')") + public CommonResult getOperateLog(@RequestParam("id") Long id) { + OperateLogDO operateLog = operateLogService.getOperateLog(id); + return success(BeanUtils.toBean(operateLog, OperateLogRespVO.class)); + } + @GetMapping("/page") @Operation(summary = "查看操作日志分页列表") @PreAuthorize("@ss.hasPermission('system:operate-log:query')") diff --git a/yudao-module-system/src/main/java/cn/iocoder/yudao/module/system/service/logger/OperateLogService.java b/yudao-module-system/src/main/java/cn/iocoder/yudao/module/system/service/logger/OperateLogService.java index c647e822d8..bc99a4d3cb 100644 --- a/yudao-module-system/src/main/java/cn/iocoder/yudao/module/system/service/logger/OperateLogService.java +++ b/yudao-module-system/src/main/java/cn/iocoder/yudao/module/system/service/logger/OperateLogService.java @@ -20,6 +20,14 @@ public interface OperateLogService { */ void createOperateLog(OperateLogCreateReqDTO createReqDTO); + /** + * 获得操作日志 + * + * @param id 编号 + * @return 操作日志 + */ + OperateLogDO getOperateLog(Long id); + /** * 获得操作日志分页列表 * diff --git a/yudao-module-system/src/main/java/cn/iocoder/yudao/module/system/service/logger/OperateLogServiceImpl.java b/yudao-module-system/src/main/java/cn/iocoder/yudao/module/system/service/logger/OperateLogServiceImpl.java index 7e61c6101c..e816fe17aa 100644 --- a/yudao-module-system/src/main/java/cn/iocoder/yudao/module/system/service/logger/OperateLogServiceImpl.java +++ b/yudao-module-system/src/main/java/cn/iocoder/yudao/module/system/service/logger/OperateLogServiceImpl.java @@ -32,6 +32,11 @@ public class OperateLogServiceImpl implements OperateLogService { operateLogMapper.insert(log); } + @Override + public OperateLogDO getOperateLog(Long id) { + return operateLogMapper.selectById(id); + } + @Override public PageResult getOperateLogPage(OperateLogPageReqVO pageReqVO) { return operateLogMapper.selectPage(pageReqVO);