【同步】BOOT 和 CLOUD 的功能

This commit is contained in:
YunaiV 2025-12-13 12:32:46 +08:00
parent b5a7350fe9
commit ec8577bdd9
29 changed files with 956 additions and 33 deletions

View File

@ -1,6 +1,7 @@
package cn.iocoder.yudao.framework.tenant.core.redis;
import cn.hutool.core.collection.CollUtil;
import cn.hutool.core.util.StrUtil;
import cn.iocoder.yudao.framework.redis.core.TimeoutRedisCacheManager;
import cn.iocoder.yudao.framework.tenant.core.context.TenantContextHolder;
import lombok.extern.slf4j.Slf4j;
@ -21,6 +22,8 @@ import java.util.Set;
@Slf4j
public class TenantRedisCacheManager extends TimeoutRedisCacheManager {
private static final String SPLIT = "#";
private final Set<String> ignoreCaches;
public TenantRedisCacheManager(RedisCacheWriter cacheWriter,
@ -32,10 +35,11 @@ public class TenantRedisCacheManager extends TimeoutRedisCacheManager {
@Override
public Cache getCache(String name) {
String[] names = StrUtil.splitToArray(name, SPLIT);
// 如果开启多租户 name 拼接租户后缀
if (!TenantContextHolder.isIgnore()
&& TenantContextHolder.getTenantId() != null
&& !CollUtil.contains(ignoreCaches, name)) {
&& !CollUtil.contains(ignoreCaches, names[0])) {
name = name + ":" + TenantContextHolder.getTenantId();
}
@ -43,4 +47,4 @@ public class TenantRedisCacheManager extends TimeoutRedisCacheManager {
return super.getCache(name);
}
}
}

View File

@ -0,0 +1,53 @@
package cn.iocoder.yudao.module.ai.enums;
import lombok.AllArgsConstructor;
import lombok.Getter;
/**
* AI 知识库文档切片策略枚举
*
* @author runzhen
*/
@AllArgsConstructor
@Getter
public enum AiDocumentSplitStrategyEnum {
/**
* 自动识别文档类型并选择最佳切片策略
*/
AUTO("auto", "自动识别"),
/**
* 基于 Token 数量机械切分默认策略
*/
TOKEN("token", "Token 切分"),
/**
* 按段落切分以双换行符为分隔
*/
PARAGRAPH("paragraph", "段落切分"),
/**
* Markdown QA 格式专用切片器
* 识别二级标题作为问题保持问答对完整性
* 长答案智能切分但保留问题作为上下文
*/
MARKDOWN_QA("markdown_qa", "Markdown QA 切分"),
/**
* 语义化切分保留句子完整性
* 在段落和句子边界处切分避免截断
*/
SEMANTIC("semantic", "语义切分");
/**
* 策略代码
*/
private final String code;
/**
* 策略名称
*/
private final String name;
}

View File

@ -20,7 +20,8 @@
</description>
<properties>
<spring-ai.version>1.1.0</spring-ai.version>
<alibaba-ai.version>1.1.0.0-M5</alibaba-ai.version>
<!-- https://mvnrepository.com/artifact/com.alibaba.cloud.ai/spring-ai-alibaba -->
<alibaba-ai.version>1.1.0.0-RC1</alibaba-ai.version>
<tinyflow.version>1.2.6</tinyflow.version>
</properties>
@ -262,6 +263,11 @@
<groupId>com.agentsflex</groupId>
<artifactId>agents-flex-store-elasticsearch</artifactId>
</exclusion>
<exclusion>
<!-- 解决 https://t.zsxq.com/pCBZC 问题 -->
<groupId>com.agentsflex</groupId>
<artifactId>agents-flex-search-engine-es</artifactId>
</exclusion>
<exclusion>
<!-- TODO @芋艿:暂时移除 groovy和 iot 冲突 -->
<groupId>org.codehaus.groovy</groupId>

View File

@ -11,7 +11,7 @@ import lombok.Data;
public class AiKnowledgeSegmentPageReqVO extends PageParam {
@Schema(description = "文档编号", example = "1")
private Integer documentId;
private Long documentId;
@Schema(description = "分段内容关键字", example = "Java 开发")
private String content;

View File

@ -4,6 +4,7 @@ import cn.hutool.core.collection.CollUtil;
import cn.hutool.core.collection.ListUtil;
import cn.hutool.core.util.ObjUtil;
import cn.hutool.core.util.StrUtil;
import cn.iocoder.yudao.framework.common.enums.CommonStatusEnum;
import cn.iocoder.yudao.framework.common.pojo.PageResult;
import cn.iocoder.yudao.framework.common.util.object.BeanUtils;
@ -15,8 +16,11 @@ import cn.iocoder.yudao.module.ai.dal.dataobject.knowledge.AiKnowledgeDO;
import cn.iocoder.yudao.module.ai.dal.dataobject.knowledge.AiKnowledgeDocumentDO;
import cn.iocoder.yudao.module.ai.dal.dataobject.knowledge.AiKnowledgeSegmentDO;
import cn.iocoder.yudao.module.ai.dal.mysql.knowledge.AiKnowledgeSegmentMapper;
import cn.iocoder.yudao.module.ai.enums.AiDocumentSplitStrategyEnum;
import cn.iocoder.yudao.module.ai.service.knowledge.bo.AiKnowledgeSegmentSearchReqBO;
import cn.iocoder.yudao.module.ai.service.knowledge.bo.AiKnowledgeSegmentSearchRespBO;
import cn.iocoder.yudao.module.ai.service.knowledge.splitter.MarkdownQaSplitter;
import cn.iocoder.yudao.module.ai.service.knowledge.splitter.SemanticTextSplitter;
import cn.iocoder.yudao.module.ai.service.model.AiModelService;
import com.alibaba.cloud.ai.dashscope.rerank.DashScopeRerankOptions;
import com.alibaba.cloud.ai.model.RerankModel;
@ -39,8 +43,7 @@ import java.util.*;
import static cn.iocoder.yudao.framework.common.exception.util.ServiceExceptionUtil.exception;
import static cn.iocoder.yudao.framework.common.util.collection.CollectionUtils.convertList;
import static cn.iocoder.yudao.module.ai.enums.ErrorCodeConstants.KNOWLEDGE_SEGMENT_CONTENT_TOO_LONG;
import static cn.iocoder.yudao.module.ai.enums.ErrorCodeConstants.KNOWLEDGE_SEGMENT_NOT_EXISTS;
import static cn.iocoder.yudao.module.ai.enums.ErrorCodeConstants.*;
import static org.springframework.ai.vectorstore.SearchRequest.SIMILARITY_THRESHOLD_ACCEPT_ALL;
/**
@ -95,8 +98,9 @@ public class AiKnowledgeSegmentServiceImpl implements AiKnowledgeSegmentService
AiKnowledgeDO knowledgeDO = knowledgeService.validateKnowledgeExists(documentDO.getKnowledgeId());
VectorStore vectorStore = getVectorStoreById(knowledgeDO);
// 2. 文档切片
List<Document> documentSegments = splitContentByToken(content, documentDO.getSegmentMaxTokens());
// 2. 文档切片使用自动检测策略
List<Document> documentSegments = splitContentByStrategy(content, documentDO.getSegmentMaxTokens(),
AiDocumentSplitStrategyEnum.AUTO, documentDO.getUrl());
// 3.1 存储切片
List<AiKnowledgeSegmentDO> segmentDOs = convertList(documentSegments, segment -> {
@ -295,8 +299,10 @@ public class AiKnowledgeSegmentServiceImpl implements AiKnowledgeSegmentService
// 1. 读取 URL 内容
String content = knowledgeDocumentService.readUrl(url);
// 2. 文档切片
List<Document> documentSegments = splitContentByToken(content, segmentMaxTokens);
// 2.1 自动检测文档类型并选择策略
AiDocumentSplitStrategyEnum strategy = detectDocumentStrategy(content, url);
// 2.2 文档切片
List<Document> documentSegments = splitContentByStrategy(content, segmentMaxTokens, strategy, url);
// 3. 转换为段落对象
return convertList(documentSegments, segment -> {
@ -333,11 +339,103 @@ public class AiKnowledgeSegmentServiceImpl implements AiKnowledgeSegmentService
return getVectorStoreById(knowledge);
}
private static List<Document> splitContentByToken(String content, Integer segmentMaxTokens) {
TextSplitter textSplitter = buildTokenTextSplitter(segmentMaxTokens);
/**
* 根据策略切分内容
*
* @param content 文档内容
* @param segmentMaxTokens 分段的最大 Token
* @param strategy 切片策略
* @param url 文档 URL用于自动检测文件类型
* @return 切片后的文档列表
*/
@SuppressWarnings("EnhancedSwitchMigration")
private List<Document> splitContentByStrategy(String content, Integer segmentMaxTokens,
AiDocumentSplitStrategyEnum strategy, String url) {
// 自动检测策略
if (strategy == AiDocumentSplitStrategyEnum.AUTO) {
strategy = detectDocumentStrategy(content, url);
log.info("[splitContentByStrategy][自动检测到文档策略: {}]", strategy.getName());
}
// 根据策略切分
TextSplitter textSplitter;
switch (strategy) {
case MARKDOWN_QA:
textSplitter = new MarkdownQaSplitter(segmentMaxTokens);
break;
case SEMANTIC:
textSplitter = new SemanticTextSplitter(segmentMaxTokens);
break;
case PARAGRAPH:
textSplitter = new SemanticTextSplitter(segmentMaxTokens, 0); // 段落切分无重叠
break;
case TOKEN:
default:
textSplitter = buildTokenTextSplitter(segmentMaxTokens);
break;
}
// 执行切分
return textSplitter.apply(Collections.singletonList(new Document(content)));
}
/**
* 自动检测文档类型并选择切片策略
*
* @param content 文档内容
* @param url 文档 URL
* @return 推荐的切片策略
*/
private AiDocumentSplitStrategyEnum detectDocumentStrategy(String content, String url) {
if (StrUtil.isEmpty(content)) {
return AiDocumentSplitStrategyEnum.TOKEN;
}
// 1. 检测 Markdown QA 格式
if (isMarkdownQaFormat(content, url)) {
return AiDocumentSplitStrategyEnum.MARKDOWN_QA;
}
// 2. 检测普通 Markdown 文档
if (isMarkdownDocument(url)) {
return AiDocumentSplitStrategyEnum.SEMANTIC;
}
// 3. 默认使用语义切分 Token 切分更智能
return AiDocumentSplitStrategyEnum.SEMANTIC;
}
/**
* 检测是否为 Markdown QA 格式
* 特征包含多个二级标题## 且标题后紧跟答案内容
*/
private boolean isMarkdownQaFormat(String content, String url) {
// 文件扩展名判断
if (StrUtil.isNotEmpty(url) && !url.toLowerCase().endsWith(".md")) {
return false;
}
// 统计二级标题数量
long h2Count = content.lines()
.filter(line -> line.trim().startsWith("## "))
.count();
// 要求一至少包含 2 个二级标题才认为是 QA 格式
if (h2Count < 2) {
return false;
}
// 要求二检查标题占比QA 文档标题行数相对较多如果二级标题占比超过 10%认为是 QA 格式
long totalLines = content.lines().count();
double h2Ratio = (double) h2Count / totalLines;
return h2Ratio > 0.1;
}
/**
* 检测是否为 Markdown 文档
*/
private boolean isMarkdownDocument(String url) {
return StrUtil.endWithAnyIgnoreCase(url, ".md", ".markdown");
}
/**
* 构建基于 Token 的文本切片器原有逻辑保留
*/
private static TextSplitter buildTokenTextSplitter(Integer segmentMaxTokens) {
return TokenTextSplitter.builder()
.withChunkSize(segmentMaxTokens)

View File

@ -0,0 +1,342 @@
package cn.iocoder.yudao.module.ai.service.knowledge.splitter;
import cn.hutool.core.collection.CollUtil;
import cn.hutool.core.util.StrUtil;
import lombok.AllArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.ai.transformer.splitter.TextSplitter;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
/**
* Markdown QA 格式专用切片器
*
* <p>功能特点
* <ul>
* <li>识别二级标题## 作为问题标记</li>
* <li> QA 对保持完整不超过 Token 限制</li>
* <li>长答案智能切分每个片段保留完整问题作为上下文</li>
* <li>支持自定义 Token 估算器</li>
* </ul>
*
* @author runzhen
*/
@Slf4j
@SuppressWarnings("SizeReplaceableByIsEmpty")
public class MarkdownQaSplitter extends TextSplitter {
/**
* 二级标题正则匹配 "## " 开头的行
*/
private static final Pattern H2_PATTERN = Pattern.compile("^##\\s+(.+)$", Pattern.MULTILINE);
/**
* 段落分隔符双换行
*/
private static final String PARAGRAPH_SEPARATOR = "\n\n";
/**
* 句子分隔符
*/
private static final Pattern SENTENCE_PATTERN = Pattern.compile("[。!?.!?]\\s*");
/**
* 分段的最大 Token
*/
private final int chunkSize;
/**
* Token 估算器简单实现中文按字符数英文按单词数的 1.3
*/
private final TokenEstimator tokenEstimator;
public MarkdownQaSplitter(int chunkSize) {
this.chunkSize = chunkSize;
this.tokenEstimator = new SimpleTokenEstimator();
}
@Override
protected List<String> splitText(String text) {
if (StrUtil.isEmpty(text)) {
return Collections.emptyList();
}
// 解析 QA
List<QaPair> qaPairs = parseQaPairs(text);
if (CollUtil.isEmpty(qaPairs)) {
// 如果没有识别到 QA 格式按段落切分
return fallbackSplit(text);
}
// 处理每个 QA
List<String> result = new ArrayList<>();
for (QaPair qaPair : qaPairs) {
result.addAll(splitQaPair(qaPair));
}
return result;
}
/**
* 解析 Markdown QA
*
* @param content 文本内容
* @return QA 对列表
*/
private List<QaPair> parseQaPairs(String content) {
// 找到所有二级标题位置
List<QaPair> qaPairs = new ArrayList<>();
List<Integer> headingPositions = new ArrayList<>();
List<String> questions = new ArrayList<>();
Matcher matcher = H2_PATTERN.matcher(content);
while (matcher.find()) {
headingPositions.add(matcher.start());
questions.add(matcher.group(1).trim());
}
if (CollUtil.isEmpty(headingPositions)) {
return qaPairs;
}
// 提取每个 QA
for (int i = 0; i < headingPositions.size(); i++) {
int start = headingPositions.get(i);
int end = (i + 1 < headingPositions.size())
? headingPositions.get(i + 1)
: content.length();
String qaText = content.substring(start, end).trim();
String question = questions.get(i);
// 提取答案部分去掉问题标题
String answer = qaText.substring(qaText.indexOf('\n') + 1).trim();
qaPairs.add(new QaPair(question, answer, qaText));
}
return qaPairs;
}
/**
* 切分单个 QA
*
* @param qaPair QA
* @return 切分后的文本片段列表
*/
private List<String> splitQaPair(QaPair qaPair) {
// 如果整个 QA 对不超过限制保持完整
List<String> chunks = new ArrayList<>();
String fullQa = qaPair.fullText;
int qaTokens = tokenEstimator.estimate(fullQa);
if (qaTokens <= chunkSize) {
chunks.add(fullQa);
return chunks;
}
// 长答案需要切分
log.debug("QA 对超过 Token 限制 ({} > {}),开始智能切分: {}", qaTokens, chunkSize, qaPair.question);
List<String> answerChunks = splitLongAnswer(qaPair.answer, qaPair.question);
for (String answerChunk : answerChunks) {
// 每个片段都包含完整问题
String chunkText = "## " + qaPair.question + "\n" + answerChunk;
chunks.add(chunkText);
}
return chunks;
}
/**
* 切分长答案
*
* @param answer 答案文本
* @param question 问题文本
* @return 切分后的答案片段列表
*/
private List<String> splitLongAnswer(String answer, String question) {
List<String> chunks = new ArrayList<>();
// 预留问题的 Token 空间
String questionHeader = "## " + question + "\n";
int questionTokens = tokenEstimator.estimate(questionHeader);
int availableTokens = chunkSize - questionTokens - 10; // 预留 10 Token 的缓冲
// 先按段落切分
String[] paragraphs = answer.split(PARAGRAPH_SEPARATOR);
StringBuilder currentChunk = new StringBuilder();
int currentTokens = 0;
for (String paragraph : paragraphs) {
if (StrUtil.isEmpty(paragraph)) {
continue;
}
int paragraphTokens = tokenEstimator.estimate(paragraph);
// 如果单个段落就超过限制需要按句子切分
if (paragraphTokens > availableTokens) {
// 先保存当前块
if (currentChunk.length() > 0) {
chunks.add(currentChunk.toString().trim());
currentChunk = new StringBuilder();
currentTokens = 0;
}
// 按句子切分长段落
chunks.addAll(splitLongParagraph(paragraph, availableTokens));
continue;
}
// 如果加上这个段落会超过限制
if (currentTokens + paragraphTokens > availableTokens && currentChunk.length() > 0) {
chunks.add(currentChunk.toString().trim());
currentChunk = new StringBuilder();
currentTokens = 0;
}
if (currentChunk.length() > 0) {
currentChunk.append("\n\n");
}
// 添加段落
currentChunk.append(paragraph);
currentTokens += paragraphTokens;
}
// 添加最后一块
if (currentChunk.length() > 0) {
chunks.add(currentChunk.toString().trim());
}
return CollUtil.isEmpty(chunks) ? Collections.singletonList(answer) : chunks;
}
/**
* 切分长段落按句子
*
* @param paragraph 段落文本
* @param availableTokens 可用的 Token
* @return 切分后的文本片段列表
*/
private List<String> splitLongParagraph(String paragraph, int availableTokens) {
// 按句子切分
List<String> chunks = new ArrayList<>();
String[] sentences = SENTENCE_PATTERN.split(paragraph);
// 按句子累积切分
StringBuilder currentChunk = new StringBuilder();
int currentTokens = 0;
for (String sentence : sentences) {
if (StrUtil.isEmpty(sentence)) {
continue;
}
int sentenceTokens = tokenEstimator.estimate(sentence);
// 如果单个句子就超过限制强制切分
if (sentenceTokens > availableTokens) {
if (currentChunk.length() > 0) {
chunks.add(currentChunk.toString().trim());
currentChunk = new StringBuilder();
currentTokens = 0;
}
chunks.add(sentence.trim());
continue;
}
// 如果加上这个句子会超过限制
if (currentTokens + sentenceTokens > availableTokens && currentChunk.length() > 0) {
chunks.add(currentChunk.toString().trim());
currentChunk = new StringBuilder();
currentTokens = 0;
}
// 添加句子
currentChunk.append(sentence);
currentTokens += sentenceTokens;
}
// 添加最后一块
if (currentChunk.length() > 0) {
chunks.add(currentChunk.toString().trim());
}
return chunks.isEmpty() ? Collections.singletonList(paragraph) : chunks;
}
/**
* 降级切分策略当未识别到 QA 格式时
*
* @param content 文本内容
* @return 切分后的文本片段列表
*/
private List<String> fallbackSplit(String content) {
// 按段落切分
List<String> chunks = new ArrayList<>();
String[] paragraphs = content.split(PARAGRAPH_SEPARATOR);
// 按段落累积切分
StringBuilder currentChunk = new StringBuilder();
int currentTokens = 0;
for (String paragraph : paragraphs) {
if (StrUtil.isEmpty(paragraph)) {
continue;
}
int paragraphTokens = tokenEstimator.estimate(paragraph);
// 如果加上这个段落会超过限制
if (currentTokens + paragraphTokens > chunkSize && currentChunk.length() > 0) {
chunks.add(currentChunk.toString().trim());
currentChunk = new StringBuilder();
currentTokens = 0;
}
// 添加段落
if (currentChunk.length() > 0) {
currentChunk.append("\n\n");
}
currentChunk.append(paragraph);
currentTokens += paragraphTokens;
}
// 添加最后一块
if (currentChunk.length() > 0) {
chunks.add(currentChunk.toString().trim());
}
return chunks.isEmpty() ? Collections.singletonList(content) : chunks;
}
/**
* QA 对数据结构
*/
@AllArgsConstructor
private static class QaPair {
String question;
String answer;
String fullText;
}
/**
* Token 估算器接口
*/
public interface TokenEstimator {
int estimate(String text);
}
/**
* 简单的 Token 估算器实现
* 中文1 字符 1 Token
* 英文1 单词 1.3 Token
*/
private static class SimpleTokenEstimator implements TokenEstimator {
@Override
public int estimate(String text) {
if (StrUtil.isEmpty(text)) {
return 0;
}
int chineseChars = 0;
int englishWords = 0;
// 简单统计中英文
for (char c : text.toCharArray()) {
if (c >= 0x4E00 && c <= 0x9FA5) {
chineseChars++;
}
}
// 英文单词估算
String[] words = text.split("\\s+");
for (String word : words) {
if (word.matches(".*[a-zA-Z].*")) {
englishWords++;
}
}
return chineseChars + (int) (englishWords * 1.3);
}
}
}

View File

@ -0,0 +1,301 @@
package cn.iocoder.yudao.module.ai.service.knowledge.splitter;
import cn.hutool.core.util.StrUtil;
import lombok.extern.slf4j.Slf4j;
import org.springframework.ai.transformer.splitter.TextSplitter;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
/**
* 语义化文本切片器
*
* <p>功能特点
* <ul>
* <li>优先在段落边界双换行处切分</li>
* <li>其次在句子边界句号问号感叹号处切分</li>
* <li>避免在句子中间截断保持语义完整性</li>
* <li>支持中英文标点符号识别</li>
* </ul>
*
* @author runzhen
*/
@Slf4j
public class SemanticTextSplitter extends TextSplitter {
/**
* 分段的最大 Token
*/
private final int chunkSize;
/**
* 段落重叠大小用于保持上下文连贯性
*/
private final int chunkOverlap;
/**
* 段落分隔符按优先级排序
*/
private static final List<String> PARAGRAPH_SEPARATORS = Arrays.asList(
"\n\n\n", // 三个换行
"\n\n", // 双换行
"\n" // 单换行
);
/**
* 句子结束标记中英文标点
*/
private static final Pattern SENTENCE_END_PATTERN = Pattern.compile(
"[。!?.!?]+[\\s\"')】\\]]*"
);
/**
* Token 估算器
*/
private final MarkdownQaSplitter.TokenEstimator tokenEstimator;
public SemanticTextSplitter(int chunkSize, int chunkOverlap) {
this.chunkSize = chunkSize;
this.chunkOverlap = Math.min(chunkOverlap, chunkSize / 2); // 重叠不超过一半
this.tokenEstimator = new SimpleTokenEstimator();
}
public SemanticTextSplitter(int chunkSize) {
this(chunkSize, 50); // 默认重叠 50 Token
}
@Override
protected List<String> splitText(String text) {
if (StrUtil.isEmpty(text)) {
return Collections.emptyList();
}
return splitTextRecursive(text);
}
/**
* 切分文本递归策略
*
* @param text 待切分文本
* @return 切分后的文本块列表
*/
private List<String> splitTextRecursive(String text) {
List<String> chunks = new ArrayList<>();
// 如果文本不超过限制直接返回
int textTokens = tokenEstimator.estimate(text);
if (textTokens <= chunkSize) {
chunks.add(text.trim());
return chunks;
}
// 尝试按不同分隔符切分
List<String> splits = null;
String usedSeparator = null;
for (String separator : PARAGRAPH_SEPARATORS) {
if (text.contains(separator)) {
splits = Arrays.asList(text.split(Pattern.quote(separator)));
usedSeparator = separator;
break;
}
}
// 如果没有找到段落分隔符按句子切分
if (splits == null || splits.size() == 1) {
splits = splitBySentences(text);
usedSeparator = ""; // 句子切分不需要分隔符
}
// 合并小片段
chunks = mergeSplits(splits, usedSeparator);
return chunks;
}
/**
* 按句子切分
*
* @param text 待切分文本
* @return 句子列表
*/
private List<String> splitBySentences(String text) {
// 使用正则表达式匹配句子结束位置
List<String> sentences = new ArrayList<>();
int lastEnd = 0;
Matcher matcher = SENTENCE_END_PATTERN.matcher(text);
while (matcher.find()) {
String sentence = text.substring(lastEnd, matcher.end()).trim();
if (StrUtil.isNotEmpty(sentence)) {
sentences.add(sentence);
}
lastEnd = matcher.end();
}
// 添加剩余部分
if (lastEnd < text.length()) {
String remaining = text.substring(lastEnd).trim();
if (StrUtil.isNotEmpty(remaining)) {
sentences.add(remaining);
}
}
return sentences.isEmpty() ? Collections.singletonList(text) : sentences;
}
/**
* 合并切分后的小片段
*
* @param splits 切分后的片段列表
* @param separator 片段间的分隔符
* @return 合并后的文本块列表
*/
private List<String> mergeSplits(List<String> splits, String separator) {
List<String> chunks = new ArrayList<>();
List<String> currentChunks = new ArrayList<>();
int currentLength = 0;
for (String split : splits) {
if (StrUtil.isEmpty(split)) {
continue;
}
int splitTokens = tokenEstimator.estimate(split);
// 如果单个片段就超过限制进一步递归切分
if (splitTokens > chunkSize) {
// 先保存当前累积的块
if (!currentChunks.isEmpty()) {
String chunkText = String.join(separator, currentChunks);
chunks.add(chunkText.trim());
currentChunks.clear();
currentLength = 0;
}
// 递归切分大片段
if (!separator.isEmpty()) {
// 如果是段落分隔符尝试按句子切分
chunks.addAll(splitTextRecursive(split));
} else {
// 如果已经是句子级别强制按字符切分
chunks.addAll(forceSplitLongText(split));
}
continue;
}
// 计算加上分隔符的 Token
int separatorTokens = StrUtil.isEmpty(separator) ? 0 : tokenEstimator.estimate(separator);
// 如果加上这个片段会超过限制
if (!currentChunks.isEmpty() && currentLength + splitTokens + separatorTokens > chunkSize) {
// 保存当前块
String chunkText = String.join(separator, currentChunks);
chunks.add(chunkText.trim());
// 处理重叠保留最后几个片段
currentChunks = getOverlappingChunks(currentChunks, separator);
currentLength = estimateTokens(currentChunks, separator);
}
// 添加当前片段
currentChunks.add(split);
currentLength += splitTokens + separatorTokens;
}
// 添加最后一块
if (!currentChunks.isEmpty()) {
String chunkText = String.join(separator, currentChunks);
chunks.add(chunkText.trim());
}
return chunks;
}
/**
* 获取重叠的片段用于保持上下文
*
* @param chunks 当前片段列表
* @param separator 片段间的分隔符
* @return 重叠的片段列表
*/
private List<String> getOverlappingChunks(List<String> chunks, String separator) {
if (chunkOverlap == 0 || chunks.isEmpty()) {
return new ArrayList<>();
}
// 从后往前取片段直到达到重叠大小
List<String> overlapping = new ArrayList<>();
int tokens = 0;
for (int i = chunks.size() - 1; i >= 0; i--) {
String chunk = chunks.get(i);
int chunkTokens = tokenEstimator.estimate(chunk);
if (tokens + chunkTokens > chunkOverlap) {
break;
}
// 添加到重叠列表前端
overlapping.add(0, chunk);
tokens += chunkTokens + (StrUtil.isEmpty(separator) ? 0 : tokenEstimator.estimate(separator));
}
return overlapping;
}
/**
* 估算片段列表的总 Token
*
* @param chunks 片段列表
* @param separator 片段间的分隔符
* @return Token
*/
private int estimateTokens(List<String> chunks, String separator) {
int total = 0;
for (int i = 0; i < chunks.size(); i++) {
total += tokenEstimator.estimate(chunks.get(i));
if (i < chunks.size() - 1 && StrUtil.isNotEmpty(separator)) {
total += tokenEstimator.estimate(separator);
}
}
return total;
}
/**
* 强制切分长文本当语义切分失败时
*
* @param text 待切分文本
* @return 切分后的文本块列表
*/
private List<String> forceSplitLongText(String text) {
List<String> chunks = new ArrayList<>();
int charsPerChunk = (int) (chunkSize * 0.8); // 保守估计
for (int i = 0; i < text.length(); i += charsPerChunk) {
int end = Math.min(i + charsPerChunk, text.length());
String chunk = text.substring(i, end);
chunks.add(chunk.trim());
}
log.warn("文本过长,已强制按字符切分,可能影响语义完整性");
return chunks;
}
/**
* 简单的 Token 估算器实现
*/
private static class SimpleTokenEstimator implements MarkdownQaSplitter.TokenEstimator {
@Override
public int estimate(String text) {
if (StrUtil.isEmpty(text)) {
return 0;
}
int chineseChars = 0;
int englishWords = 0;
// 简单统计中英文
for (char c : text.toCharArray()) {
if (c >= 0x4E00 && c <= 0x9FA5) {
chineseChars++;
}
}
// 英文单词估算
String[] words = text.split("\\s+");
for (String word : words) {
if (word.matches(".*[a-zA-Z].*")) {
englishWords++;
}
}
return chineseChars + (int) (englishWords * 1.3);
}
}
}

View File

@ -13,6 +13,7 @@ import cn.iocoder.yudao.module.infra.service.file.FileService;
import io.swagger.v3.oas.annotations.Operation;
import io.swagger.v3.oas.annotations.Parameter;
import io.swagger.v3.oas.annotations.Parameters;
import io.swagger.v3.oas.annotations.media.Schema;
import io.swagger.v3.oas.annotations.tags.Tag;
import jakarta.annotation.Resource;
import jakarta.annotation.security.PermitAll;
@ -44,6 +45,8 @@ public class FileController {
@PostMapping("/upload")
@Operation(summary = "上传文件", description = "模式一:后端上传文件")
@Parameter(name = "file", description = "文件附件", required = true,
schema = @Schema(type = "string", format = "binary"))
public CommonResult<String> uploadFile(@Valid FileUploadReqVO uploadReqVO) throws Exception {
MultipartFile file = uploadReqVO.getFile();
byte[] content = IoUtil.readBytes(file.getInputStream());

View File

@ -22,7 +22,14 @@ public class FileUploadReqVO {
@AssertTrue(message = "文件目录不正确")
@JsonIgnore
public boolean isDirectoryValid() {
return !StrUtil.containsAny(directory, "..", "/", "\\");
return isDirectoryValid(directory);
}
public static boolean isDirectoryValid(String directory) {
// 1. 不能包含 .. 防止目录穿越
// 2. 不能以 / \ 开头防止上传到根目录
return !StrUtil.contains(directory, "..")
&& !StrUtil.startWithAny(directory, "/", "\\");
}
}

View File

@ -11,6 +11,7 @@ import cn.iocoder.yudao.module.infra.controller.admin.logger.vo.apiaccesslog.Api
import cn.iocoder.yudao.module.infra.dal.dataobject.logger.ApiAccessLogDO;
import cn.iocoder.yudao.module.infra.service.logger.ApiAccessLogService;
import io.swagger.v3.oas.annotations.Operation;
import io.swagger.v3.oas.annotations.Parameter;
import io.swagger.v3.oas.annotations.tags.Tag;
import jakarta.annotation.Resource;
import jakarta.servlet.http.HttpServletResponse;
@ -19,6 +20,7 @@ import org.springframework.security.access.prepost.PreAuthorize;
import org.springframework.validation.annotation.Validated;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.RestController;
import java.io.IOException;
@ -36,6 +38,15 @@ public class ApiAccessLogController {
@Resource
private ApiAccessLogService apiAccessLogService;
@GetMapping("/get")
@Operation(summary = "获得 API 访问日志")
@Parameter(name = "id", description = "编号", required = true, example = "1024")
@PreAuthorize("@ss.hasPermission('infra:api-access-log:query')")
public CommonResult<ApiAccessLogRespVO> getApiAccessLog(@RequestParam("id") Long id) {
ApiAccessLogDO apiAccessLog = apiAccessLogService.getApiAccessLog(id);
return success(BeanUtils.toBean(apiAccessLog, ApiAccessLogRespVO.class));
}
@GetMapping("/page")
@Operation(summary = "获得API 访问日志分页")
@PreAuthorize("@ss.hasPermission('infra:api-access-log:query')")

View File

@ -50,6 +50,15 @@ public class ApiErrorLogController {
return success(true);
}
@GetMapping("/get")
@Operation(summary = "获得 API 错误日志")
@Parameter(name = "id", description = "编号", required = true, example = "1024")
@PreAuthorize("@ss.hasPermission('infra:api-error-log:query')")
public CommonResult<ApiErrorLogRespVO> getApiErrorLog(@RequestParam("id") Long id) {
ApiErrorLogDO apiErrorLog = apiErrorLogService.getApiErrorLog(id);
return success(BeanUtils.toBean(apiErrorLog, ApiErrorLogRespVO.class));
}
@GetMapping("/page")
@Operation(summary = "获得 API 错误日志分页")
@PreAuthorize("@ss.hasPermission('infra:api-error-log:query')")
@ -63,7 +72,7 @@ public class ApiErrorLogController {
@PreAuthorize("@ss.hasPermission('infra:api-error-log:export')")
@ApiAccessLog(operateType = EXPORT)
public void exportApiErrorLogExcel(@Valid ApiErrorLogPageReqVO exportReqVO,
HttpServletResponse response) throws IOException {
HttpServletResponse response) throws IOException {
exportReqVO.setPageSize(PageParam.PAGE_SIZE_NONE);
List<ApiErrorLogDO> list = apiErrorLogService.getApiErrorLogPage(exportReqVO).getList();
// 导出 Excel

View File

@ -9,6 +9,7 @@ import cn.iocoder.yudao.module.infra.service.file.FileService;
import io.swagger.v3.oas.annotations.Operation;
import io.swagger.v3.oas.annotations.Parameter;
import io.swagger.v3.oas.annotations.Parameters;
import io.swagger.v3.oas.annotations.media.Schema;
import io.swagger.v3.oas.annotations.tags.Tag;
import jakarta.annotation.Resource;
import jakarta.annotation.security.PermitAll;
@ -32,6 +33,8 @@ public class AppFileController {
@PostMapping("/upload")
@Operation(summary = "上传文件")
@Parameter(name = "file", description = "文件附件", required = true,
schema = @Schema(type = "string", format = "binary"))
@PermitAll
public CommonResult<String> uploadFile(AppFileUploadReqVO uploadReqVO) throws Exception {
MultipartFile file = uploadReqVO.getFile();

View File

@ -1,6 +1,6 @@
package cn.iocoder.yudao.module.infra.controller.app.file.vo;
import cn.hutool.core.util.StrUtil;
import cn.iocoder.yudao.module.infra.controller.admin.file.vo.file.FileUploadReqVO;
import com.fasterxml.jackson.annotation.JsonIgnore;
import io.swagger.v3.oas.annotations.media.Schema;
import jakarta.validation.constraints.AssertTrue;
@ -22,7 +22,7 @@ public class AppFileUploadReqVO {
@AssertTrue(message = "文件目录不正确")
@JsonIgnore
public boolean isDirectoryValid() {
return !StrUtil.containsAny(directory, "..", "/", "\\");
return FileUploadReqVO.isDirectoryValid(directory);
}
}

View File

@ -19,6 +19,14 @@ public interface ApiAccessLogService {
*/
void createApiAccessLog(ApiAccessLogCreateReqDTO createReqDTO);
/**
* 获得 API 访问日志
*
* @param id 编号
* @return API 访问日志
*/
ApiAccessLogDO getApiAccessLog(Long id);
/**
* 获得 API 访问日志分页
*

View File

@ -45,6 +45,11 @@ public class ApiAccessLogServiceImpl implements ApiAccessLogService {
}
}
@Override
public ApiAccessLogDO getApiAccessLog(Long id) {
return apiAccessLogMapper.selectById(id);
}
@Override
public PageResult<ApiAccessLogDO> getApiAccessLogPage(ApiAccessLogPageReqVO pageReqVO) {
return apiAccessLogMapper.selectPage(pageReqVO);

View File

@ -19,6 +19,14 @@ public interface ApiErrorLogService {
*/
void createApiErrorLog(ApiErrorLogCreateReqDTO createReqDTO);
/**
* 获得 API 错误日志
*
* @param id 编号
* @return API 错误日志
*/
ApiErrorLogDO getApiErrorLog(Long id);
/**
* 获得 API 错误日志分页
*

View File

@ -58,6 +58,11 @@ public class ApiErrorLogServiceImpl implements ApiErrorLogService {
return apiErrorLogMapper.selectPage(pageReqVO);
}
@Override
public ApiErrorLogDO getApiErrorLog(Long id) {
return apiErrorLogMapper.selectById(id);
}
@Override
public void updateApiErrorLogProcess(Long id, Integer processStatus, Long processUserId) {
ApiErrorLogDO errorLog = apiErrorLogMapper.selectById(id);

View File

@ -5,9 +5,8 @@ import cn.iocoder.yudao.framework.mybatis.core.mapper.BaseMapperX;
import cn.iocoder.yudao.framework.mybatis.core.query.LambdaQueryWrapperX;
import cn.iocoder.yudao.module.product.dal.dataobject.sku.ProductSkuDO;
import com.baomidou.mybatisplus.core.conditions.update.LambdaUpdateWrapper;
import org.apache.ibatis.annotations.Mapper;
import org.apache.ibatis.annotations.Param;
import org.apache.ibatis.annotations.Select;
import com.baomidou.mybatisplus.extension.handlers.JacksonTypeHandler;
import org.apache.ibatis.annotations.*;
import java.util.Collection;
import java.util.List;
@ -15,7 +14,14 @@ import java.util.List;
@Mapper
public interface ProductSkuMapper extends BaseMapperX<ProductSkuDO> {
/**
* 查询商品 SKU包含已删除
* 注意使用 @Results 手动指定 typeHandler否则 @Select 不会应用 autoResultMapproperties 字段无法解析 JSON
*/
@Select("SELECT * FROM product_sku WHERE id = #{id}")
@Results({
@Result(column = "properties", property = "properties", typeHandler = JacksonTypeHandler.class),
})
ProductSkuDO selectByIdIncludeDeleted(@Param("id") Long id);
default List<ProductSkuDO> selectListBySpuId(Long spuId) {

View File

@ -4,15 +4,15 @@ import cn.hutool.core.util.ObjectUtil;
import cn.iocoder.yudao.framework.common.pojo.PageResult;
import cn.iocoder.yudao.framework.mybatis.core.mapper.BaseMapperX;
import cn.iocoder.yudao.framework.mybatis.core.query.LambdaQueryWrapperX;
import cn.iocoder.yudao.framework.mybatis.core.type.IntegerListTypeHandler;
import cn.iocoder.yudao.module.product.controller.admin.spu.vo.ProductSpuPageReqVO;
import cn.iocoder.yudao.module.product.controller.app.spu.vo.AppProductSpuPageReqVO;
import cn.iocoder.yudao.module.product.dal.dataobject.spu.ProductSpuDO;
import cn.iocoder.yudao.module.product.enums.ProductConstants;
import cn.iocoder.yudao.module.product.enums.spu.ProductSpuStatusEnum;
import com.baomidou.mybatisplus.core.conditions.update.LambdaUpdateWrapper;
import org.apache.ibatis.annotations.Mapper;
import org.apache.ibatis.annotations.Param;
import org.apache.ibatis.annotations.Select;
import com.baomidou.mybatisplus.extension.handlers.JacksonTypeHandler;
import org.apache.ibatis.annotations.*;
import java.util.Objects;
import java.util.Set;
@ -20,7 +20,15 @@ import java.util.Set;
@Mapper
public interface ProductSpuMapper extends BaseMapperX<ProductSpuDO> {
/**
* 查询商品 SPU包含已删除
* 注意使用 @Results 手动指定 typeHandler否则 @Select 不会应用 autoResultMapsliderPicUrlsdeliveryTypes 字段无法解析 JSON
*/
@Select("SELECT * FROM product_spu WHERE id = #{id}")
@Results({
@Result(column = "slider_pic_urls", property = "sliderPicUrls", typeHandler = JacksonTypeHandler.class),
@Result(column = "delivery_types", property = "deliveryTypes", typeHandler = IntegerListTypeHandler.class),
})
ProductSpuDO selectByIdIncludeDeleted(@Param("id") Long id);
/**

View File

@ -113,6 +113,9 @@ public class AfterSaleLogAspect {
* @return 用户类型
*/
private static Long getUserId() {
if (USER_ID.get() != null) {
return USER_ID.get();
}
return ObjectUtil.defaultIfNull(WebFrameworkUtils.getLoginUserId(), TradeOrderLogDO.USER_ID_SYSTEM);
}

View File

@ -109,6 +109,9 @@ public class TradeOrderLogAspect {
* @return 用户类型
*/
private static Long getUserId() {
if (USER_ID.get() != null) {
return USER_ID.get();
}
return ObjectUtil.defaultIfNull(WebFrameworkUtils.getLoginUserId(), TradeOrderLogDO.USER_ID_SYSTEM);
}

View File

@ -51,7 +51,7 @@ public class PayWalletRechargeController {
public CommonResult<Boolean> updateWalletRechargeRefunded(@RequestBody PayRefundNotifyReqDTO notifyReqDTO) {
walletRechargeService.updateWalletRechargeRefunded(
Long.valueOf(notifyReqDTO.getMerchantOrderId()),
Long.valueOf(notifyReqDTO.getMerchantRefundId()),
notifyReqDTO.getMerchantRefundId(),
notifyReqDTO.getPayRefundId());
return success(true);
}

View File

@ -56,9 +56,9 @@ public interface PayWalletRechargeService {
* 更新钱包充值记录为已退款
*
* @param id 钱包充值记录编号
* @param refundId 钱包充值退款编号实际和 id 相同
* @param refundId 钱包充值退款编号格式{id}-refund
* @param payRefundId 退款单id
*/
void updateWalletRechargeRefunded(Long id, Long refundId, Long payRefundId);
void updateWalletRechargeRefunded(Long id, String refundId, Long payRefundId);
}

View File

@ -220,9 +220,8 @@ public class PayWalletRechargeServiceImpl implements PayWalletRechargeService {
@Override
@Transactional(rollbackFor = Exception.class)
public void updateWalletRechargeRefunded(Long id, Long refundId, Long payRefundId) {
public void updateWalletRechargeRefunded(Long id, String refundId, Long payRefundId) {
// 1.1 获取钱包充值记录
// 说明因为 id refundId 是相同的所以直接使用 id 查询即可
PayWalletRechargeDO walletRecharge = walletRechargeMapper.selectById(id);
if (walletRecharge == null) {
log.error("[updateWalletRechargerPaid][钱包充值记录不存在,钱包充值记录 id({})]", id);
@ -274,8 +273,8 @@ public class PayWalletRechargeServiceImpl implements PayWalletRechargeService {
walletRecharge.getId(), payRefundId, toJsonString(walletRecharge), toJsonString(payRefund));
throw exception(WALLET_RECHARGE_REFUND_FAIL_REFUND_PRICE_NOT_MATCH);
}
// 2.3 校验退款订单商户单是否匹配
if (notEqual(payRefund.getMerchantRefundId(), walletRecharge.getId().toString())) {
// 2.3 校验退款订单商户退款单是否匹配
if (notEqual(payRefund.getMerchantRefundId(), walletRecharge.getId() + "-refund")) {
log.error("[validateWalletRechargeCanRefunded][钱包({}) 退款单不匹配({})请进行处理payRefund 数据是:{}]",
walletRecharge.getId(), payRefundId, toJsonString(payRefund));
throw exception(WALLET_RECHARGE_REFUND_FAIL_REFUND_ORDER_ID_ERROR);

View File

@ -58,13 +58,22 @@ public class PayWalletServiceImpl implements PayWalletService {
private PayRefundService refundService;
@Override
@SneakyThrows
public PayWalletDO getOrCreateWallet(Long userId, Integer userType) {
PayWalletDO wallet = walletMapper.selectByUserIdAndType(userId, userType);
if (wallet == null) {
wallet = new PayWalletDO().setUserId(userId).setUserType(userType)
.setBalance(0).setTotalExpense(0).setTotalRecharge(0);
wallet.setCreateTime(LocalDateTime.now());
walletMapper.insert(wallet);
// 使用双重检查锁保证钱包创建并发问题
// https://gitee.com/zhijiantianya/ruoyi-vue-pro/pulls/1475/files
wallet = lockRedisDAO.lock(userId, UPDATE_TIMEOUT_MILLIS, () -> {
PayWalletDO newWallet = walletMapper.selectByUserIdAndType(userId, userType);
if (newWallet == null) {
newWallet = new PayWalletDO().setUserId(userId).setUserType(userType)
.setBalance(0).setTotalExpense(0).setTotalRecharge(0);
newWallet.setCreateTime(LocalDateTime.now());
walletMapper.insert(newWallet);
}
return newWallet;
});
}
return wallet;
}

View File

@ -64,6 +64,14 @@ public class PostController {
return success(true);
}
@DeleteMapping("delete-list")
@Operation(summary = "批量删除岗位")
@PreAuthorize("@ss.hasPermission('system:post:delete')")
public CommonResult<Boolean> deletePostList(@RequestParam("ids") List<Long> ids) {
postService.deletePostList(ids);
return success(true);
}
@GetMapping(value = "/get")
@Operation(summary = "获得岗位信息")
@Parameter(name = "id", description = "岗位编号", required = true, example = "1024")

View File

@ -13,6 +13,7 @@ import cn.iocoder.yudao.module.system.dal.dataobject.logger.OperateLogDO;
import cn.iocoder.yudao.module.system.service.logger.OperateLogService;
import com.fhs.core.trans.anno.TransMethodResult;
import io.swagger.v3.oas.annotations.Operation;
import io.swagger.v3.oas.annotations.Parameter;
import io.swagger.v3.oas.annotations.tags.Tag;
import jakarta.annotation.Resource;
import jakarta.servlet.http.HttpServletResponse;
@ -21,6 +22,7 @@ import org.springframework.security.access.prepost.PreAuthorize;
import org.springframework.validation.annotation.Validated;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.RestController;
import java.io.IOException;
@ -38,6 +40,15 @@ public class OperateLogController {
@Resource
private OperateLogService operateLogService;
@GetMapping("/get")
@Operation(summary = "查看操作日志")
@Parameter(name = "id", description = "编号", required = true, example = "1024")
@PreAuthorize("@ss.hasPermission('system:operate-log:query')")
public CommonResult<OperateLogRespVO> getOperateLog(@RequestParam("id") Long id) {
OperateLogDO operateLog = operateLogService.getOperateLog(id);
return success(BeanUtils.toBean(operateLog, OperateLogRespVO.class));
}
@GetMapping("/page")
@Operation(summary = "查看操作日志分页列表")
@PreAuthorize("@ss.hasPermission('system:operate-log:query')")

View File

@ -20,6 +20,14 @@ public interface OperateLogService {
*/
void createOperateLog(OperateLogCreateReqDTO createReqDTO);
/**
* 获得操作日志
*
* @param id 编号
* @return 操作日志
*/
OperateLogDO getOperateLog(Long id);
/**
* 获得操作日志分页列表
*

View File

@ -31,6 +31,11 @@ public class OperateLogServiceImpl implements OperateLogService {
operateLogMapper.insert(log);
}
@Override
public OperateLogDO getOperateLog(Long id) {
return operateLogMapper.selectById(id);
}
@Override
public PageResult<OperateLogDO> getOperateLogPage(OperateLogPageReqVO pageReqVO) {
return operateLogMapper.selectPage(pageReqVO);