mirror of
https://gitee.com/zhijiantianya/yudao-cloud.git
synced 2025-12-25 23:00:06 +08:00
Merge branch 'master-jdk17' of https://gitee.com/zhijiantianya/yudao-cloud
This commit is contained in:
commit
4017f71d10
@ -1,6 +1,7 @@
|
||||
package cn.iocoder.yudao.framework.tenant.core.redis;
|
||||
|
||||
import cn.hutool.core.collection.CollUtil;
|
||||
import cn.hutool.core.util.StrUtil;
|
||||
import cn.iocoder.yudao.framework.redis.core.TimeoutRedisCacheManager;
|
||||
import cn.iocoder.yudao.framework.tenant.core.context.TenantContextHolder;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
@ -21,6 +22,8 @@ import java.util.Set;
|
||||
@Slf4j
|
||||
public class TenantRedisCacheManager extends TimeoutRedisCacheManager {
|
||||
|
||||
private static final String SPLIT = "#";
|
||||
|
||||
private final Set<String> ignoreCaches;
|
||||
|
||||
public TenantRedisCacheManager(RedisCacheWriter cacheWriter,
|
||||
@ -32,10 +35,11 @@ public class TenantRedisCacheManager extends TimeoutRedisCacheManager {
|
||||
|
||||
@Override
|
||||
public Cache getCache(String name) {
|
||||
String[] names = StrUtil.splitToArray(name, SPLIT);
|
||||
// 如果开启多租户,则 name 拼接租户后缀
|
||||
if (!TenantContextHolder.isIgnore()
|
||||
&& TenantContextHolder.getTenantId() != null
|
||||
&& !CollUtil.contains(ignoreCaches, name)) {
|
||||
&& !CollUtil.contains(ignoreCaches, names[0])) {
|
||||
name = name + ":" + TenantContextHolder.getTenantId();
|
||||
}
|
||||
|
||||
@ -43,4 +47,4 @@ public class TenantRedisCacheManager extends TimeoutRedisCacheManager {
|
||||
return super.getCache(name);
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,53 @@
|
||||
package cn.iocoder.yudao.module.ai.enums;
|
||||
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Getter;
|
||||
|
||||
/**
|
||||
* AI 知识库文档切片策略枚举
|
||||
*
|
||||
* @author runzhen
|
||||
*/
|
||||
@AllArgsConstructor
|
||||
@Getter
|
||||
public enum AiDocumentSplitStrategyEnum {
|
||||
|
||||
/**
|
||||
* 自动识别文档类型并选择最佳切片策略
|
||||
*/
|
||||
AUTO("auto", "自动识别"),
|
||||
|
||||
/**
|
||||
* 基于 Token 数量机械切分(默认策略)
|
||||
*/
|
||||
TOKEN("token", "Token 切分"),
|
||||
|
||||
/**
|
||||
* 按段落切分(以双换行符为分隔)
|
||||
*/
|
||||
PARAGRAPH("paragraph", "段落切分"),
|
||||
|
||||
/**
|
||||
* Markdown QA 格式专用切片器
|
||||
* 识别二级标题作为问题,保持问答对完整性
|
||||
* 长答案智能切分但保留问题作为上下文
|
||||
*/
|
||||
MARKDOWN_QA("markdown_qa", "Markdown QA 切分"),
|
||||
|
||||
/**
|
||||
* 语义化切分,保留句子完整性
|
||||
* 在段落和句子边界处切分,避免截断
|
||||
*/
|
||||
SEMANTIC("semantic", "语义切分");
|
||||
|
||||
/**
|
||||
* 策略代码
|
||||
*/
|
||||
private final String code;
|
||||
|
||||
/**
|
||||
* 策略名称
|
||||
*/
|
||||
private final String name;
|
||||
|
||||
}
|
||||
@ -20,7 +20,8 @@
|
||||
</description>
|
||||
<properties>
|
||||
<spring-ai.version>1.1.0</spring-ai.version>
|
||||
<alibaba-ai.version>1.1.0.0-M5</alibaba-ai.version>
|
||||
<!-- https://mvnrepository.com/artifact/com.alibaba.cloud.ai/spring-ai-alibaba -->
|
||||
<alibaba-ai.version>1.1.0.0-RC1</alibaba-ai.version>
|
||||
<tinyflow.version>1.2.6</tinyflow.version>
|
||||
</properties>
|
||||
|
||||
@ -262,6 +263,11 @@
|
||||
<groupId>com.agentsflex</groupId>
|
||||
<artifactId>agents-flex-store-elasticsearch</artifactId>
|
||||
</exclusion>
|
||||
<exclusion>
|
||||
<!-- 解决 https://t.zsxq.com/pCBZC 问题 -->
|
||||
<groupId>com.agentsflex</groupId>
|
||||
<artifactId>agents-flex-search-engine-es</artifactId>
|
||||
</exclusion>
|
||||
<exclusion>
|
||||
<!-- TODO @芋艿:暂时移除 groovy,和 iot 冲突 -->
|
||||
<groupId>org.codehaus.groovy</groupId>
|
||||
|
||||
@ -11,7 +11,7 @@ import lombok.Data;
|
||||
public class AiKnowledgeSegmentPageReqVO extends PageParam {
|
||||
|
||||
@Schema(description = "文档编号", example = "1")
|
||||
private Integer documentId;
|
||||
private Long documentId;
|
||||
|
||||
@Schema(description = "分段内容关键字", example = "Java 开发")
|
||||
private String content;
|
||||
|
||||
@ -4,6 +4,7 @@ import cn.hutool.core.collection.CollUtil;
|
||||
import cn.hutool.core.collection.ListUtil;
|
||||
import cn.hutool.core.util.ObjUtil;
|
||||
import cn.hutool.core.util.StrUtil;
|
||||
|
||||
import cn.iocoder.yudao.framework.common.enums.CommonStatusEnum;
|
||||
import cn.iocoder.yudao.framework.common.pojo.PageResult;
|
||||
import cn.iocoder.yudao.framework.common.util.object.BeanUtils;
|
||||
@ -15,8 +16,11 @@ import cn.iocoder.yudao.module.ai.dal.dataobject.knowledge.AiKnowledgeDO;
|
||||
import cn.iocoder.yudao.module.ai.dal.dataobject.knowledge.AiKnowledgeDocumentDO;
|
||||
import cn.iocoder.yudao.module.ai.dal.dataobject.knowledge.AiKnowledgeSegmentDO;
|
||||
import cn.iocoder.yudao.module.ai.dal.mysql.knowledge.AiKnowledgeSegmentMapper;
|
||||
import cn.iocoder.yudao.module.ai.enums.AiDocumentSplitStrategyEnum;
|
||||
import cn.iocoder.yudao.module.ai.service.knowledge.bo.AiKnowledgeSegmentSearchReqBO;
|
||||
import cn.iocoder.yudao.module.ai.service.knowledge.bo.AiKnowledgeSegmentSearchRespBO;
|
||||
import cn.iocoder.yudao.module.ai.service.knowledge.splitter.MarkdownQaSplitter;
|
||||
import cn.iocoder.yudao.module.ai.service.knowledge.splitter.SemanticTextSplitter;
|
||||
import cn.iocoder.yudao.module.ai.service.model.AiModelService;
|
||||
import com.alibaba.cloud.ai.dashscope.rerank.DashScopeRerankOptions;
|
||||
import com.alibaba.cloud.ai.model.RerankModel;
|
||||
@ -39,8 +43,7 @@ import java.util.*;
|
||||
|
||||
import static cn.iocoder.yudao.framework.common.exception.util.ServiceExceptionUtil.exception;
|
||||
import static cn.iocoder.yudao.framework.common.util.collection.CollectionUtils.convertList;
|
||||
import static cn.iocoder.yudao.module.ai.enums.ErrorCodeConstants.KNOWLEDGE_SEGMENT_CONTENT_TOO_LONG;
|
||||
import static cn.iocoder.yudao.module.ai.enums.ErrorCodeConstants.KNOWLEDGE_SEGMENT_NOT_EXISTS;
|
||||
import static cn.iocoder.yudao.module.ai.enums.ErrorCodeConstants.*;
|
||||
import static org.springframework.ai.vectorstore.SearchRequest.SIMILARITY_THRESHOLD_ACCEPT_ALL;
|
||||
|
||||
/**
|
||||
@ -95,8 +98,9 @@ public class AiKnowledgeSegmentServiceImpl implements AiKnowledgeSegmentService
|
||||
AiKnowledgeDO knowledgeDO = knowledgeService.validateKnowledgeExists(documentDO.getKnowledgeId());
|
||||
VectorStore vectorStore = getVectorStoreById(knowledgeDO);
|
||||
|
||||
// 2. 文档切片
|
||||
List<Document> documentSegments = splitContentByToken(content, documentDO.getSegmentMaxTokens());
|
||||
// 2. 文档切片(使用自动检测策略)
|
||||
List<Document> documentSegments = splitContentByStrategy(content, documentDO.getSegmentMaxTokens(),
|
||||
AiDocumentSplitStrategyEnum.AUTO, documentDO.getUrl());
|
||||
|
||||
// 3.1 存储切片
|
||||
List<AiKnowledgeSegmentDO> segmentDOs = convertList(documentSegments, segment -> {
|
||||
@ -295,8 +299,10 @@ public class AiKnowledgeSegmentServiceImpl implements AiKnowledgeSegmentService
|
||||
// 1. 读取 URL 内容
|
||||
String content = knowledgeDocumentService.readUrl(url);
|
||||
|
||||
// 2. 文档切片
|
||||
List<Document> documentSegments = splitContentByToken(content, segmentMaxTokens);
|
||||
// 2.1 自动检测文档类型并选择策略
|
||||
AiDocumentSplitStrategyEnum strategy = detectDocumentStrategy(content, url);
|
||||
// 2.2 文档切片
|
||||
List<Document> documentSegments = splitContentByStrategy(content, segmentMaxTokens, strategy, url);
|
||||
|
||||
// 3. 转换为段落对象
|
||||
return convertList(documentSegments, segment -> {
|
||||
@ -333,11 +339,103 @@ public class AiKnowledgeSegmentServiceImpl implements AiKnowledgeSegmentService
|
||||
return getVectorStoreById(knowledge);
|
||||
}
|
||||
|
||||
private static List<Document> splitContentByToken(String content, Integer segmentMaxTokens) {
|
||||
TextSplitter textSplitter = buildTokenTextSplitter(segmentMaxTokens);
|
||||
/**
|
||||
* 根据策略切分内容
|
||||
*
|
||||
* @param content 文档内容
|
||||
* @param segmentMaxTokens 分段的最大 Token 数
|
||||
* @param strategy 切片策略
|
||||
* @param url 文档 URL(用于自动检测文件类型)
|
||||
* @return 切片后的文档列表
|
||||
*/
|
||||
@SuppressWarnings("EnhancedSwitchMigration")
|
||||
private List<Document> splitContentByStrategy(String content, Integer segmentMaxTokens,
|
||||
AiDocumentSplitStrategyEnum strategy, String url) {
|
||||
// 自动检测策略
|
||||
if (strategy == AiDocumentSplitStrategyEnum.AUTO) {
|
||||
strategy = detectDocumentStrategy(content, url);
|
||||
log.info("[splitContentByStrategy][自动检测到文档策略: {}]", strategy.getName());
|
||||
}
|
||||
// 根据策略切分
|
||||
TextSplitter textSplitter;
|
||||
switch (strategy) {
|
||||
case MARKDOWN_QA:
|
||||
textSplitter = new MarkdownQaSplitter(segmentMaxTokens);
|
||||
break;
|
||||
case SEMANTIC:
|
||||
textSplitter = new SemanticTextSplitter(segmentMaxTokens);
|
||||
break;
|
||||
case PARAGRAPH:
|
||||
textSplitter = new SemanticTextSplitter(segmentMaxTokens, 0); // 段落切分,无重叠
|
||||
break;
|
||||
case TOKEN:
|
||||
default:
|
||||
textSplitter = buildTokenTextSplitter(segmentMaxTokens);
|
||||
break;
|
||||
}
|
||||
// 执行切分
|
||||
return textSplitter.apply(Collections.singletonList(new Document(content)));
|
||||
}
|
||||
|
||||
/**
|
||||
* 自动检测文档类型并选择切片策略
|
||||
*
|
||||
* @param content 文档内容
|
||||
* @param url 文档 URL
|
||||
* @return 推荐的切片策略
|
||||
*/
|
||||
private AiDocumentSplitStrategyEnum detectDocumentStrategy(String content, String url) {
|
||||
if (StrUtil.isEmpty(content)) {
|
||||
return AiDocumentSplitStrategyEnum.TOKEN;
|
||||
}
|
||||
// 1. 检测 Markdown QA 格式
|
||||
if (isMarkdownQaFormat(content, url)) {
|
||||
return AiDocumentSplitStrategyEnum.MARKDOWN_QA;
|
||||
}
|
||||
// 2. 检测普通 Markdown 文档
|
||||
if (isMarkdownDocument(url)) {
|
||||
return AiDocumentSplitStrategyEnum.SEMANTIC;
|
||||
}
|
||||
// 3. 默认使用语义切分(比 Token 切分更智能)
|
||||
return AiDocumentSplitStrategyEnum.SEMANTIC;
|
||||
}
|
||||
|
||||
/**
|
||||
* 检测是否为 Markdown QA 格式
|
||||
* 特征:包含多个二级标题(## )且标题后紧跟答案内容
|
||||
*/
|
||||
private boolean isMarkdownQaFormat(String content, String url) {
|
||||
// 文件扩展名判断
|
||||
if (StrUtil.isNotEmpty(url) && !url.toLowerCase().endsWith(".md")) {
|
||||
return false;
|
||||
}
|
||||
|
||||
// 统计二级标题数量
|
||||
long h2Count = content.lines()
|
||||
.filter(line -> line.trim().startsWith("## "))
|
||||
.count();
|
||||
|
||||
// 要求一:至少包含 2 个二级标题才认为是 QA 格式
|
||||
if (h2Count < 2) {
|
||||
return false;
|
||||
}
|
||||
|
||||
// 要求二:检查标题占比(QA 文档标题行数相对较多),如果二级标题占比超过 10%,认为是 QA 格式
|
||||
long totalLines = content.lines().count();
|
||||
double h2Ratio = (double) h2Count / totalLines;
|
||||
return h2Ratio > 0.1;
|
||||
}
|
||||
|
||||
/**
|
||||
* 检测是否为 Markdown 文档
|
||||
*/
|
||||
private boolean isMarkdownDocument(String url) {
|
||||
return StrUtil.endWithAnyIgnoreCase(url, ".md", ".markdown");
|
||||
}
|
||||
|
||||
/**
|
||||
* 构建基于 Token 的文本切片器(原有逻辑保留)
|
||||
*/
|
||||
private static TextSplitter buildTokenTextSplitter(Integer segmentMaxTokens) {
|
||||
return TokenTextSplitter.builder()
|
||||
.withChunkSize(segmentMaxTokens)
|
||||
|
||||
@ -0,0 +1,342 @@
|
||||
package cn.iocoder.yudao.module.ai.service.knowledge.splitter;
|
||||
|
||||
import cn.hutool.core.collection.CollUtil;
|
||||
import cn.hutool.core.util.StrUtil;
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.ai.transformer.splitter.TextSplitter;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collections;
|
||||
import java.util.List;
|
||||
import java.util.regex.Matcher;
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
/**
|
||||
* Markdown QA 格式专用切片器
|
||||
*
|
||||
* <p>功能特点:
|
||||
* <ul>
|
||||
* <li>识别二级标题(## )作为问题标记</li>
|
||||
* <li>短 QA 对保持完整(不超过 Token 限制)</li>
|
||||
* <li>长答案智能切分,每个片段保留完整问题作为上下文</li>
|
||||
* <li>支持自定义 Token 估算器</li>
|
||||
* </ul>
|
||||
*
|
||||
* @author runzhen
|
||||
*/
|
||||
@Slf4j
|
||||
@SuppressWarnings("SizeReplaceableByIsEmpty")
|
||||
public class MarkdownQaSplitter extends TextSplitter {
|
||||
|
||||
/**
|
||||
* 二级标题正则:匹配 "## " 开头的行
|
||||
*/
|
||||
private static final Pattern H2_PATTERN = Pattern.compile("^##\\s+(.+)$", Pattern.MULTILINE);
|
||||
|
||||
/**
|
||||
* 段落分隔符:双换行
|
||||
*/
|
||||
private static final String PARAGRAPH_SEPARATOR = "\n\n";
|
||||
|
||||
/**
|
||||
* 句子分隔符
|
||||
*/
|
||||
private static final Pattern SENTENCE_PATTERN = Pattern.compile("[。!?.!?]\\s*");
|
||||
|
||||
/**
|
||||
* 分段的最大 Token 数
|
||||
*/
|
||||
private final int chunkSize;
|
||||
|
||||
/**
|
||||
* Token 估算器(简单实现:中文按字符数,英文按单词数的 1.3 倍)
|
||||
*/
|
||||
private final TokenEstimator tokenEstimator;
|
||||
|
||||
public MarkdownQaSplitter(int chunkSize) {
|
||||
this.chunkSize = chunkSize;
|
||||
this.tokenEstimator = new SimpleTokenEstimator();
|
||||
}
|
||||
|
||||
@Override
|
||||
protected List<String> splitText(String text) {
|
||||
if (StrUtil.isEmpty(text)) {
|
||||
return Collections.emptyList();
|
||||
}
|
||||
|
||||
// 解析 QA 对
|
||||
List<QaPair> qaPairs = parseQaPairs(text);
|
||||
if (CollUtil.isEmpty(qaPairs)) {
|
||||
// 如果没有识别到 QA 格式,按段落切分
|
||||
return fallbackSplit(text);
|
||||
}
|
||||
|
||||
// 处理每个 QA 对
|
||||
List<String> result = new ArrayList<>();
|
||||
for (QaPair qaPair : qaPairs) {
|
||||
result.addAll(splitQaPair(qaPair));
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* 解析 Markdown QA 对
|
||||
*
|
||||
* @param content 文本内容
|
||||
* @return QA 对列表
|
||||
*/
|
||||
private List<QaPair> parseQaPairs(String content) {
|
||||
// 找到所有二级标题位置
|
||||
List<QaPair> qaPairs = new ArrayList<>();
|
||||
List<Integer> headingPositions = new ArrayList<>();
|
||||
List<String> questions = new ArrayList<>();
|
||||
Matcher matcher = H2_PATTERN.matcher(content);
|
||||
while (matcher.find()) {
|
||||
headingPositions.add(matcher.start());
|
||||
questions.add(matcher.group(1).trim());
|
||||
}
|
||||
if (CollUtil.isEmpty(headingPositions)) {
|
||||
return qaPairs;
|
||||
}
|
||||
|
||||
// 提取每个 QA 对
|
||||
for (int i = 0; i < headingPositions.size(); i++) {
|
||||
int start = headingPositions.get(i);
|
||||
int end = (i + 1 < headingPositions.size())
|
||||
? headingPositions.get(i + 1)
|
||||
: content.length();
|
||||
String qaText = content.substring(start, end).trim();
|
||||
String question = questions.get(i);
|
||||
// 提取答案部分(去掉问题标题)
|
||||
String answer = qaText.substring(qaText.indexOf('\n') + 1).trim();
|
||||
qaPairs.add(new QaPair(question, answer, qaText));
|
||||
}
|
||||
return qaPairs;
|
||||
}
|
||||
|
||||
/**
|
||||
* 切分单个 QA 对
|
||||
*
|
||||
* @param qaPair QA 对
|
||||
* @return 切分后的文本片段列表
|
||||
*/
|
||||
private List<String> splitQaPair(QaPair qaPair) {
|
||||
// 如果整个 QA 对不超过限制,保持完整
|
||||
List<String> chunks = new ArrayList<>();
|
||||
String fullQa = qaPair.fullText;
|
||||
int qaTokens = tokenEstimator.estimate(fullQa);
|
||||
if (qaTokens <= chunkSize) {
|
||||
chunks.add(fullQa);
|
||||
return chunks;
|
||||
}
|
||||
|
||||
// 长答案需要切分
|
||||
log.debug("QA 对超过 Token 限制 ({} > {}),开始智能切分: {}", qaTokens, chunkSize, qaPair.question);
|
||||
List<String> answerChunks = splitLongAnswer(qaPair.answer, qaPair.question);
|
||||
for (String answerChunk : answerChunks) {
|
||||
// 每个片段都包含完整问题
|
||||
String chunkText = "## " + qaPair.question + "\n" + answerChunk;
|
||||
chunks.add(chunkText);
|
||||
}
|
||||
return chunks;
|
||||
}
|
||||
|
||||
/**
|
||||
* 切分长答案
|
||||
*
|
||||
* @param answer 答案文本
|
||||
* @param question 问题文本
|
||||
* @return 切分后的答案片段列表
|
||||
*/
|
||||
private List<String> splitLongAnswer(String answer, String question) {
|
||||
List<String> chunks = new ArrayList<>();
|
||||
// 预留问题的 Token 空间
|
||||
String questionHeader = "## " + question + "\n";
|
||||
int questionTokens = tokenEstimator.estimate(questionHeader);
|
||||
int availableTokens = chunkSize - questionTokens - 10; // 预留 10 个 Token 的缓冲
|
||||
|
||||
// 先按段落切分
|
||||
String[] paragraphs = answer.split(PARAGRAPH_SEPARATOR);
|
||||
StringBuilder currentChunk = new StringBuilder();
|
||||
int currentTokens = 0;
|
||||
for (String paragraph : paragraphs) {
|
||||
if (StrUtil.isEmpty(paragraph)) {
|
||||
continue;
|
||||
}
|
||||
int paragraphTokens = tokenEstimator.estimate(paragraph);
|
||||
// 如果单个段落就超过限制,需要按句子切分
|
||||
if (paragraphTokens > availableTokens) {
|
||||
// 先保存当前块
|
||||
if (currentChunk.length() > 0) {
|
||||
chunks.add(currentChunk.toString().trim());
|
||||
currentChunk = new StringBuilder();
|
||||
currentTokens = 0;
|
||||
}
|
||||
// 按句子切分长段落
|
||||
chunks.addAll(splitLongParagraph(paragraph, availableTokens));
|
||||
continue;
|
||||
}
|
||||
// 如果加上这个段落会超过限制
|
||||
if (currentTokens + paragraphTokens > availableTokens && currentChunk.length() > 0) {
|
||||
chunks.add(currentChunk.toString().trim());
|
||||
currentChunk = new StringBuilder();
|
||||
currentTokens = 0;
|
||||
}
|
||||
if (currentChunk.length() > 0) {
|
||||
currentChunk.append("\n\n");
|
||||
}
|
||||
// 添加段落
|
||||
currentChunk.append(paragraph);
|
||||
currentTokens += paragraphTokens;
|
||||
}
|
||||
|
||||
// 添加最后一块
|
||||
if (currentChunk.length() > 0) {
|
||||
chunks.add(currentChunk.toString().trim());
|
||||
}
|
||||
return CollUtil.isEmpty(chunks) ? Collections.singletonList(answer) : chunks;
|
||||
}
|
||||
|
||||
/**
|
||||
* 切分长段落(按句子)
|
||||
*
|
||||
* @param paragraph 段落文本
|
||||
* @param availableTokens 可用的 Token 数
|
||||
* @return 切分后的文本片段列表
|
||||
*/
|
||||
private List<String> splitLongParagraph(String paragraph, int availableTokens) {
|
||||
// 按句子切分
|
||||
List<String> chunks = new ArrayList<>();
|
||||
String[] sentences = SENTENCE_PATTERN.split(paragraph);
|
||||
|
||||
// 按句子累积切分
|
||||
StringBuilder currentChunk = new StringBuilder();
|
||||
int currentTokens = 0;
|
||||
for (String sentence : sentences) {
|
||||
if (StrUtil.isEmpty(sentence)) {
|
||||
continue;
|
||||
}
|
||||
int sentenceTokens = tokenEstimator.estimate(sentence);
|
||||
// 如果单个句子就超过限制,强制切分
|
||||
if (sentenceTokens > availableTokens) {
|
||||
if (currentChunk.length() > 0) {
|
||||
chunks.add(currentChunk.toString().trim());
|
||||
currentChunk = new StringBuilder();
|
||||
currentTokens = 0;
|
||||
}
|
||||
chunks.add(sentence.trim());
|
||||
continue;
|
||||
}
|
||||
// 如果加上这个句子会超过限制
|
||||
if (currentTokens + sentenceTokens > availableTokens && currentChunk.length() > 0) {
|
||||
chunks.add(currentChunk.toString().trim());
|
||||
currentChunk = new StringBuilder();
|
||||
currentTokens = 0;
|
||||
}
|
||||
// 添加句子
|
||||
currentChunk.append(sentence);
|
||||
currentTokens += sentenceTokens;
|
||||
}
|
||||
|
||||
// 添加最后一块
|
||||
if (currentChunk.length() > 0) {
|
||||
chunks.add(currentChunk.toString().trim());
|
||||
}
|
||||
return chunks.isEmpty() ? Collections.singletonList(paragraph) : chunks;
|
||||
}
|
||||
|
||||
/**
|
||||
* 降级切分策略(当未识别到 QA 格式时)
|
||||
*
|
||||
* @param content 文本内容
|
||||
* @return 切分后的文本片段列表
|
||||
*/
|
||||
private List<String> fallbackSplit(String content) {
|
||||
// 按段落切分
|
||||
List<String> chunks = new ArrayList<>();
|
||||
String[] paragraphs = content.split(PARAGRAPH_SEPARATOR);
|
||||
|
||||
// 按段落累积切分
|
||||
StringBuilder currentChunk = new StringBuilder();
|
||||
int currentTokens = 0;
|
||||
for (String paragraph : paragraphs) {
|
||||
if (StrUtil.isEmpty(paragraph)) {
|
||||
continue;
|
||||
}
|
||||
int paragraphTokens = tokenEstimator.estimate(paragraph);
|
||||
// 如果加上这个段落会超过限制
|
||||
if (currentTokens + paragraphTokens > chunkSize && currentChunk.length() > 0) {
|
||||
chunks.add(currentChunk.toString().trim());
|
||||
currentChunk = new StringBuilder();
|
||||
currentTokens = 0;
|
||||
}
|
||||
// 添加段落
|
||||
if (currentChunk.length() > 0) {
|
||||
currentChunk.append("\n\n");
|
||||
}
|
||||
currentChunk.append(paragraph);
|
||||
currentTokens += paragraphTokens;
|
||||
}
|
||||
|
||||
// 添加最后一块
|
||||
if (currentChunk.length() > 0) {
|
||||
chunks.add(currentChunk.toString().trim());
|
||||
}
|
||||
return chunks.isEmpty() ? Collections.singletonList(content) : chunks;
|
||||
}
|
||||
|
||||
/**
|
||||
* QA 对数据结构
|
||||
*/
|
||||
@AllArgsConstructor
|
||||
private static class QaPair {
|
||||
|
||||
String question;
|
||||
String answer;
|
||||
String fullText;
|
||||
|
||||
}
|
||||
|
||||
/**
|
||||
* Token 估算器接口
|
||||
*/
|
||||
public interface TokenEstimator {
|
||||
|
||||
int estimate(String text);
|
||||
|
||||
}
|
||||
|
||||
/**
|
||||
* 简单的 Token 估算器实现
|
||||
* 中文:1 字符 ≈ 1 Token
|
||||
* 英文:1 单词 ≈ 1.3 Token
|
||||
*/
|
||||
private static class SimpleTokenEstimator implements TokenEstimator {
|
||||
|
||||
@Override
|
||||
public int estimate(String text) {
|
||||
if (StrUtil.isEmpty(text)) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
int chineseChars = 0;
|
||||
int englishWords = 0;
|
||||
// 简单统计中英文
|
||||
for (char c : text.toCharArray()) {
|
||||
if (c >= 0x4E00 && c <= 0x9FA5) {
|
||||
chineseChars++;
|
||||
}
|
||||
}
|
||||
// 英文单词估算
|
||||
String[] words = text.split("\\s+");
|
||||
for (String word : words) {
|
||||
if (word.matches(".*[a-zA-Z].*")) {
|
||||
englishWords++;
|
||||
}
|
||||
}
|
||||
return chineseChars + (int) (englishWords * 1.3);
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
@ -0,0 +1,301 @@
|
||||
package cn.iocoder.yudao.module.ai.service.knowledge.splitter;
|
||||
|
||||
import cn.hutool.core.util.StrUtil;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.ai.transformer.splitter.TextSplitter;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.Arrays;
|
||||
import java.util.Collections;
|
||||
import java.util.List;
|
||||
import java.util.regex.Matcher;
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
/**
|
||||
* 语义化文本切片器
|
||||
*
|
||||
* <p>功能特点:
|
||||
* <ul>
|
||||
* <li>优先在段落边界(双换行)处切分</li>
|
||||
* <li>其次在句子边界(句号、问号、感叹号)处切分</li>
|
||||
* <li>避免在句子中间截断,保持语义完整性</li>
|
||||
* <li>支持中英文标点符号识别</li>
|
||||
* </ul>
|
||||
*
|
||||
* @author runzhen
|
||||
*/
|
||||
@Slf4j
|
||||
public class SemanticTextSplitter extends TextSplitter {
|
||||
|
||||
/**
|
||||
* 分段的最大 Token 数
|
||||
*/
|
||||
private final int chunkSize;
|
||||
|
||||
/**
|
||||
* 段落重叠大小(用于保持上下文连贯性)
|
||||
*/
|
||||
private final int chunkOverlap;
|
||||
|
||||
/**
|
||||
* 段落分隔符(按优先级排序)
|
||||
*/
|
||||
private static final List<String> PARAGRAPH_SEPARATORS = Arrays.asList(
|
||||
"\n\n\n", // 三个换行
|
||||
"\n\n", // 双换行
|
||||
"\n" // 单换行
|
||||
);
|
||||
|
||||
/**
|
||||
* 句子结束标记(中英文标点)
|
||||
*/
|
||||
private static final Pattern SENTENCE_END_PATTERN = Pattern.compile(
|
||||
"[。!?.!?]+[\\s\"'))】\\]]*"
|
||||
);
|
||||
|
||||
/**
|
||||
* Token 估算器
|
||||
*/
|
||||
private final MarkdownQaSplitter.TokenEstimator tokenEstimator;
|
||||
|
||||
public SemanticTextSplitter(int chunkSize, int chunkOverlap) {
|
||||
this.chunkSize = chunkSize;
|
||||
this.chunkOverlap = Math.min(chunkOverlap, chunkSize / 2); // 重叠不超过一半
|
||||
this.tokenEstimator = new SimpleTokenEstimator();
|
||||
}
|
||||
|
||||
public SemanticTextSplitter(int chunkSize) {
|
||||
this(chunkSize, 50); // 默认重叠 50 个 Token
|
||||
}
|
||||
|
||||
@Override
|
||||
protected List<String> splitText(String text) {
|
||||
if (StrUtil.isEmpty(text)) {
|
||||
return Collections.emptyList();
|
||||
}
|
||||
return splitTextRecursive(text);
|
||||
}
|
||||
|
||||
/**
|
||||
* 切分文本(递归策略)
|
||||
*
|
||||
* @param text 待切分文本
|
||||
* @return 切分后的文本块列表
|
||||
*/
|
||||
private List<String> splitTextRecursive(String text) {
|
||||
List<String> chunks = new ArrayList<>();
|
||||
|
||||
// 如果文本不超过限制,直接返回
|
||||
int textTokens = tokenEstimator.estimate(text);
|
||||
if (textTokens <= chunkSize) {
|
||||
chunks.add(text.trim());
|
||||
return chunks;
|
||||
}
|
||||
|
||||
// 尝试按不同分隔符切分
|
||||
List<String> splits = null;
|
||||
String usedSeparator = null;
|
||||
for (String separator : PARAGRAPH_SEPARATORS) {
|
||||
if (text.contains(separator)) {
|
||||
splits = Arrays.asList(text.split(Pattern.quote(separator)));
|
||||
usedSeparator = separator;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
// 如果没有找到段落分隔符,按句子切分
|
||||
if (splits == null || splits.size() == 1) {
|
||||
splits = splitBySentences(text);
|
||||
usedSeparator = ""; // 句子切分不需要分隔符
|
||||
}
|
||||
|
||||
// 合并小片段
|
||||
chunks = mergeSplits(splits, usedSeparator);
|
||||
return chunks;
|
||||
}
|
||||
|
||||
/**
|
||||
* 按句子切分
|
||||
*
|
||||
* @param text 待切分文本
|
||||
* @return 句子列表
|
||||
*/
|
||||
private List<String> splitBySentences(String text) {
|
||||
// 使用正则表达式匹配句子结束位置
|
||||
List<String> sentences = new ArrayList<>();
|
||||
int lastEnd = 0;
|
||||
Matcher matcher = SENTENCE_END_PATTERN.matcher(text);
|
||||
while (matcher.find()) {
|
||||
String sentence = text.substring(lastEnd, matcher.end()).trim();
|
||||
if (StrUtil.isNotEmpty(sentence)) {
|
||||
sentences.add(sentence);
|
||||
}
|
||||
lastEnd = matcher.end();
|
||||
}
|
||||
|
||||
// 添加剩余部分
|
||||
if (lastEnd < text.length()) {
|
||||
String remaining = text.substring(lastEnd).trim();
|
||||
if (StrUtil.isNotEmpty(remaining)) {
|
||||
sentences.add(remaining);
|
||||
}
|
||||
}
|
||||
return sentences.isEmpty() ? Collections.singletonList(text) : sentences;
|
||||
}
|
||||
|
||||
/**
|
||||
* 合并切分后的小片段
|
||||
*
|
||||
* @param splits 切分后的片段列表
|
||||
* @param separator 片段间的分隔符
|
||||
* @return 合并后的文本块列表
|
||||
*/
|
||||
private List<String> mergeSplits(List<String> splits, String separator) {
|
||||
List<String> chunks = new ArrayList<>();
|
||||
List<String> currentChunks = new ArrayList<>();
|
||||
int currentLength = 0;
|
||||
|
||||
for (String split : splits) {
|
||||
if (StrUtil.isEmpty(split)) {
|
||||
continue;
|
||||
}
|
||||
int splitTokens = tokenEstimator.estimate(split);
|
||||
// 如果单个片段就超过限制,进一步递归切分
|
||||
if (splitTokens > chunkSize) {
|
||||
// 先保存当前累积的块
|
||||
if (!currentChunks.isEmpty()) {
|
||||
String chunkText = String.join(separator, currentChunks);
|
||||
chunks.add(chunkText.trim());
|
||||
currentChunks.clear();
|
||||
currentLength = 0;
|
||||
}
|
||||
// 递归切分大片段
|
||||
if (!separator.isEmpty()) {
|
||||
// 如果是段落分隔符,尝试按句子切分
|
||||
chunks.addAll(splitTextRecursive(split));
|
||||
} else {
|
||||
// 如果已经是句子级别,强制按字符切分
|
||||
chunks.addAll(forceSplitLongText(split));
|
||||
}
|
||||
continue;
|
||||
}
|
||||
// 计算加上分隔符的 Token 数
|
||||
int separatorTokens = StrUtil.isEmpty(separator) ? 0 : tokenEstimator.estimate(separator);
|
||||
// 如果加上这个片段会超过限制
|
||||
if (!currentChunks.isEmpty() && currentLength + splitTokens + separatorTokens > chunkSize) {
|
||||
// 保存当前块
|
||||
String chunkText = String.join(separator, currentChunks);
|
||||
chunks.add(chunkText.trim());
|
||||
|
||||
// 处理重叠:保留最后几个片段
|
||||
currentChunks = getOverlappingChunks(currentChunks, separator);
|
||||
currentLength = estimateTokens(currentChunks, separator);
|
||||
}
|
||||
// 添加当前片段
|
||||
currentChunks.add(split);
|
||||
currentLength += splitTokens + separatorTokens;
|
||||
}
|
||||
|
||||
// 添加最后一块
|
||||
if (!currentChunks.isEmpty()) {
|
||||
String chunkText = String.join(separator, currentChunks);
|
||||
chunks.add(chunkText.trim());
|
||||
}
|
||||
return chunks;
|
||||
}
|
||||
|
||||
/**
|
||||
* 获取重叠的片段(用于保持上下文)
|
||||
*
|
||||
* @param chunks 当前片段列表
|
||||
* @param separator 片段间的分隔符
|
||||
* @return 重叠的片段列表
|
||||
*/
|
||||
private List<String> getOverlappingChunks(List<String> chunks, String separator) {
|
||||
if (chunkOverlap == 0 || chunks.isEmpty()) {
|
||||
return new ArrayList<>();
|
||||
}
|
||||
|
||||
// 从后往前取片段,直到达到重叠大小
|
||||
List<String> overlapping = new ArrayList<>();
|
||||
int tokens = 0;
|
||||
for (int i = chunks.size() - 1; i >= 0; i--) {
|
||||
String chunk = chunks.get(i);
|
||||
int chunkTokens = tokenEstimator.estimate(chunk);
|
||||
if (tokens + chunkTokens > chunkOverlap) {
|
||||
break;
|
||||
}
|
||||
// 添加到重叠列表前端
|
||||
overlapping.add(0, chunk);
|
||||
tokens += chunkTokens + (StrUtil.isEmpty(separator) ? 0 : tokenEstimator.estimate(separator));
|
||||
}
|
||||
return overlapping;
|
||||
}
|
||||
|
||||
/**
|
||||
* 估算片段列表的总 Token 数
|
||||
*
|
||||
* @param chunks 片段列表
|
||||
* @param separator 片段间的分隔符
|
||||
* @return 总 Token 数
|
||||
*/
|
||||
private int estimateTokens(List<String> chunks, String separator) {
|
||||
int total = 0;
|
||||
for (int i = 0; i < chunks.size(); i++) {
|
||||
total += tokenEstimator.estimate(chunks.get(i));
|
||||
if (i < chunks.size() - 1 && StrUtil.isNotEmpty(separator)) {
|
||||
total += tokenEstimator.estimate(separator);
|
||||
}
|
||||
}
|
||||
return total;
|
||||
}
|
||||
|
||||
/**
|
||||
* 强制切分长文本(当语义切分失败时)
|
||||
*
|
||||
* @param text 待切分文本
|
||||
* @return 切分后的文本块列表
|
||||
*/
|
||||
private List<String> forceSplitLongText(String text) {
|
||||
List<String> chunks = new ArrayList<>();
|
||||
int charsPerChunk = (int) (chunkSize * 0.8); // 保守估计
|
||||
for (int i = 0; i < text.length(); i += charsPerChunk) {
|
||||
int end = Math.min(i + charsPerChunk, text.length());
|
||||
String chunk = text.substring(i, end);
|
||||
chunks.add(chunk.trim());
|
||||
}
|
||||
log.warn("文本过长,已强制按字符切分,可能影响语义完整性");
|
||||
return chunks;
|
||||
}
|
||||
|
||||
/**
|
||||
* 简单的 Token 估算器实现
|
||||
*/
|
||||
private static class SimpleTokenEstimator implements MarkdownQaSplitter.TokenEstimator {
|
||||
|
||||
@Override
|
||||
public int estimate(String text) {
|
||||
if (StrUtil.isEmpty(text)) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
int chineseChars = 0;
|
||||
int englishWords = 0;
|
||||
// 简单统计中英文
|
||||
for (char c : text.toCharArray()) {
|
||||
if (c >= 0x4E00 && c <= 0x9FA5) {
|
||||
chineseChars++;
|
||||
}
|
||||
}
|
||||
// 英文单词估算
|
||||
String[] words = text.split("\\s+");
|
||||
for (String word : words) {
|
||||
if (word.matches(".*[a-zA-Z].*")) {
|
||||
englishWords++;
|
||||
}
|
||||
}
|
||||
return chineseChars + (int) (englishWords * 1.3);
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
@ -13,6 +13,7 @@ import cn.iocoder.yudao.module.infra.service.file.FileService;
|
||||
import io.swagger.v3.oas.annotations.Operation;
|
||||
import io.swagger.v3.oas.annotations.Parameter;
|
||||
import io.swagger.v3.oas.annotations.Parameters;
|
||||
import io.swagger.v3.oas.annotations.media.Schema;
|
||||
import io.swagger.v3.oas.annotations.tags.Tag;
|
||||
import javax.annotation.Resource;
|
||||
import javax.annotation.security.PermitAll;
|
||||
@ -44,6 +45,8 @@ public class FileController {
|
||||
|
||||
@PostMapping("/upload")
|
||||
@Operation(summary = "上传文件", description = "模式一:后端上传文件")
|
||||
@Parameter(name = "file", description = "文件附件", required = true,
|
||||
schema = @Schema(type = "string", format = "binary"))
|
||||
public CommonResult<String> uploadFile(@Valid FileUploadReqVO uploadReqVO) throws Exception {
|
||||
MultipartFile file = uploadReqVO.getFile();
|
||||
byte[] content = IoUtil.readBytes(file.getInputStream());
|
||||
|
||||
@ -23,7 +23,14 @@ public class FileUploadReqVO {
|
||||
@AssertTrue(message = "文件目录不正确")
|
||||
@JsonIgnore
|
||||
public boolean isDirectoryValid() {
|
||||
return !StrUtil.containsAny(directory, "..", "/", "\\");
|
||||
return isDirectoryValid(directory);
|
||||
}
|
||||
|
||||
public static boolean isDirectoryValid(String directory) {
|
||||
// 1. 不能包含 .. 防止目录穿越
|
||||
// 2. 不能以 / 或 \ 开头,防止上传到根目录
|
||||
return !StrUtil.contains(directory, "..")
|
||||
&& !StrUtil.startWithAny(directory, "/", "\\");
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@ -11,11 +11,13 @@ import cn.iocoder.yudao.module.infra.controller.admin.logger.vo.apiaccesslog.Api
|
||||
import cn.iocoder.yudao.module.infra.dal.dataobject.logger.ApiAccessLogDO;
|
||||
import cn.iocoder.yudao.module.infra.service.logger.ApiAccessLogService;
|
||||
import io.swagger.v3.oas.annotations.Operation;
|
||||
import io.swagger.v3.oas.annotations.Parameter;
|
||||
import io.swagger.v3.oas.annotations.tags.Tag;
|
||||
import org.springframework.security.access.prepost.PreAuthorize;
|
||||
import org.springframework.validation.annotation.Validated;
|
||||
import org.springframework.web.bind.annotation.GetMapping;
|
||||
import org.springframework.web.bind.annotation.RequestMapping;
|
||||
import org.springframework.web.bind.annotation.RequestParam;
|
||||
import org.springframework.web.bind.annotation.RestController;
|
||||
|
||||
import javax.annotation.Resource;
|
||||
@ -36,6 +38,15 @@ public class ApiAccessLogController {
|
||||
@Resource
|
||||
private ApiAccessLogService apiAccessLogService;
|
||||
|
||||
@GetMapping("/get")
|
||||
@Operation(summary = "获得 API 访问日志")
|
||||
@Parameter(name = "id", description = "编号", required = true, example = "1024")
|
||||
@PreAuthorize("@ss.hasPermission('infra:api-access-log:query')")
|
||||
public CommonResult<ApiAccessLogRespVO> getApiAccessLog(@RequestParam("id") Long id) {
|
||||
ApiAccessLogDO apiAccessLog = apiAccessLogService.getApiAccessLog(id);
|
||||
return success(BeanUtils.toBean(apiAccessLog, ApiAccessLogRespVO.class));
|
||||
}
|
||||
|
||||
@GetMapping("/page")
|
||||
@Operation(summary = "获得API 访问日志分页")
|
||||
@PreAuthorize("@ss.hasPermission('infra:api-access-log:query')")
|
||||
|
||||
@ -50,6 +50,15 @@ public class ApiErrorLogController {
|
||||
return success(true);
|
||||
}
|
||||
|
||||
@GetMapping("/get")
|
||||
@Operation(summary = "获得 API 错误日志")
|
||||
@Parameter(name = "id", description = "编号", required = true, example = "1024")
|
||||
@PreAuthorize("@ss.hasPermission('infra:api-error-log:query')")
|
||||
public CommonResult<ApiErrorLogRespVO> getApiErrorLog(@RequestParam("id") Long id) {
|
||||
ApiErrorLogDO apiErrorLog = apiErrorLogService.getApiErrorLog(id);
|
||||
return success(BeanUtils.toBean(apiErrorLog, ApiErrorLogRespVO.class));
|
||||
}
|
||||
|
||||
@GetMapping("/page")
|
||||
@Operation(summary = "获得 API 错误日志分页")
|
||||
@PreAuthorize("@ss.hasPermission('infra:api-error-log:query')")
|
||||
@ -63,7 +72,7 @@ public class ApiErrorLogController {
|
||||
@PreAuthorize("@ss.hasPermission('infra:api-error-log:export')")
|
||||
@ApiAccessLog(operateType = EXPORT)
|
||||
public void exportApiErrorLogExcel(@Valid ApiErrorLogPageReqVO exportReqVO,
|
||||
HttpServletResponse response) throws IOException {
|
||||
HttpServletResponse response) throws IOException {
|
||||
exportReqVO.setPageSize(PageParam.PAGE_SIZE_NONE);
|
||||
List<ApiErrorLogDO> list = apiErrorLogService.getApiErrorLogPage(exportReqVO).getList();
|
||||
// 导出 Excel
|
||||
|
||||
@ -9,6 +9,7 @@ import cn.iocoder.yudao.module.infra.service.file.FileService;
|
||||
import io.swagger.v3.oas.annotations.Operation;
|
||||
import io.swagger.v3.oas.annotations.Parameter;
|
||||
import io.swagger.v3.oas.annotations.Parameters;
|
||||
import io.swagger.v3.oas.annotations.media.Schema;
|
||||
import io.swagger.v3.oas.annotations.tags.Tag;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.validation.annotation.Validated;
|
||||
@ -33,6 +34,8 @@ public class AppFileController {
|
||||
|
||||
@PostMapping("/upload")
|
||||
@Operation(summary = "上传文件")
|
||||
@Parameter(name = "file", description = "文件附件", required = true,
|
||||
schema = @Schema(type = "string", format = "binary"))
|
||||
@PermitAll
|
||||
public CommonResult<String> uploadFile(AppFileUploadReqVO uploadReqVO) throws Exception {
|
||||
MultipartFile file = uploadReqVO.getFile();
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
package cn.iocoder.yudao.module.infra.controller.app.file.vo;
|
||||
|
||||
import cn.hutool.core.util.StrUtil;
|
||||
import cn.iocoder.yudao.module.infra.controller.admin.file.vo.file.FileUploadReqVO;
|
||||
import com.fasterxml.jackson.annotation.JsonIgnore;
|
||||
import io.swagger.v3.oas.annotations.media.Schema;
|
||||
import lombok.Data;
|
||||
@ -23,7 +23,7 @@ public class AppFileUploadReqVO {
|
||||
@AssertTrue(message = "文件目录不正确")
|
||||
@JsonIgnore
|
||||
public boolean isDirectoryValid() {
|
||||
return !StrUtil.containsAny(directory, "..", "/", "\\");
|
||||
return FileUploadReqVO.isDirectoryValid(directory);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@ -19,6 +19,14 @@ public interface ApiAccessLogService {
|
||||
*/
|
||||
void createApiAccessLog(ApiAccessLogCreateReqDTO createReqDTO);
|
||||
|
||||
/**
|
||||
* 获得 API 访问日志
|
||||
*
|
||||
* @param id 编号
|
||||
* @return API 访问日志
|
||||
*/
|
||||
ApiAccessLogDO getApiAccessLog(Long id);
|
||||
|
||||
/**
|
||||
* 获得 API 访问日志分页
|
||||
*
|
||||
|
||||
@ -45,6 +45,11 @@ public class ApiAccessLogServiceImpl implements ApiAccessLogService {
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public ApiAccessLogDO getApiAccessLog(Long id) {
|
||||
return apiAccessLogMapper.selectById(id);
|
||||
}
|
||||
|
||||
@Override
|
||||
public PageResult<ApiAccessLogDO> getApiAccessLogPage(ApiAccessLogPageReqVO pageReqVO) {
|
||||
return apiAccessLogMapper.selectPage(pageReqVO);
|
||||
|
||||
@ -19,6 +19,14 @@ public interface ApiErrorLogService {
|
||||
*/
|
||||
void createApiErrorLog(ApiErrorLogCreateReqDTO createReqDTO);
|
||||
|
||||
/**
|
||||
* 获得 API 错误日志
|
||||
*
|
||||
* @param id 编号
|
||||
* @return API 错误日志
|
||||
*/
|
||||
ApiErrorLogDO getApiErrorLog(Long id);
|
||||
|
||||
/**
|
||||
* 获得 API 错误日志分页
|
||||
*
|
||||
|
||||
@ -58,6 +58,11 @@ public class ApiErrorLogServiceImpl implements ApiErrorLogService {
|
||||
return apiErrorLogMapper.selectPage(pageReqVO);
|
||||
}
|
||||
|
||||
@Override
|
||||
public ApiErrorLogDO getApiErrorLog(Long id) {
|
||||
return apiErrorLogMapper.selectById(id);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void updateApiErrorLogProcess(Long id, Integer processStatus, Long processUserId) {
|
||||
ApiErrorLogDO errorLog = apiErrorLogMapper.selectById(id);
|
||||
|
||||
@ -5,9 +5,8 @@ import cn.iocoder.yudao.framework.mybatis.core.mapper.BaseMapperX;
|
||||
import cn.iocoder.yudao.framework.mybatis.core.query.LambdaQueryWrapperX;
|
||||
import cn.iocoder.yudao.module.product.dal.dataobject.sku.ProductSkuDO;
|
||||
import com.baomidou.mybatisplus.core.conditions.update.LambdaUpdateWrapper;
|
||||
import org.apache.ibatis.annotations.Mapper;
|
||||
import org.apache.ibatis.annotations.Param;
|
||||
import org.apache.ibatis.annotations.Select;
|
||||
import com.baomidou.mybatisplus.extension.handlers.JacksonTypeHandler;
|
||||
import org.apache.ibatis.annotations.*;
|
||||
|
||||
import java.util.Collection;
|
||||
import java.util.List;
|
||||
@ -15,7 +14,14 @@ import java.util.List;
|
||||
@Mapper
|
||||
public interface ProductSkuMapper extends BaseMapperX<ProductSkuDO> {
|
||||
|
||||
/**
|
||||
* 查询商品 SKU(包含已删除)
|
||||
* 注意:使用 @Results 手动指定 typeHandler,否则 @Select 不会应用 autoResultMap,properties 字段无法解析 JSON
|
||||
*/
|
||||
@Select("SELECT * FROM product_sku WHERE id = #{id}")
|
||||
@Results({
|
||||
@Result(column = "properties", property = "properties", typeHandler = JacksonTypeHandler.class),
|
||||
})
|
||||
ProductSkuDO selectByIdIncludeDeleted(@Param("id") Long id);
|
||||
|
||||
default List<ProductSkuDO> selectListBySpuId(Long spuId) {
|
||||
|
||||
@ -4,15 +4,15 @@ import cn.hutool.core.util.ObjectUtil;
|
||||
import cn.iocoder.yudao.framework.common.pojo.PageResult;
|
||||
import cn.iocoder.yudao.framework.mybatis.core.mapper.BaseMapperX;
|
||||
import cn.iocoder.yudao.framework.mybatis.core.query.LambdaQueryWrapperX;
|
||||
import cn.iocoder.yudao.framework.mybatis.core.type.IntegerListTypeHandler;
|
||||
import cn.iocoder.yudao.module.product.controller.admin.spu.vo.ProductSpuPageReqVO;
|
||||
import cn.iocoder.yudao.module.product.controller.app.spu.vo.AppProductSpuPageReqVO;
|
||||
import cn.iocoder.yudao.module.product.dal.dataobject.spu.ProductSpuDO;
|
||||
import cn.iocoder.yudao.module.product.enums.ProductConstants;
|
||||
import cn.iocoder.yudao.module.product.enums.spu.ProductSpuStatusEnum;
|
||||
import com.baomidou.mybatisplus.core.conditions.update.LambdaUpdateWrapper;
|
||||
import org.apache.ibatis.annotations.Mapper;
|
||||
import org.apache.ibatis.annotations.Param;
|
||||
import org.apache.ibatis.annotations.Select;
|
||||
import com.baomidou.mybatisplus.extension.handlers.JacksonTypeHandler;
|
||||
import org.apache.ibatis.annotations.*;
|
||||
|
||||
import java.util.Objects;
|
||||
import java.util.Set;
|
||||
@ -20,7 +20,15 @@ import java.util.Set;
|
||||
@Mapper
|
||||
public interface ProductSpuMapper extends BaseMapperX<ProductSpuDO> {
|
||||
|
||||
/**
|
||||
* 查询商品 SPU(包含已删除)
|
||||
* 注意:使用 @Results 手动指定 typeHandler,否则 @Select 不会应用 autoResultMap,sliderPicUrls,deliveryTypes 字段无法解析 JSON
|
||||
*/
|
||||
@Select("SELECT * FROM product_spu WHERE id = #{id}")
|
||||
@Results({
|
||||
@Result(column = "slider_pic_urls", property = "sliderPicUrls", typeHandler = JacksonTypeHandler.class),
|
||||
@Result(column = "delivery_types", property = "deliveryTypes", typeHandler = IntegerListTypeHandler.class),
|
||||
})
|
||||
ProductSpuDO selectByIdIncludeDeleted(@Param("id") Long id);
|
||||
|
||||
/**
|
||||
|
||||
@ -113,6 +113,9 @@ public class AfterSaleLogAspect {
|
||||
* @return 用户类型
|
||||
*/
|
||||
private static Long getUserId() {
|
||||
if (USER_ID.get() != null) {
|
||||
return USER_ID.get();
|
||||
}
|
||||
return ObjectUtil.defaultIfNull(WebFrameworkUtils.getLoginUserId(), TradeOrderLogDO.USER_ID_SYSTEM);
|
||||
}
|
||||
|
||||
|
||||
@ -109,6 +109,9 @@ public class TradeOrderLogAspect {
|
||||
* @return 用户类型
|
||||
*/
|
||||
private static Long getUserId() {
|
||||
if (USER_ID.get() != null) {
|
||||
return USER_ID.get();
|
||||
}
|
||||
return ObjectUtil.defaultIfNull(WebFrameworkUtils.getLoginUserId(), TradeOrderLogDO.USER_ID_SYSTEM);
|
||||
}
|
||||
|
||||
|
||||
@ -51,7 +51,7 @@ public class PayWalletRechargeController {
|
||||
public CommonResult<Boolean> updateWalletRechargeRefunded(@RequestBody PayRefundNotifyReqDTO notifyReqDTO) {
|
||||
walletRechargeService.updateWalletRechargeRefunded(
|
||||
Long.valueOf(notifyReqDTO.getMerchantOrderId()),
|
||||
Long.valueOf(notifyReqDTO.getMerchantRefundId()),
|
||||
notifyReqDTO.getMerchantRefundId(),
|
||||
notifyReqDTO.getPayRefundId());
|
||||
return success(true);
|
||||
}
|
||||
|
||||
@ -56,9 +56,9 @@ public interface PayWalletRechargeService {
|
||||
* 更新钱包充值记录为已退款
|
||||
*
|
||||
* @param id 钱包充值记录编号
|
||||
* @param refundId 钱包充值退款编号(实际和 id 相同)
|
||||
* @param refundId 钱包充值退款编号(格式:{id}-refund)
|
||||
* @param payRefundId 退款单id
|
||||
*/
|
||||
void updateWalletRechargeRefunded(Long id, Long refundId, Long payRefundId);
|
||||
void updateWalletRechargeRefunded(Long id, String refundId, Long payRefundId);
|
||||
|
||||
}
|
||||
|
||||
@ -220,9 +220,8 @@ public class PayWalletRechargeServiceImpl implements PayWalletRechargeService {
|
||||
|
||||
@Override
|
||||
@Transactional(rollbackFor = Exception.class)
|
||||
public void updateWalletRechargeRefunded(Long id, Long refundId, Long payRefundId) {
|
||||
public void updateWalletRechargeRefunded(Long id, String refundId, Long payRefundId) {
|
||||
// 1.1 获取钱包充值记录
|
||||
// 说明:因为 id 和 refundId 是相同的,所以直接使用 id 查询即可!
|
||||
PayWalletRechargeDO walletRecharge = walletRechargeMapper.selectById(id);
|
||||
if (walletRecharge == null) {
|
||||
log.error("[updateWalletRechargerPaid][钱包充值记录不存在,钱包充值记录 id({})]", id);
|
||||
@ -274,8 +273,8 @@ public class PayWalletRechargeServiceImpl implements PayWalletRechargeService {
|
||||
walletRecharge.getId(), payRefundId, toJsonString(walletRecharge), toJsonString(payRefund));
|
||||
throw exception(WALLET_RECHARGE_REFUND_FAIL_REFUND_PRICE_NOT_MATCH);
|
||||
}
|
||||
// 2.3 校验退款订单商户订单是否匹配
|
||||
if (notEqual(payRefund.getMerchantRefundId(), walletRecharge.getId().toString())) {
|
||||
// 2.3 校验退款订单商户退款单是否匹配
|
||||
if (notEqual(payRefund.getMerchantRefundId(), walletRecharge.getId() + "-refund")) {
|
||||
log.error("[validateWalletRechargeCanRefunded][钱包({}) 退款单不匹配({}),请进行处理!payRefund 数据是:{}]",
|
||||
walletRecharge.getId(), payRefundId, toJsonString(payRefund));
|
||||
throw exception(WALLET_RECHARGE_REFUND_FAIL_REFUND_ORDER_ID_ERROR);
|
||||
|
||||
@ -58,13 +58,22 @@ public class PayWalletServiceImpl implements PayWalletService {
|
||||
private PayRefundService refundService;
|
||||
|
||||
@Override
|
||||
@SneakyThrows
|
||||
public PayWalletDO getOrCreateWallet(Long userId, Integer userType) {
|
||||
PayWalletDO wallet = walletMapper.selectByUserIdAndType(userId, userType);
|
||||
if (wallet == null) {
|
||||
wallet = new PayWalletDO().setUserId(userId).setUserType(userType)
|
||||
.setBalance(0).setTotalExpense(0).setTotalRecharge(0);
|
||||
wallet.setCreateTime(LocalDateTime.now());
|
||||
walletMapper.insert(wallet);
|
||||
// 使用双重检查锁,保证钱包创建并发问题
|
||||
// https://gitee.com/zhijiantianya/ruoyi-vue-pro/pulls/1475/files
|
||||
wallet = lockRedisDAO.lock(userId, UPDATE_TIMEOUT_MILLIS, () -> {
|
||||
PayWalletDO newWallet = walletMapper.selectByUserIdAndType(userId, userType);
|
||||
if (newWallet == null) {
|
||||
newWallet = new PayWalletDO().setUserId(userId).setUserType(userType)
|
||||
.setBalance(0).setTotalExpense(0).setTotalRecharge(0);
|
||||
newWallet.setCreateTime(LocalDateTime.now());
|
||||
walletMapper.insert(newWallet);
|
||||
}
|
||||
return newWallet;
|
||||
});
|
||||
}
|
||||
return wallet;
|
||||
}
|
||||
|
||||
@ -64,6 +64,14 @@ public class PostController {
|
||||
return success(true);
|
||||
}
|
||||
|
||||
@DeleteMapping("delete-list")
|
||||
@Operation(summary = "批量删除岗位")
|
||||
@PreAuthorize("@ss.hasPermission('system:post:delete')")
|
||||
public CommonResult<Boolean> deletePostList(@RequestParam("ids") List<Long> ids) {
|
||||
postService.deletePostList(ids);
|
||||
return success(true);
|
||||
}
|
||||
|
||||
@GetMapping(value = "/get")
|
||||
@Operation(summary = "获得岗位信息")
|
||||
@Parameter(name = "id", description = "岗位编号", required = true, example = "1024")
|
||||
|
||||
@ -13,11 +13,13 @@ import cn.iocoder.yudao.module.system.dal.dataobject.logger.OperateLogDO;
|
||||
import cn.iocoder.yudao.module.system.service.logger.OperateLogService;
|
||||
import com.fhs.core.trans.anno.TransMethodResult;
|
||||
import io.swagger.v3.oas.annotations.Operation;
|
||||
import io.swagger.v3.oas.annotations.Parameter;
|
||||
import io.swagger.v3.oas.annotations.tags.Tag;
|
||||
import org.springframework.security.access.prepost.PreAuthorize;
|
||||
import org.springframework.validation.annotation.Validated;
|
||||
import org.springframework.web.bind.annotation.GetMapping;
|
||||
import org.springframework.web.bind.annotation.RequestMapping;
|
||||
import org.springframework.web.bind.annotation.RequestParam;
|
||||
import org.springframework.web.bind.annotation.RestController;
|
||||
|
||||
import javax.annotation.Resource;
|
||||
@ -38,6 +40,15 @@ public class OperateLogController {
|
||||
@Resource
|
||||
private OperateLogService operateLogService;
|
||||
|
||||
@GetMapping("/get")
|
||||
@Operation(summary = "查看操作日志")
|
||||
@Parameter(name = "id", description = "编号", required = true, example = "1024")
|
||||
@PreAuthorize("@ss.hasPermission('system:operate-log:query')")
|
||||
public CommonResult<OperateLogRespVO> getOperateLog(@RequestParam("id") Long id) {
|
||||
OperateLogDO operateLog = operateLogService.getOperateLog(id);
|
||||
return success(BeanUtils.toBean(operateLog, OperateLogRespVO.class));
|
||||
}
|
||||
|
||||
@GetMapping("/page")
|
||||
@Operation(summary = "查看操作日志分页列表")
|
||||
@PreAuthorize("@ss.hasPermission('system:operate-log:query')")
|
||||
|
||||
@ -20,6 +20,14 @@ public interface OperateLogService {
|
||||
*/
|
||||
void createOperateLog(OperateLogCreateReqDTO createReqDTO);
|
||||
|
||||
/**
|
||||
* 获得操作日志
|
||||
*
|
||||
* @param id 编号
|
||||
* @return 操作日志
|
||||
*/
|
||||
OperateLogDO getOperateLog(Long id);
|
||||
|
||||
/**
|
||||
* 获得操作日志分页列表
|
||||
*
|
||||
|
||||
@ -32,6 +32,11 @@ public class OperateLogServiceImpl implements OperateLogService {
|
||||
operateLogMapper.insert(log);
|
||||
}
|
||||
|
||||
@Override
|
||||
public OperateLogDO getOperateLog(Long id) {
|
||||
return operateLogMapper.selectById(id);
|
||||
}
|
||||
|
||||
@Override
|
||||
public PageResult<OperateLogDO> getOperateLogPage(OperateLogPageReqVO pageReqVO) {
|
||||
return operateLogMapper.selectPage(pageReqVO);
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user