Merge remote-tracking branch 'origin/main'

# Conflicts:
#	mallchat-common/src/main/java/com/abin/mallchat/common/common/utils/SensitiveWordUtils.java
This commit is contained in:
xiaocairush 2023-06-18 21:09:46 +08:00
commit 3943f44d19
13 changed files with 284 additions and 14687 deletions

View File

@ -25,6 +25,7 @@ INSERT INTO `item_config` VALUES (2, 2, 'https://cdn-icons-png.flaticon.com/128/
INSERT INTO `item_config` VALUES (3, 2, 'https://cdn-icons-png.flaticon.com/512/6198/6198527.png ', '抹茶聊天前10名注册的用户才能获得的专属徽章', '2023-05-07 17:50:31.100', '2023-05-07 18:12:01.448');
INSERT INTO `item_config` VALUES (4, 2, 'https://cdn-icons-png.flaticon.com/512/10232/10232583.png', '抹茶聊天前100名注册的用户才能获得的专属徽章', '2023-05-07 17:50:31.109', '2023-05-07 17:56:36.059');
INSERT INTO `item_config` VALUES (5, 2, 'https://cdn-icons-png.flaticon.com/128/2909/2909937.png', '抹茶知识星球成员的专属徽章', '2023-05-07 17:50:31.109', '2023-05-07 17:56:36.059');
INSERT INTO `item_config` VALUES (6, 2, 'https://s2.loli.net/2023/06/15/O9FwjH4ciAuMSnL.png', '抹茶项目contributor专属徽章', '2023-05-07 17:50:31.109', '2023-05-07 17:56:36.059');
-- ----------------------------
-- Table structure for message
@ -181,4 +182,11 @@ CREATE TABLE `user_role` (
KEY `idx_role_id` (`role_id`) USING BTREE,
KEY `idx_create_time` (`create_time`) USING BTREE,
KEY `idx_update_time` (`update_time`) USING BTREE
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci COMMENT='用户角色关系表';
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci COMMENT='用户角色关系表';
DROP TABLE IF EXISTS `sensitive_word`;
CREATE TABLE `sensitive_word` (
`word` varchar(255) COLLATE utf8mb4_unicode_ci NOT NULL COMMENT '敏感词'
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci COMMENT='敏感词库';
INSERT INTO `sensitive_word` (`word`) VALUES ('TMD');
INSERT INTO `sensitive_word` (`word`) VALUES ('tmd');

File diff suppressed because it is too large Load Diff

View File

@ -0,0 +1,6 @@
DROP TABLE IF EXISTS `sensitive_word`;
CREATE TABLE `sensitive_word` (
`word` varchar(255) COLLATE utf8mb4_unicode_ci NOT NULL COMMENT '敏感词'
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci COMMENT='敏感词库';
INSERT INTO `sensitive_word` (`word`) VALUES ('TMD');
INSERT INTO `sensitive_word` (`word`) VALUES ('tmd');

View File

@ -1,7 +1,5 @@
package com.abin.mallchat.common.common.utils;
import com.abin.mallchat.common.common.algorithm.ac.ACTrie;
import com.abin.mallchat.common.common.algorithm.ac.MatchResult;
import org.apache.commons.lang3.StringUtils;
import java.io.BufferedReader;
@ -9,33 +7,33 @@ import java.io.File;
import java.io.IOException;
import java.io.InputStreamReader;
import java.nio.file.Files;
import java.util.ArrayList;
import java.util.List;
import java.util.Objects;
import java.util.*;
import java.util.stream.Collectors;
/**
* 敏感词过滤
*
* @author zhaoyuhang
* @since 2023/06/11
*/
public final class SensitiveWordUtils {
private final static char mask_char = '*'; // 替代字符
private static ACTrie ac_trie = null;
private static Map<Character, Word> wordMap; // 敏感词Map
private final static char replace = '*'; // 替代字符
private final static char[] skip = new char[]{ // 遇到这些字符就会跳过
' ', '!', '*', '-', '+', '_', '=', ',', '', '.', '@', ';', ':', '', ''
};
/**
* 敏感词
* 判断文本中是否存在敏感词
*
* @param text 文本
* @return boolean
* @return true: 存在敏感词, false: 不存在敏感词
*/
public static boolean hasSensitiveWord(String text) {
if (StringUtils.isBlank(text)) return false;
return !Objects.equals(filter(text), text);
}
/**
* 过滤敏感词并替换为指定字符
*
* @param text 待替换文本
* @return 替换后的文本
*/
/**
* 敏感词替换
*
@ -43,23 +41,48 @@ public final class SensitiveWordUtils {
* @return 替换后的文本
*/
public static String filter(String text) {
if (StringUtils.isBlank(text)) return text;
List<MatchResult> matchResults = ac_trie.matches(text);
StringBuffer result = new StringBuffer(text);
// matchResults是按照startIndex排序的因此可以通过不断更新endIndex最大值的方式算出尚未被替代部分
int endIndex = 0;
for (MatchResult matchResult : matchResults) {
endIndex = Math.max(endIndex, matchResult.getEndIndex());
replaceBetween(result, matchResult.getStartIndex(), endIndex);
if (wordMap == null || wordMap.isEmpty() || StringUtils.isBlank(text)) return text;
char[] chars = text.toCharArray(); // 将文本转换为字符数组
int length = chars.length; // 文本长度
StringBuilder result = new StringBuilder(length); // 存储替换后的结果
int i = 0; // 当前遍历的字符索引
while (i < length) {
char c = chars[i]; // 当前字符
if (skip(c)) { // 如果是需要跳过的字符则直接追加到结果中
i++;
continue;
}
int startIndex = i; // 敏感词匹配的起始索引
Map<Character, Word> currentMap = wordMap; // 当前层级的敏感词字典
int matchLength = 0; // 匹配到的敏感词长度
for (int j = i; j < length; j++) {
char ch = chars[j]; // 当前遍历的字符
if (skip(ch)) { // 如果是需要跳过的字符则直接追加到结果中
continue;
}
Word word = currentMap.get(ch); // 获取当前字符在当前层级的敏感词字典中对应的敏感词节点
if (word == null) { // 如果未匹配到敏感词节点则终止循环
break;
}
if (word.end) { // 如果当前节点是敏感词的最后一个节点则记录匹配长度
matchLength = j - startIndex + 1;
}
currentMap = word.next; // 进入下一层级的敏感词字典
if (word.next == null) { // 如果当前节点是敏感词的最后一个节点则记录匹配长度
matchLength = j - startIndex + 1;
}
}
if (matchLength > 0) { // 如果匹配到敏感词则将对应的字符替换为指定替代字符
for (int j = startIndex; j < startIndex + matchLength; j++) {
chars[j] = replace;
}
}
i += matchLength > 0 ? matchLength : 1; // 更新当前索引跳过匹配到的敏感词
}
result.append(chars); // 将匹配到的敏感词追加到结果中
return result.toString();
}
private static void replaceBetween(StringBuffer buffer, int startIndex, int endIndex) {
for (int i = startIndex; i < endIndex; i++) {
buffer.setCharAt(i, mask_char);
}
}
/**
* 加载敏感词列表
@ -68,13 +91,34 @@ public final class SensitiveWordUtils {
*/
public static void loadWord(List<String> words) {
if (words == null) return;
ac_trie = new ACTrie(words);
words = words.stream().distinct().collect(Collectors.toList()); // 去重
wordMap = new HashMap<>(); // 创建敏感词字典的根节点
for (String word : words) {
if (word == null) continue;
char[] chars = word.toCharArray();
Map<Character, Word> currentMap = wordMap; // 当前层级的敏感词字典
for (int i = 0; i < chars.length; i++) {
char c = chars[i];
Word currentWord = currentMap.get(c);
if (currentWord == null) {
Word newWord = new Word(c); // 创建新的敏感词节点
currentMap.put(c, newWord); // 将节点添加到当前层级的敏感词字典中
if (i == chars.length - 1) {
newWord.end = true; // 添加结束标志
}
currentMap = newWord.next = new HashMap<>(); // 进入下一层级
} else {
currentMap = currentWord.next; // 存在该字符的节点则进入下一层级
}
}
}
}
/**
* 加载敏感词txt文件每个敏感词独占一行不可出现空格空行逗号等非文字内容,必须使用UTF-8编码
* 从文本文件中加载敏感词列表
*
* @param path txt文件的绝对地址
* @param path 文本文件的绝对路径
*/
public static void loadWordFromFile(String path) {
String encoding = "UTF-8";
@ -99,8 +143,40 @@ public final class SensitiveWordUtils {
}
}
/**
* 判断是否需要跳过当前字符
*
* @param c 待检测字符
* @return true: 需要跳过, false: 不需要跳过
*/
private static boolean skip(char c) {
for (char skipChar : skip) {
if (skipChar == c) return true;
}
return false;
}
/**
* 敏感词类
*/
private static class Word {
// 当前字符
private char c;
// 结束标识
private boolean end;
// 下一层级的敏感词字典
private Map<Character, Word> next;
public Word(char c) {
this.c = c;
}
}
public static void main(String[] args) {
List<String> strings = Arrays.asList("白日梦", "白痴", "白痴是你","TMD");
loadWord(strings);
System.out.println(filter("TMD,白痴是你吗"));
}
}

View File

@ -4,8 +4,10 @@ import com.abin.mallchat.common.common.utils.SensitiveWordUtils;
import com.abin.mallchat.common.sensitive.dao.SensitiveWordDao;
import com.abin.mallchat.common.sensitive.domain.SensitiveWord;
import com.abin.mallchat.common.sensitive.service.ISensitiveWordService;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.collections.CollectionUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor;
import org.springframework.stereotype.Service;
import javax.annotation.PostConstruct;
@ -13,18 +15,25 @@ import java.util.List;
import java.util.stream.Collectors;
@Service
@Slf4j
public class SensitiveWordServiceImpl implements ISensitiveWordService {
@Autowired
private SensitiveWordDao sensitiveWordDao;
@Autowired
private ThreadPoolTaskExecutor threadPoolTaskExecutor;
@PostConstruct
public void initSensitiveWord() {
List<SensitiveWord> list = sensitiveWordDao.list();
if (!CollectionUtils.isEmpty(list)) {
List<String> wordList = list.stream()
.map(SensitiveWord::getWord)
.collect(Collectors.toList());
SensitiveWordUtils.loadWord(wordList);
}
threadPoolTaskExecutor.execute(() -> {
log.info("[initSensitiveWord] start");
List<SensitiveWord> list = sensitiveWordDao.list();
if (!CollectionUtils.isEmpty(list)) {
List<String> wordList = list.stream()
.map(SensitiveWord::getWord)
.collect(Collectors.toList());
SensitiveWordUtils.loadWord(wordList);
}
log.info("[initSensitiveWord] end; loading sensitiveWords num:{}", list.size());
});
}
}

View File

@ -19,9 +19,17 @@ import lombok.NoArgsConstructor;
public class ItemInfoDTO {
@ApiModelProperty(value = "徽章id")
private Long itemId;
@ApiModelProperty(value = "是否需要刷新")
private Boolean needRefresh = Boolean.TRUE;
@ApiModelProperty("徽章图像")
private String img;
@ApiModelProperty("徽章说明")
private String describe;
public static ItemInfoDTO skip(Long itemId) {
ItemInfoDTO dto = new ItemInfoDTO();
dto.setItemId(itemId);
dto.setNeedRefresh(Boolean.FALSE);
return dto;
}
}

View File

@ -21,6 +21,8 @@ import java.util.List;
public class SummeryInfoDTO {
@ApiModelProperty(value = "用户id")
private Long uid;
@ApiModelProperty(value = "是否需要刷新")
private Boolean needRefresh = Boolean.TRUE;
@ApiModelProperty(value = "用户昵称")
private String name;
@ApiModelProperty(value = "用户头像")
@ -32,4 +34,10 @@ public class SummeryInfoDTO {
@ApiModelProperty(value = "用户拥有的徽章id列表")
List<Long> itemIds;
public static SummeryInfoDTO skip(Long uid) {
SummeryInfoDTO dto = new SummeryInfoDTO();
dto.setUid(uid);
dto.setNeedRefresh(Boolean.FALSE);
return dto;
}
}

View File

@ -15,6 +15,7 @@ import com.abin.mallchat.custom.user.service.impl.UserServiceImpl;
import io.swagger.annotations.Api;
import io.swagger.annotations.ApiOperation;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.BeanUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.web.bind.annotation.*;
@ -22,6 +23,7 @@ import javax.validation.Valid;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import java.util.stream.Collectors;
/**
* <p>
@ -57,6 +59,20 @@ public class ChatController {
return ApiResult.success(memberPage);
}
@GetMapping("/public/member/page/v1")
@ApiOperation("群成员列表/v1")
@FrequencyControl(time = 120, count = 20, target = FrequencyControl.Target.IP)
public ApiResult<CursorPageBaseResp<ChatMemberRespV1>> getMemberPage1(@Valid CursorPageBaseReq request) {
CursorPageBaseResp<ChatMemberResp> memberPage = chatService.getMemberPage(request);
filterBlackMember(memberPage);
List<ChatMemberRespV1> collect = memberPage.getList().stream().map(a -> {
ChatMemberRespV1 v1 = new ChatMemberRespV1();
BeanUtils.copyProperties(a, v1);
return v1;
}).collect(Collectors.toList());
return ApiResult.success(CursorPageBaseResp.init(memberPage, collect));
}
@GetMapping("/member/list")
@ApiOperation("房间内的所有群成员列表-@专用")
public ApiResult<List<ChatMemberListResp>> getMemberList(@Valid ChatMessageMemberReq chatMessageMemberReq) {
@ -81,25 +97,30 @@ public class ChatController {
@Autowired
private UserServiceImpl userService;
@GetMapping("/public/msg/page/v1")
@ApiOperation("消息列表/v1")
@FrequencyControl(time = 120, count = 20, target = FrequencyControl.Target.IP)
public ApiResult<CursorPageBaseResp<ChatMessageRespV1>> getMsgPage(@Valid ChatMessagePageReq request) {
CursorPageBaseResp<ChatMessageResp> msgPage = chatService.getMsgPage(request, RequestHolder.get().getUid());
filterBlackMsg(msgPage);
List<ChatMessageRespV1> collect = msgPage.getList().stream().map(a -> {
ChatMessageRespV1 v1 = new ChatMessageRespV1();
BeanUtils.copyProperties(a, v1);
return v1;
}).collect(Collectors.toList());
return ApiResult.success(CursorPageBaseResp.init(msgPage, collect));
}
@GetMapping("/public/msg/page")
@ApiOperation("消息列表")
@FrequencyControl(time = 120, count = 20, target = FrequencyControl.Target.IP)
public ApiResult<CursorPageBaseResp<ChatMessageResp>> getMsgPage(@Valid ChatMessagePageReq request) {
public ApiResult<CursorPageBaseResp<ChatMessageResp>> getMsgPage1(@Valid ChatMessagePageReq request) {
// black(request);
CursorPageBaseResp<ChatMessageResp> msgPage = chatService.getMsgPage(request, RequestHolder.get().getUid());
filterBlackMsg(msgPage);
return ApiResult.success(msgPage);
}
private void black(CursorPageBaseReq baseReq) {
if (baseReq.getPageSize() > 50) {
log.info("limit request:{}", baseReq);
baseReq.setPageSize(10);
userService.blackIp(RequestHolder.get().getIp());
}
}
private void filterBlackMsg(CursorPageBaseResp<ChatMessageResp> memberPage) {
Set<String> blackMembers = getBlackUidSet();
memberPage.getList().removeIf(a -> blackMembers.contains(a.getFromUser().getUid().toString()));

View File

@ -0,0 +1,30 @@
package com.abin.mallchat.custom.chat.domain.vo.response;
import io.swagger.annotations.ApiModelProperty;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
import java.util.Date;
/**
* Description: 群成员列表的成员信息
* Author: <a href="https://github.com/zongzibinbin">abin</a>
* Date: 2023-03-23
*/
@Data
@Builder
@AllArgsConstructor
@NoArgsConstructor
public class ChatMemberRespV1 {
@ApiModelProperty("uid")
private Long uid;
/**
* @see com.abin.mallchat.common.user.domain.enums.ChatActiveStatusEnum
*/
@ApiModelProperty("在线状态 1在线 2离线")
private Integer activeStatus;
@ApiModelProperty("最后一次上下线时间")
private Date lastOptTime;
}

View File

@ -0,0 +1,60 @@
package com.abin.mallchat.custom.chat.domain.vo.response;
import io.swagger.annotations.ApiModelProperty;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
import java.util.Date;
/**
* Description: 消息
* Author: <a href="https://github.com/zongzibinbin">abin</a>
* Date: 2023-03-23
*/
@Data
@Builder
@AllArgsConstructor
@NoArgsConstructor
public class ChatMessageRespV1 {
@ApiModelProperty("发送者信息")
private UserInfo fromUser;
@ApiModelProperty("消息详情")
private Message message;
@Data
public static class UserInfo {
@ApiModelProperty("用户id")
private Long uid;
}
@Data
public static class Message {
@ApiModelProperty("消息id")
private Long id;
@ApiModelProperty("消息发送时间")
private Date sendTime;
@ApiModelProperty("消息类型 1正常文本 2.撤回消息")
private Integer type;
@ApiModelProperty("消息内容不同的消息类型内容体不同见https://www.yuque.com/snab/mallcaht/rkb2uz5k1qqdmcmd")
private Object body;
@ApiModelProperty("消息标记")
private MessageMark messageMark;
}
@Data
public static class MessageMark {
@ApiModelProperty("点赞数")
private Integer likeCount;
@ApiModelProperty("该用户是否已经点赞 0否 1是")
private Integer userLike;
@ApiModelProperty("举报数")
private Integer dislikeCount;
@ApiModelProperty("该用户是否已经举报 0否 1是")
private Integer userDislike;
}
}

View File

@ -143,7 +143,10 @@ public class UserServiceImpl implements UserService {
List<Long> uidList = getNeedSyncUidList(req.getReqList());
//加载用户信息
Map<Long, SummeryInfoDTO> batch = userSummaryCache.getBatch(uidList);
return new ArrayList<>(batch.values());
return req.getReqList()
.stream()
.map(a -> batch.containsKey(a.getUid()) ? batch.get(a.getUid()) : SummeryInfoDTO.skip(a.getUid()))
.collect(Collectors.toList());
}
@Override
@ -151,7 +154,7 @@ public class UserServiceImpl implements UserService {
return req.getReqList().stream().map(a -> {
ItemConfig itemConfig = itemCache.getById(a.getItemId());
if (Objects.nonNull(a.getLastModifyTime()) && a.getLastModifyTime() >= itemConfig.getUpdateTime().getTime()) {
return null;
return ItemInfoDTO.skip(a.getItemId());
}
ItemInfoDTO dto = new ItemInfoDTO();
dto.setItemId(itemConfig.getId());

View File

@ -90,7 +90,6 @@ public class WebSocketServiceImpl implements WebSocketService {
*/
@SneakyThrows
@Override
@FrequencyControl(time = 10, count = 2, spEl = "T(com.abin.mallchat.common.common.utils.RequestHolder).get().getIp()")
@FrequencyControl(time = 100, count = 5, spEl = "T(com.abin.mallchat.common.common.utils.RequestHolder).get().getIp()")
public void handleLoginReq(Channel channel) {
//生成随机不重复的登录码

View File

@ -96,7 +96,6 @@ public class NettyWebSocketServer {
});
// 启动服务器监听端口阻塞直到启动成功
serverBootstrap.bind(WEB_SOCKET_PORT).sync();
System.out.println("启动成功");
}
}