[fixed] 英文全词匹配问题

This commit is contained in:
binbin.hou
2026-03-23 18:30:02 +08:00
parent 855659a864
commit 67d47194fd
6 changed files with 451 additions and 25 deletions
+7 -1
View File
@@ -488,4 +488,10 @@
| 序号 | 变更类型 | 说明 | 时间 | 备注 |
|:---|:-----|------------|:--------------------|:-----------|
| 1 | O | 默认改为英文全词匹配 | 2025-12-25 00:37:52 | 更加符合实际使用场景 |
| 1 | O | 默认改为英文全词匹配 | 2025-12-25 00:37:52 | 更加符合实际使用场景 |
# release_0.29.5
| 序号 | 变更类型 | 说明 | 时间 | 备注 |
|:---|:-----|------------------------|:-------------------|:-----------|
| 1 | F | 修正英文全词匹配策略,避免字母汉字混用BUG | 2026-3-23 18:18:08 | |
+1 -1
View File
@@ -6,7 +6,7 @@
<groupId>com.github.houbb</groupId>
<artifactId>sensitive-word</artifactId>
<version>0.29.4</version>
<version>0.29.5</version>
<properties>
<!--============================== All Plugins START ==============================-->
+2 -2
View File
@@ -10,9 +10,9 @@ ECHO "============================= RELEASE START..."
:: 版本号信息(需要手动指定)
:::: 旧版本名称
SET version=0.29.4
SET version=0.29.5
:::: 新版本名称
SET newVersion=0.29.4
SET newVersion=0.29.6
:::: 组织名称
SET groupName=com.github.houbb
:::: 项目名称
@@ -1,7 +1,6 @@
package com.github.houbb.sensitive.word.support.resultcondition;
import com.github.houbb.heaven.util.lang.CharUtil;
import com.github.houbb.heaven.util.util.CharsetUtil;
import com.github.houbb.sensitive.word.api.IWordContext;
import com.github.houbb.sensitive.word.api.IWordResult;
import com.github.houbb.sensitive.word.constant.enums.WordValidModeEnum;
@@ -19,33 +18,26 @@ public class WordResultConditionEnglishWordMatch extends AbstractWordResultCondi
protected boolean doMatch(IWordResult wordResult, String text, WordValidModeEnum modeEnum, IWordContext context) {
final int startIndex = wordResult.startIndex();
final int endIndex = wordResult.endIndex();
// 判断处理,判断前一个字符是否为英文。如果是,则不满足
if(startIndex > 0) {
char preC = text.charAt(startIndex-1);
if(CharUtil.isEnglish(preC)) {
return false;
}
}
// 判断后一个字符是否为英文
// v0.19.1 修正 cp cpm 单个字符错误命中问题
if(endIndex < text.length()) {
char afterC = text.charAt(endIndex);
if(CharUtil.isEnglish(afterC)) {
return false;
}
}
// 判断当前是否为英文单词
for(int i = startIndex; i < endIndex; i++) {
// 检查匹配的文本是否全为英文(可以包含空格)
// 如果包含中文、数字等非英文字符(除了空格),直接返回 true(不需要检查边界)
for (int i = startIndex; i < endIndex; i++) {
char c = text.charAt(i);
if(!CharUtil.isEnglish(c)) {
if (!CharUtil.isEnglish(c) && !CharUtil.isSpace(c)) {
// 包含非英文字符(如中文、数字),直接返回 true
return true;
}
}
// 全英文敏感词(可能包含空格):检查前后字符是否为英文(需要全词匹配)
if (startIndex > 0 && CharUtil.isEnglish(text.charAt(startIndex - 1))) {
return false;
}
if (endIndex < text.length() && CharUtil.isEnglish(text.charAt(endIndex))) {
return false;
}
return true;
}
}
@@ -0,0 +1,360 @@
package com.github.houbb.sensitive.word.bugs.b20260323;
import com.github.houbb.sensitive.word.bs.SensitiveWordBs;
import com.github.houbb.sensitive.word.support.allow.WordAllows;
import com.github.houbb.sensitive.word.support.deny.WordDenys;
import com.github.houbb.sensitive.word.support.resultcondition.WordResultConditions;
import org.junit.Assert;
import org.junit.Test;
import java.util.List;
/**
* Bug 修复测试:中文敏感词在英文前后无法被检测到
*
* 问题原因:WordResultConditionEnglishWordMatch 对所有敏感词都检查前后字符是否为英文
* 修复方案:只有敏感词本身全是英文时,才检查前后字符
*/
public class AddWordBugTest {
// ==================== 基础测试 ====================
/**
* 测试中文敏感词在英文前后能被检测到(核心 bug 修复)
*/
@Test
public void addTest() {
SensitiveWordBs sensitiveWordBs = SensitiveWordBs.newInstance()
.wordAllow(WordAllows.empty())
.wordDeny(WordDenys.empty())
.init();
sensitiveWordBs.addWord("微信");
sensitiveWordBs.addWord("弹药");
sensitiveWordBs.addWord("DP");
Assert.assertTrue(sensitiveWordBs.contains("弹药"));
Assert.assertTrue(sensitiveWordBs.contains("a弹药b"));
}
/**
* 使用 alwaysTrue() 策略测试(不做任何过滤)
*/
@Test
public void addTest2() {
SensitiveWordBs sensitiveWordBs = SensitiveWordBs.newInstance()
.wordAllow(WordAllows.empty())
.wordDeny(WordDenys.empty())
.wordResultCondition(WordResultConditions.alwaysTrue())
.init();
sensitiveWordBs.addWord("微信");
sensitiveWordBs.addWord("弹药");
sensitiveWordBs.addWord("DP");
Assert.assertTrue(sensitiveWordBs.contains("弹药"));
Assert.assertTrue(sensitiveWordBs.contains("a弹药b"));
}
// ==================== 英文敏感词全词匹配测试 ====================
/**
* 测试英文敏感词必须全词匹配
*/
@Test
public void addTest3() {
SensitiveWordBs sensitiveWordBs = SensitiveWordBs.newInstance()
.wordAllow(WordAllows.empty())
.wordDeny(WordDenys.empty())
.init();
sensitiveWordBs.addWord("av");
sensitiveWordBs.addWord("cp");
// 英文敏感词在其他英文单词中不应该匹配
Assert.assertFalse(sensitiveWordBs.contains("cpm"));
Assert.assertFalse(sensitiveWordBs.contains("have"));
// 英文敏感词直接匹配
Assert.assertTrue(sensitiveWordBs.contains("av"));
// 英文敏感词在中文前后应该能匹配
Assert.assertTrue(sensitiveWordBs.contains("av他"));
// 英文敏感词在数字前后应该能匹配
Assert.assertTrue(sensitiveWordBs.contains("1av2"));
// 英文敏感词后跟中文不应该匹配前一个英文
Assert.assertFalse(sensitiveWordBs.contains("have他"));
}
// ==================== 边界情况测试 ====================
/**
* 测试边界情况:空字符串、开头、结尾
*/
@Test
public void boundaryTest() {
SensitiveWordBs sensitiveWordBs = SensitiveWordBs.newInstance()
.wordAllow(WordAllows.empty())
.wordDeny(WordDenys.empty())
.init();
sensitiveWordBs.addWord("弹药");
sensitiveWordBs.addWord("test");
// 空字符串
Assert.assertFalse(sensitiveWordBs.contains(""));
// 敏感词在开头
Assert.assertTrue(sensitiveWordBs.contains("弹药abc"));
Assert.assertTrue(sensitiveWordBs.contains("test abc"));
// 敏感词在结尾
Assert.assertTrue(sensitiveWordBs.contains("abc弹药"));
Assert.assertTrue(sensitiveWordBs.contains("abc test"));
// 只有敏感词
Assert.assertTrue(sensitiveWordBs.contains("弹药"));
Assert.assertTrue(sensitiveWordBs.contains("test"));
}
// ==================== 中英文混合场景测试 ====================
/**
* 测试中英文混合场景
*/
@Test
public void mixedChineseEnglishTest() {
SensitiveWordBs sensitiveWordBs = SensitiveWordBs.newInstance()
.wordAllow(WordAllows.empty())
.wordDeny(WordDenys.empty())
.init();
sensitiveWordBs.addWord("微信");
sensitiveWordBs.addWord("av");
// 中文敏感词在英文前后
Assert.assertTrue(sensitiveWordBs.contains("test微信test"));
Assert.assertTrue(sensitiveWordBs.contains("a微信b"));
// 英文敏感词在中文前后
Assert.assertTrue(sensitiveWordBs.contains("测试av测试"));
Assert.assertTrue(sensitiveWordBs.contains("测av试"));
// 中文敏感词在中文前后
Assert.assertTrue(sensitiveWordBs.contains("测试微信测试"));
// 英文敏感词在英文前后不应该匹配
Assert.assertFalse(sensitiveWordBs.contains("haveavtest"));
}
// ==================== 数字和特殊字符场景测试 ====================
/**
* 测试数字和特殊字符场景
*/
@Test
public void numberAndSpecialCharTest() {
SensitiveWordBs sensitiveWordBs = SensitiveWordBs.newInstance()
.wordAllow(WordAllows.empty())
.wordDeny(WordDenys.empty())
.init();
sensitiveWordBs.addWord("弹药");
sensitiveWordBs.addWord("av");
sensitiveWordBs.addWord("123");
// 中文敏感词与数字混合
Assert.assertTrue(sensitiveWordBs.contains("1弹药2"));
Assert.assertTrue(sensitiveWordBs.contains("123弹药456"));
// 英文敏感词与数字混合
Assert.assertTrue(sensitiveWordBs.contains("1av2"));
Assert.assertTrue(sensitiveWordBs.contains("123av456"));
// 纯数字敏感词
Assert.assertTrue(sensitiveWordBs.contains("123"));
Assert.assertTrue(sensitiveWordBs.contains("a123b"));
// 特殊字符(下划线等)
Assert.assertTrue(sensitiveWordBs.contains("_弹药_"));
Assert.assertTrue(sensitiveWordBs.contains("_av_"));
}
// ==================== 多个敏感词场景测试 ====================
/**
* 测试多个敏感词同时出现的场景
*/
@Test
public void multipleWordsTest() {
SensitiveWordBs sensitiveWordBs = SensitiveWordBs.newInstance()
.wordAllow(WordAllows.empty())
.wordDeny(WordDenys.empty())
.init();
sensitiveWordBs.addWord("微信");
sensitiveWordBs.addWord("弹药");
sensitiveWordBs.addWord("av");
// 多个中文敏感词
Assert.assertTrue(sensitiveWordBs.contains("微信弹药"));
List<String> words1 = sensitiveWordBs.findAll("微信弹药");
Assert.assertEquals(2, words1.size());
// 多个英文敏感词
Assert.assertTrue(sensitiveWordBs.contains("test av test"));
List<String> words2 = sensitiveWordBs.findAll("av av av");
Assert.assertEquals(3, words2.size());
// 中英文混合
Assert.assertTrue(sensitiveWordBs.contains("test微信test av弹药test"));
List<String> words3 = sensitiveWordBs.findAll("test微信test av弹药test");
Assert.assertEquals(3, words3.size());
}
// ==================== findAll 和 findFirst 测试 ====================
/**
* 测试 findAll 和 findFirst 方法
*/
@Test
public void findAllAndFirstTest() {
SensitiveWordBs sensitiveWordBs = SensitiveWordBs.newInstance()
.wordAllow(WordAllows.empty())
.wordDeny(WordDenys.empty())
.init();
sensitiveWordBs.addWord("弹药");
sensitiveWordBs.addWord("av");
// findAll - 中文敏感词在英文前后
List<String> words1 = sensitiveWordBs.findAll("a弹药b");
Assert.assertEquals(1, words1.size());
Assert.assertEquals("弹药", words1.get(0));
// findFirst - 中文敏感词在英文前后
String first1 = sensitiveWordBs.findFirst("a弹药b");
Assert.assertEquals("弹药", first1);
// findAll - 英文敏感词
List<String> words2 = sensitiveWordBs.findAll("test av test");
Assert.assertEquals(1, words2.size());
Assert.assertEquals("av", words2.get(0));
// findFirst - 英文敏感词
String first2 = sensitiveWordBs.findFirst("test av test");
Assert.assertEquals("av", first2);
}
// ==================== 长敏感词场景测试 ====================
/**
* 测试长敏感词场景
*/
@Test
public void longWordTest() {
SensitiveWordBs sensitiveWordBs = SensitiveWordBs.newInstance()
.wordAllow(WordAllows.empty())
.wordDeny(WordDenys.empty())
.init();
sensitiveWordBs.addWord("这是一个非常长的敏感词");
sensitiveWordBs.addWord("thisisaverylongword");
// 长中文敏感词在英文前后
Assert.assertTrue(sensitiveWordBs.contains("test这是一个非常长的敏感词test"));
// 长英文敏感词
Assert.assertTrue(sensitiveWordBs.contains("test thisisaverylongword test"));
Assert.assertFalse(sensitiveWordBs.contains("thisisaverylongwordtest"));
}
// ==================== 单字符敏感词测试 ====================
/**
* 测试单字符敏感词
*/
@Test
public void singleCharWordTest() {
SensitiveWordBs sensitiveWordBs = SensitiveWordBs.newInstance()
.wordAllow(WordAllows.empty())
.wordDeny(WordDenys.empty())
.init();
sensitiveWordBs.addWord("坏");
sensitiveWordBs.addWord("x");
// 单字符中文敏感词
Assert.assertTrue(sensitiveWordBs.contains("坏"));
Assert.assertTrue(sensitiveWordBs.contains("a坏b"));
Assert.assertTrue(sensitiveWordBs.contains("这是一个坏词"));
// 单字符英文敏感词
Assert.assertTrue(sensitiveWordBs.contains("x"));
Assert.assertTrue(sensitiveWordBs.contains("1x2"));
Assert.assertTrue(sensitiveWordBs.contains("x测试"));
// 单字符英文敏感词不应该在其他英文中匹配
Assert.assertFalse(sensitiveWordBs.contains("example"));
}
// ==================== 连续添加敏感词测试 ====================
/**
* 测试连续添加敏感词
*/
@Test
public void continuousAddWordTest() {
SensitiveWordBs sensitiveWordBs = SensitiveWordBs.newInstance()
.wordAllow(WordAllows.empty())
.wordDeny(WordDenys.empty())
.init();
// 连续添加
sensitiveWordBs.addWord("词1");
Assert.assertTrue(sensitiveWordBs.contains("a词1b"));
sensitiveWordBs.addWord("词2");
Assert.assertTrue(sensitiveWordBs.contains("a词2b"));
// 两个词都应该存在
Assert.assertTrue(sensitiveWordBs.contains("词1词2"));
List<String> words = sensitiveWordBs.findAll("词1词2");
Assert.assertEquals(2, words.size());
}
// ==================== 特殊英文字符测试 ====================
/**
* 测试包含大写字母的英文敏感词
*/
@Test
public void uppercaseEnglishWordTest() {
SensitiveWordBs sensitiveWordBs = SensitiveWordBs.newInstance()
.wordAllow(WordAllows.empty())
.wordDeny(WordDenys.empty())
.init();
sensitiveWordBs.addWord("AV");
sensitiveWordBs.addWord("Test");
// 大写英文敏感词直接匹配
Assert.assertTrue(sensitiveWordBs.contains("AV"));
Assert.assertTrue(sensitiveWordBs.contains("Test"));
// 在数字前后
Assert.assertTrue(sensitiveWordBs.contains("1AV2"));
Assert.assertTrue(sensitiveWordBs.contains("1Test2"));
// 在中文前后
Assert.assertTrue(sensitiveWordBs.contains("测AV试"));
Assert.assertTrue(sensitiveWordBs.contains("测Test试"));
// 在英文前后不应该匹配(全词匹配)
Assert.assertFalse(sensitiveWordBs.contains("HAVE"));
Assert.assertFalse(sensitiveWordBs.contains("ATESTB"));
}
}
@@ -0,0 +1,68 @@
package com.github.houbb.sensitive.word.bugs.b20260323;
import com.github.houbb.sensitive.word.bs.SensitiveWordBs;
import com.github.houbb.sensitive.word.support.allow.WordAllows;
import com.github.houbb.sensitive.word.support.deny.WordDenys;
import org.junit.Assert;
import org.junit.Test;
/**
* 测试英文敏感词全词匹配功能
*/
public class EnglishWordMatchTest {
/**
* 测试英文敏感词全词匹配(使用默认策略)
*/
@Test
public void testEnglishWordMatchWithDefaultStrategy() {
SensitiveWordBs sensitiveWordBs = SensitiveWordBs.newInstance()
.wordAllow(WordAllows.empty())
.wordDeny(WordDenys.empty())
// 使用默认策略 englishWordMatch()
.init();
sensitiveWordBs.addWord("av");
sensitiveWordBs.addWord("sex");
// 直接匹配
Assert.assertTrue(sensitiveWordBs.contains("av"));
Assert.assertTrue(sensitiveWordBs.contains("sex"));
// 英文敏感词全词匹配
Assert.assertTrue(sensitiveWordBs.contains("av is bad"));
Assert.assertTrue(sensitiveWordBs.contains("test sex test"));
// 英文敏感词在其他英文单词中不应该匹配
Assert.assertFalse(sensitiveWordBs.contains("have")); // "av" 在 "have" 中
Assert.assertFalse(sensitiveWordBs.contains("Middlesex")); // "sex" 在 "Middlesex" 中
// 英文敏感词在非英文字符前后应该能匹配
Assert.assertTrue(sensitiveWordBs.contains("1av2"));
Assert.assertTrue(sensitiveWordBs.contains("av测试"));
Assert.assertTrue(sensitiveWordBs.contains("测试sex"));
}
/**
* 测试中文敏感词在英文前后能被检测到(使用默认策略)
*/
@Test
public void testChineseWordWithEnglishAround() {
SensitiveWordBs sensitiveWordBs = SensitiveWordBs.newInstance()
.wordAllow(WordAllows.empty())
.wordDeny(WordDenys.empty())
// 使用默认策略 englishWordMatch()
.init();
sensitiveWordBs.addWord("弹药");
sensitiveWordBs.addWord("微信");
// 中文敏感词直接匹配
Assert.assertTrue(sensitiveWordBs.contains("弹药"));
// 中文敏感词在英文前后应该能被检测到(这是 bug 修复的核心)
Assert.assertTrue(sensitiveWordBs.contains("a弹药b"));
Assert.assertTrue(sensitiveWordBs.contains("test微信test"));
}
}