diff --git a/CHANGE_LOG.md b/CHANGE_LOG.md index 6e293d8..de316ec 100644 --- a/CHANGE_LOG.md +++ b/CHANGE_LOG.md @@ -293,6 +293,24 @@ |:---|:-----|------------|:-------------------|:-------------------------------------------------| | 1 | A | 结果添加敏感词的类别 | 2024-4-11 15:02:25 | | +# release_0.15.0 + +| 序号 | 变更类型 | 说明 | 时间 | 备注 | +|:---|:-----|---------------------|:-------------------|:-------------------------------------------------| +| 1 | F | 调整默认文件名称，避免和其他框架重合。 | 2024-4-23 21:02:25 | https://github.com/houbb/sensitive-word/issues/54 | + +# release_0.16.0 + +| 序号 | 变更类型 | 说明 | 时间 | 备注 | +|:---|:-----|----------------------|:-------------------|:-------------------------------------------------------| +| 1 | A | 支持资源的释放，如 andriod 场景 | 2024-4-26 21:02:25 | 废弃版本，https://github.com/houbb/sensitive-word/issues/53 | + +# release_0.16.1 + +| 序号 | 变更类型 | 说明 | 时间 | 备注 | +|:---|:-----|----------------------|:-------------------|:----------------------------------------------------------| +| 1 | F | 支持资源的释放，如 andriod 场景 | 2024-4-26 21:02:25 | 修正未调用问题，https://github.com/houbb/sensitive-word/issues/53 | + # release_0.14.1 | 序号 | 变更类型 | 说明 | 时间 | 备注 | diff --git a/README.md b/README.md index e6198b0..69d4946 100644 --- a/README.md +++ b/README.md @@ -56,6 +56,9 @@ V0.14.1: - 移除部分敏感词 - 默认关闭 url/email/num 的校验 +V0.16.1: + +- [x] 支持内存释放 [#53](https://github.com/houbb/sensitive-word/issues/53) ## 更多资料 @@ -87,7 +90,7 @@ V0.14.1: com.github.houbb sensitive-word - 0.14.1 + 0.16.1 ``` @@ -454,6 +457,19 @@ Assert.assertTrue(wordBs.contains(text)); | 13 | charIgnore | 忽略的字符 | none | | 14 | wordResultCondition | 针对匹配的敏感词额外加工，比如可以限制英文单词必须全匹配 | 恒为真 | +## 内存的释放 + +v0.16.1 开始支持，有时候我们需要释放内存，可以如下： + +> [关于内存回收问题](https://github.com/houbb/sensitive-word/issues/53) + +```java +SensitiveWordBs wordBs = SensitiveWordBs.newInstance() + .init(); +// 后续因为一些原因移除了对应信息，希望释放内存。 +wordBs.destroy(); +``` + # wordResultCondition-针对匹配词进一步判断 ## 说明 @@ -604,6 +620,15 @@ Assert.assertEquals("[政治, 国家]", sensitiveWordBs.tags("五星红旗").toS 后续会考虑引入一个内置的标签文件策略。 +### 敏感词标签文件 + +梳理了大量的敏感词标签文件，可以让我们的敏感词更加方便。 + +这两个资料阅读可在下方文章获取： + +> [v0.11.0-敏感词新特性及对应标签文件](https://mp.weixin.qq.com/s/m40ZnR6YF6WgPrArUSZ_0g) + + # 动态加载（用户自定义） ## 情景说明 diff --git a/pom.xml b/pom.xml index 7507e64..6ee59cd 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ com.github.houbb sensitive-word - 0.14.1 + 0.16.1 diff --git a/release.bat b/release.bat index 43a4377..6491403 100644 --- a/release.bat +++ b/release.bat @@ -10,9 +10,9 @@ ECHO "============================= RELEASE START..." :: 版本号信息(需要手动指定) :::: 旧版本名称 -SET version=0.14.1 +SET version=0.16.1 :::: 新版本名称 -SET newVersion=0.15.0 +SET newVersion=0.17.0 :::: 组织名称 SET groupName=com.github.houbb :::: 项目名称 diff --git a/src/main/java/com/github/houbb/sensitive/word/api/ISensitiveWordDestroy.java b/src/main/java/com/github/houbb/sensitive/word/api/ISensitiveWordDestroy.java new file mode 100644 index 0000000..ca81210 --- /dev/null +++ b/src/main/java/com/github/houbb/sensitive/word/api/ISensitiveWordDestroy.java @@ -0,0 +1,11 @@ +package com.github.houbb.sensitive.word.api; + +public interface ISensitiveWordDestroy { + + /** + * 资源的销毁 + * @since 0.16.0 + */ + void destroy(); + +} diff --git a/src/main/java/com/github/houbb/sensitive/word/api/IWordData.java b/src/main/java/com/github/houbb/sensitive/word/api/IWordData.java index 0a4f90c..24cfab8 100644 --- a/src/main/java/com/github/houbb/sensitive/word/api/IWordData.java +++ b/src/main/java/com/github/houbb/sensitive/word/api/IWordData.java @@ -11,7 +11,7 @@ import java.util.Collection; * @author binbin.hou * @since 0.0.1 */ -public interface IWordData { +public interface IWordData extends ISensitiveWordDestroy { /** diff --git a/src/main/java/com/github/houbb/sensitive/word/bs/SensitiveWordBs.java b/src/main/java/com/github/houbb/sensitive/word/bs/SensitiveWordBs.java index 71c7254..da742ad 100644 --- a/src/main/java/com/github/houbb/sensitive/word/bs/SensitiveWordBs.java +++ b/src/main/java/com/github/houbb/sensitive/word/bs/SensitiveWordBs.java @@ -32,7 +32,7 @@ import java.util.Set; * @author binbin.hou * @since 0.0.1 */ -public class SensitiveWordBs { +public class SensitiveWordBs implements ISensitiveWordDestroy { /** * 私有化构造器 @@ -573,6 +573,11 @@ public class SensitiveWordBs { return wordTag.getTag(word); } + @Override + public void destroy() { + this.wordData.destroy(); + } + //------------------------------------------------------------------------------------ 公开方法 END } diff --git a/src/main/java/com/github/houbb/sensitive/word/support/data/WordDataHashMap.java b/src/main/java/com/github/houbb/sensitive/word/support/data/WordDataHashMap.java index 0840522..696f5b0 100644 --- a/src/main/java/com/github/houbb/sensitive/word/support/data/WordDataHashMap.java +++ b/src/main/java/com/github/houbb/sensitive/word/support/data/WordDataHashMap.java @@ -183,4 +183,11 @@ public class WordDataHashMap extends AbstractWordData { return currentMap; } + @Override + public synchronized void destroy() { + if(innerWordMap != null) { + innerWordMap.clear(); + } + } + } diff --git a/src/main/java/com/github/houbb/sensitive/word/support/data/WordDataTree.java b/src/main/java/com/github/houbb/sensitive/word/support/data/WordDataTree.java index 0cc6da0..53c67a9 100644 --- a/src/main/java/com/github/houbb/sensitive/word/support/data/WordDataTree.java +++ b/src/main/java/com/github/houbb/sensitive/word/support/data/WordDataTree.java @@ -117,4 +117,10 @@ public class WordDataTree implements IWordData { return currentMap; } + @Override + public void destroy() { + if(this.root != null) { + this.root.destroy(); + } + } } diff --git a/src/main/java/com/github/houbb/sensitive/word/support/data/WordDataTreeNode.java b/src/main/java/com/github/houbb/sensitive/word/support/data/WordDataTreeNode.java index 350adce..95ee43c 100644 --- a/src/main/java/com/github/houbb/sensitive/word/support/data/WordDataTreeNode.java +++ b/src/main/java/com/github/houbb/sensitive/word/support/data/WordDataTreeNode.java @@ -1,5 +1,7 @@ package com.github.houbb.sensitive.word.support.data; +import com.github.houbb.sensitive.word.api.ISensitiveWordDestroy; + import java.util.HashMap; import java.util.Map; @@ -8,7 +10,7 @@ import java.util.Map; * * @since 0.7.0 */ -public class WordDataTreeNode { +public class WordDataTreeNode implements ISensitiveWordDestroy { /** * 关键词结束标识 @@ -46,4 +48,11 @@ public class WordDataTreeNode { return this; } + @Override + public void destroy() { + if(subNodeMap != null) { + subNodeMap.clear(); + } + } + } diff --git a/src/main/java/com/github/houbb/sensitive/word/support/deny/WordDenySystem.java b/src/main/java/com/github/houbb/sensitive/word/support/deny/WordDenySystem.java index 8c91837..81e6286 100644 --- a/src/main/java/com/github/houbb/sensitive/word/support/deny/WordDenySystem.java +++ b/src/main/java/com/github/houbb/sensitive/word/support/deny/WordDenySystem.java @@ -25,8 +25,8 @@ public class WordDenySystem implements IWordDeny { @Override public List deny() { - List results = StreamUtil.readAllLines("/dict.txt"); - results.addAll(StreamUtil.readAllLines("/dict_en.txt")); + List results = StreamUtil.readAllLines("/sensitive_word_dict.txt"); + results.addAll(StreamUtil.readAllLines("/sensitive_word_dict_en.txt")); results.addAll(StreamUtil.readAllLines("/sensitive_word_deny.txt")); return results; } diff --git a/src/main/resources/dict.txt b/src/main/resources/sensitive_word_dict.txt similarity index 100% rename from src/main/resources/dict.txt rename to src/main/resources/sensitive_word_dict.txt diff --git a/src/main/resources/dict_en.txt b/src/main/resources/sensitive_word_dict_en.txt similarity index 100% rename from src/main/resources/dict_en.txt rename to src/main/resources/sensitive_word_dict_en.txt diff --git a/src/test/java/ai/com/github/houbb/sensitive/word/utils/package-info.java b/src/test/java/ai/com/github/houbb/sensitive/word/utils/package-info.java new file mode 100644 index 0000000..f27d3ec --- /dev/null +++ b/src/test/java/ai/com/github/houbb/sensitive/word/utils/package-info.java @@ -0,0 +1 @@ +package ai.com.github.houbb.sensitive.word.utils; \ No newline at end of file diff --git a/src/test/java/com/github/houbb/sensitive/word/bs/SensitiveWordBsDestroyTest.java b/src/test/java/com/github/houbb/sensitive/word/bs/SensitiveWordBsDestroyTest.java new file mode 100644 index 0000000..d117830 --- /dev/null +++ b/src/test/java/com/github/houbb/sensitive/word/bs/SensitiveWordBsDestroyTest.java @@ -0,0 +1,20 @@ +package com.github.houbb.sensitive.word.bs; + +import org.junit.Test; + +/** + * 资源的销毁 + * + * @since 0.16.0 + */ +public class SensitiveWordBsDestroyTest { + + @Test + public void destroyTest() { + SensitiveWordBs wordBs = SensitiveWordBs.newInstance() + .init(); + // 后续因为一些原因移除了对应信息，希望释放内存。 + wordBs.destroy(); + } + +} diff --git a/src/test/java/com/github/houbb/sensitive/word/bugs/b55/Bug55Test.java b/src/test/java/com/github/houbb/sensitive/word/bugs/b55/Bug55Test.java new file mode 100644 index 0000000..b06c7ec --- /dev/null +++ b/src/test/java/com/github/houbb/sensitive/word/bugs/b55/Bug55Test.java @@ -0,0 +1,17 @@ +package com.github.houbb.sensitive.word.bugs.b55; + +import com.github.houbb.sensitive.word.bs.SensitiveWordBs; +import org.junit.Assert; +import org.junit.Test; + +public class Bug55Test { + + @Test + public void test() { + SensitiveWordBs sensitiveWordBs = SensitiveWordBs.newInstance() + .init(); + final String text = "以个人账户或现金收取资金、现场或即时交付本金即给予部分提成、分红、利息;"; + Assert.assertEquals("[]", sensitiveWordBs.findAll(text).toString()); + } + +} diff --git a/src/test/java/com/github/houbb/sensitive/word/data/DataInitTest.java b/src/test/java/com/github/houbb/sensitive/word/data/DataInitTest.java index 8563d12..e1e0ab0 100644 --- a/src/test/java/com/github/houbb/sensitive/word/data/DataInitTest.java +++ b/src/test/java/com/github/houbb/sensitive/word/data/DataInitTest.java @@ -34,11 +34,11 @@ public class DataInitTest { @Test @Ignore public void trimTest() { - final String source = "D:\\github\\sensitive-word\\src\\main\\resources\\dict.txt"; + final String source = "D:\\github\\sensitive-word\\src\\main\\resources\\sensitive_word_dict.txt"; List lines = FileUtil.readAllLines(source); List trimLines = CollectionUtil.distinct(CollectionUtil.trimCollection(lines)); - final String target = "D:\\github\\sensitive-word\\src\\main\\resources\\dict.txt"; + final String target = "D:\\github\\sensitive-word\\src\\main\\resources\\sensitive_word_dict.txt"; } /** @@ -75,7 +75,7 @@ public class DataInitTest { @Test @Ignore public void dictAllInOneTest() { - final List allLines = distinctLines("dict.txt"); + final List allLines = distinctLines("sensitive_word_dict.txt"); allLines.addAll(distinctLines("不正当竞争.txt")); allLines.addAll(distinctLines("人名.txt")); @@ -97,7 +97,7 @@ public class DataInitTest { Collections.sort(disctinct); - final String target = "D:\\github\\sensitive-word\\src\\main\\resources\\dict.txt"; + final String target = "D:\\github\\sensitive-word\\src\\main\\resources\\sensitive_word_dict.txt"; FileUtil.write(target, disctinct); } @@ -105,7 +105,7 @@ public class DataInitTest { @Test @Ignore public void oneWordTest() { - final String source = "D:\\_github\\sensitive-word\\src\\main\\resources\\dict.txt"; + final String source = "D:\\_github\\sensitive-word\\src\\main\\resources\\sensitive_word_dict.txt"; List lines = FileUtil.readAllLines(source); for(int i = 0; i < lines.size(); i++) { diff --git a/src/test/java/com/github/houbb/sensitive/word/data/DataUtil.java b/src/test/java/com/github/houbb/sensitive/word/data/DataUtil.java index 207ef05..4f6a66c 100644 --- a/src/test/java/com/github/houbb/sensitive/word/data/DataUtil.java +++ b/src/test/java/com/github/houbb/sensitive/word/data/DataUtil.java @@ -38,7 +38,7 @@ public class DataUtil { @Test @Ignore public void singleCharTest() { - final String path = "D:\\github\\sensitive-word\\src\\main\\resources\\dict.txt"; + final String path = "D:\\github\\sensitive-word\\src\\main\\resources\\sensitive_word_dict.txt"; List stringList = FileUtil.readAllLines(path); for(String s : stringList) { diff --git a/src/test/java/com/github/houbb/sensitive/word/data/DictNumTest.java b/src/test/java/com/github/houbb/sensitive/word/data/DictNumTest.java index 07e56e8..7b23512 100644 --- a/src/test/java/com/github/houbb/sensitive/word/data/DictNumTest.java +++ b/src/test/java/com/github/houbb/sensitive/word/data/DictNumTest.java @@ -26,8 +26,8 @@ public class DictNumTest { @Test @Ignore public void formatTest() { - final String sourceFile = "D:\\_github\\sensitive-word\\src\\main\\resources\\dict.txt"; - final String targetFile = "D:\\_github\\sensitive-word\\src\\main\\resources\\dict.txt"; + final String sourceFile = "D:\\_github\\sensitive-word\\src\\main\\resources\\sensitive_word_dict.txt"; + final String targetFile = "D:\\_github\\sensitive-word\\src\\main\\resources\\sensitive_word_dict.txt"; List words = FileUtil.readAllLines(sourceFile); // List formats = CollectionUtil.toList(words, new IHandler() { diff --git a/src/test/java/com/github/houbb/sensitive/word/data/DictRemoveSingleTest.java b/src/test/java/com/github/houbb/sensitive/word/data/DictRemoveSingleTest.java index 4d4d597..768bb3a 100644 --- a/src/test/java/com/github/houbb/sensitive/word/data/DictRemoveSingleTest.java +++ b/src/test/java/com/github/houbb/sensitive/word/data/DictRemoveSingleTest.java @@ -27,7 +27,7 @@ public class DictRemoveSingleTest { @Ignore public void removeSingleWord() { final String sourceFile = "D:\\code\\github\\sensitive-word\\src\\test\\resources\\dict_20231117.txt"; - final String targetFile = "D:\\code\\github\\sensitive-word\\src\\main\\resources\\dict.txt"; + final String targetFile = "D:\\code\\github\\sensitive-word\\src\\main\\resources\\sensitive_word_dict.txt"; List words = FileUtil.readAllLines(sourceFile); diff --git a/src/test/java/com/github/houbb/sensitive/word/data/DictRemoveTwoEnglishTest.java b/src/test/java/com/github/houbb/sensitive/word/data/DictRemoveTwoEnglishTest.java index a3bd712..f476b5e 100644 --- a/src/test/java/com/github/houbb/sensitive/word/data/DictRemoveTwoEnglishTest.java +++ b/src/test/java/com/github/houbb/sensitive/word/data/DictRemoveTwoEnglishTest.java @@ -17,7 +17,7 @@ import java.util.List; public class DictRemoveTwoEnglishTest { public static void main(String[] args) { - final String sourceFile = "D:\\github\\sensitive-word\\src\\main\\resources\\dict.txt"; + final String sourceFile = "D:\\github\\sensitive-word\\src\\main\\resources\\sensitive_word_dict.txt"; final String targetFile = "D:\\github\\sensitive-word\\src\\test\\resources\\dict_v20240407.txt"; List words = FileUtil.readAllLines(sourceFile); diff --git a/src/test/java/com/github/houbb/sensitive/word/data/DictSlimTest.java b/src/test/java/com/github/houbb/sensitive/word/data/DictSlimTest.java index 7507aa1..535f121 100644 --- a/src/test/java/com/github/houbb/sensitive/word/data/DictSlimTest.java +++ b/src/test/java/com/github/houbb/sensitive/word/data/DictSlimTest.java @@ -33,8 +33,8 @@ public class DictSlimTest { @Test @Ignore public void formatTest() { - final String sourceFile = "D:\\github\\sensitive-word\\src\\main\\resources\\dict.txt"; - final String targetFile = "D:\\github\\sensitive-word\\src\\main\\resources\\dict.txt"; + final String sourceFile = "D:\\github\\sensitive-word\\src\\main\\resources\\sensitive_word_dict.txt"; + final String targetFile = "D:\\github\\sensitive-word\\src\\main\\resources\\sensitive_word_dict.txt"; List words = FileUtil.readAllLines(sourceFile); @@ -67,8 +67,8 @@ public class DictSlimTest { @Test @Ignore public void removeTest() { - final String sourceFile = "D:\\github\\sensitive-word\\src\\main\\resources\\dict.txt"; - final String targetFile = "D:\\github\\sensitive-word\\src\\main\\resources\\dict.txt"; + final String sourceFile = "D:\\github\\sensitive-word\\src\\main\\resources\\sensitive_word_dict.txt"; + final String targetFile = "D:\\github\\sensitive-word\\src\\main\\resources\\sensitive_word_dict.txt"; List words = FileUtil.readAllLines(sourceFile); @@ -92,8 +92,8 @@ public class DictSlimTest { */ @Test public void removeNumberMappingTest() { - final String sourceFile = "D:\\_github\\sensitive-word\\src\\main\\resources\\dict.txt"; - final String targetFile = "D:\\_github\\sensitive-word\\src\\main\\resources\\dict.txt"; + final String sourceFile = "D:\\_github\\sensitive-word\\src\\main\\resources\\sensitive_word_dict.txt"; + final String targetFile = "D:\\_github\\sensitive-word\\src\\main\\resources\\sensitive_word_dict.txt"; List words = FileUtil.readAllLines(sourceFile); List formats = CollectionUtil.toList(words, new IHandler() { diff --git a/src/test/java/com/github/houbb/sensitive/word/memory/DataMemoryTest.java b/src/test/java/com/github/houbb/sensitive/word/memory/DataMemoryTest.java index 6e30cae..af1321d 100644 --- a/src/test/java/com/github/houbb/sensitive/word/memory/DataMemoryTest.java +++ b/src/test/java/com/github/houbb/sensitive/word/memory/DataMemoryTest.java @@ -22,7 +22,7 @@ public class DataMemoryTest { */ @Test public void hashMapTest() { - List allLines = StreamUtil.readAllLines("/dict.txt"); + List allLines = StreamUtil.readAllLines("/sensitive_word_dict.txt"); IWordData wordData = WordDatas.defaults(); wordData.initWordData(allLines); @@ -36,7 +36,7 @@ public class DataMemoryTest { //33.4 MB @Test public void treeTest() { - List allLines = StreamUtil.readAllLines("/dict.txt"); + List allLines = StreamUtil.readAllLines("/sensitive_word_dict.txt"); IWordData wordData = WordDatas.tree(); wordData.initWordData(allLines);