From 72ec11c7372ae555b9b16c790057ec7244c093fd Mon Sep 17 00:00:00 2001 From: guor Date: Thu, 28 Nov 2019 18:08:39 +0800 Subject: [PATCH 1/7] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E6=9C=BA=E5=99=A8?= =?UTF-8?q?=E5=AD=A6=E4=B9=A0=E6=A8=A1=E5=9E=8B=E7=9A=84=E5=BC=95=E5=85=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../com.pgmmers.radar/dal/model/MoldDal.java | 7 ++ .../dal/model/impl/MoldDalImpl.java | 44 ++++++++++ .../vo/model/MoldParamVO.java | 24 +++++ .../com.pgmmers.radar/vo/model/MoldVO.java | 88 +++++++++++++++++++ .../com/pgmmers/radar/mapper/MoldMapper.java | 7 ++ .../pgmmers/radar/mapper/MoldParamMapper.java | 7 ++ .../java/com/pgmmers/radar/model/MoldPO.java | 28 ++++++ .../com/pgmmers/radar/model/MoldParamPO.java | 17 ++++ .../service/impl/dnn/TensorFlowEstimator.java | 19 ++++ .../service/impl/model/MoldServiceImpl.java | 19 ++++ .../pgmmers/radar/service/dnn/Estimator.java | 23 +++++ .../radar/service/model/MoldService.java | 7 ++ 12 files changed, 290 insertions(+) create mode 100644 radar-dal/src/main/java/com.pgmmers.radar/dal/model/MoldDal.java create mode 100644 radar-dal/src/main/java/com.pgmmers.radar/dal/model/impl/MoldDalImpl.java create mode 100644 radar-dal/src/main/java/com.pgmmers.radar/vo/model/MoldParamVO.java create mode 100644 radar-dal/src/main/java/com.pgmmers.radar/vo/model/MoldVO.java create mode 100644 radar-dao/src/main/java/com/pgmmers/radar/mapper/MoldMapper.java create mode 100644 radar-dao/src/main/java/com/pgmmers/radar/mapper/MoldParamMapper.java create mode 100644 radar-dao/src/main/java/com/pgmmers/radar/model/MoldPO.java create mode 100644 radar-dao/src/main/java/com/pgmmers/radar/model/MoldParamPO.java create mode 100644 radar-service-impl/src/main/java/com/pgmmers/radar/service/impl/dnn/TensorFlowEstimator.java create mode 100644 radar-service-impl/src/main/java/com/pgmmers/radar/service/impl/model/MoldServiceImpl.java create mode 100644 radar-service/src/main/java/com/pgmmers/radar/service/dnn/Estimator.java create mode 100644 radar-service/src/main/java/com/pgmmers/radar/service/model/MoldService.java diff --git a/radar-dal/src/main/java/com.pgmmers.radar/dal/model/MoldDal.java b/radar-dal/src/main/java/com.pgmmers.radar/dal/model/MoldDal.java new file mode 100644 index 0000000..642a973 --- /dev/null +++ b/radar-dal/src/main/java/com.pgmmers.radar/dal/model/MoldDal.java @@ -0,0 +1,7 @@ +package com.pgmmers.radar.dal.model; + +import com.pgmmers.radar.vo.model.MoldVO; + +public interface MoldDal { + MoldVO get(Long id); +} diff --git a/radar-dal/src/main/java/com.pgmmers.radar/dal/model/impl/MoldDalImpl.java b/radar-dal/src/main/java/com.pgmmers.radar/dal/model/impl/MoldDalImpl.java new file mode 100644 index 0000000..0103200 --- /dev/null +++ b/radar-dal/src/main/java/com.pgmmers.radar/dal/model/impl/MoldDalImpl.java @@ -0,0 +1,44 @@ +package com.pgmmers.radar.dal.model.impl; + +import com.pgmmers.radar.dal.model.MoldDal; +import com.pgmmers.radar.mapper.MoldMapper; +import com.pgmmers.radar.mapper.MoldParamMapper; +import com.pgmmers.radar.model.MoldPO; +import com.pgmmers.radar.model.MoldParamPO; +import com.pgmmers.radar.vo.model.MoldParamVO; +import com.pgmmers.radar.vo.model.MoldVO; +import org.springframework.beans.BeanUtils; +import org.springframework.stereotype.Service; +import tk.mybatis.mapper.entity.Example; + +import javax.annotation.Resource; +import java.util.List; +import java.util.stream.Collectors; + +@Service +public class MoldDalImpl implements MoldDal { + @Resource + private MoldMapper moldMapper; + @Resource + private MoldParamMapper moldParamMapper; + + @Override + public MoldVO get(Long id) { + MoldVO vo = new MoldVO(); + MoldPO moldPO = moldMapper.selectByPrimaryKey(id); + if (moldPO != null) { + Example example = new Example(MoldParamPO.class); + example.createCriteria().andEqualTo("moldId", id); + List moldParamList = moldParamMapper.selectByExample(example); + List list = moldParamList.stream().map(moldParamPO -> { + MoldParamVO moldParamVO = new MoldParamVO(); + BeanUtils.copyProperties(moldParamPO, moldParamVO); + return moldParamVO; + }).collect(Collectors.toList()); + BeanUtils.copyProperties(moldPO, vo); + vo.setParams(list); + return vo; + } + return null; + } +} diff --git a/radar-dal/src/main/java/com.pgmmers.radar/vo/model/MoldParamVO.java b/radar-dal/src/main/java/com.pgmmers.radar/vo/model/MoldParamVO.java new file mode 100644 index 0000000..27cf7f6 --- /dev/null +++ b/radar-dal/src/main/java/com.pgmmers.radar/vo/model/MoldParamVO.java @@ -0,0 +1,24 @@ +package com.pgmmers.radar.vo.model; + +import java.io.Serializable; + +/** + *

+ * 机器学习模型配置,目前只考虑输入层为离散值的情况,不考虑需要词嵌入和融入卷积层,其中 + * 离散值通过表达式取数从前置流程传递过来. + *

+ * + * @author guor + * @date 2019/11/28 + */ +public class MoldParamVO implements Serializable { + private Long id; + /** + * 取数表达式 + */ + private String exp; + /** + * 参数顺序,显式配置,强制有序 + */ + private int order; +} diff --git a/radar-dal/src/main/java/com.pgmmers.radar/vo/model/MoldVO.java b/radar-dal/src/main/java/com.pgmmers.radar/vo/model/MoldVO.java new file mode 100644 index 0000000..444f64e --- /dev/null +++ b/radar-dal/src/main/java/com.pgmmers.radar/vo/model/MoldVO.java @@ -0,0 +1,88 @@ +package com.pgmmers.radar.vo.model; + +import java.io.Serializable; +import java.util.Date; +import java.util.List; + +/** + *

+ * 机器学习模型配置,定义模型文件路径和参数 + *

+ * + * @author guor + * @date 2019/11/28 + */ +public class MoldVO implements Serializable { + /** + * 自增ID,主键 + */ + private Long id; + /** + * 模型名称 + */ + private String name; + /** + * 输入层参数的key + */ + private String feed; + /** + * 模型参数定义,需要注意参数顺序,改变顺序会使模型调用出错或者失效 + */ + private List params; + /** + * 模型文件路径 + */ + private String path; + /** + * 模型更新时间 + */ + private Date updateDate; + + public Long getId() { + return id; + } + + public void setId(Long id) { + this.id = id; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public String getFeed() { + return feed; + } + + public void setFeed(String feed) { + this.feed = feed; + } + + public List getParams() { + return params; + } + + public void setParams(List params) { + this.params = params; + } + + public String getPath() { + return path; + } + + public void setPath(String path) { + this.path = path; + } + + public Date getUpdateDate() { + return updateDate; + } + + public void setUpdateDate(Date updateDate) { + this.updateDate = updateDate; + } +} diff --git a/radar-dao/src/main/java/com/pgmmers/radar/mapper/MoldMapper.java b/radar-dao/src/main/java/com/pgmmers/radar/mapper/MoldMapper.java new file mode 100644 index 0000000..22930bd --- /dev/null +++ b/radar-dao/src/main/java/com/pgmmers/radar/mapper/MoldMapper.java @@ -0,0 +1,7 @@ +package com.pgmmers.radar.mapper; + +import com.pgmmers.radar.model.MoldPO; +import tk.mybatis.mapper.common.Mapper; + +public interface MoldMapper extends Mapper { +} diff --git a/radar-dao/src/main/java/com/pgmmers/radar/mapper/MoldParamMapper.java b/radar-dao/src/main/java/com/pgmmers/radar/mapper/MoldParamMapper.java new file mode 100644 index 0000000..583d663 --- /dev/null +++ b/radar-dao/src/main/java/com/pgmmers/radar/mapper/MoldParamMapper.java @@ -0,0 +1,7 @@ +package com.pgmmers.radar.mapper; + +import com.pgmmers.radar.model.MoldParamPO; +import tk.mybatis.mapper.common.Mapper; + +public interface MoldParamMapper extends Mapper { +} diff --git a/radar-dao/src/main/java/com/pgmmers/radar/model/MoldPO.java b/radar-dao/src/main/java/com/pgmmers/radar/model/MoldPO.java new file mode 100644 index 0000000..55f7974 --- /dev/null +++ b/radar-dao/src/main/java/com/pgmmers/radar/model/MoldPO.java @@ -0,0 +1,28 @@ +package com.pgmmers.radar.model; + +import javax.persistence.Table; +import java.util.Date; + +@Table(name = "engine_mold") +public class MoldPO { + /** + * 自增ID,主键 + */ + private Long id; + /** + * 模型名称 + */ + private String name; + /** + * 输入层参数的key + */ + private String feed; + /** + * 模型文件路径 + */ + private String path; + /** + * 模型更新时间 + */ + private Date updateDate; +} diff --git a/radar-dao/src/main/java/com/pgmmers/radar/model/MoldParamPO.java b/radar-dao/src/main/java/com/pgmmers/radar/model/MoldParamPO.java new file mode 100644 index 0000000..5ea76d8 --- /dev/null +++ b/radar-dao/src/main/java/com/pgmmers/radar/model/MoldParamPO.java @@ -0,0 +1,17 @@ +package com.pgmmers.radar.model; + +import javax.persistence.Table; + +@Table(name = "engine_mold_param") +public class MoldParamPO { + private Long id; + private Long moldId; + /** + * 取数表达式 + */ + private String exp; + /** + * 参数顺序,显式配置,强制有序 + */ + private int order; +} diff --git a/radar-service-impl/src/main/java/com/pgmmers/radar/service/impl/dnn/TensorFlowEstimator.java b/radar-service-impl/src/main/java/com/pgmmers/radar/service/impl/dnn/TensorFlowEstimator.java new file mode 100644 index 0000000..28a4010 --- /dev/null +++ b/radar-service-impl/src/main/java/com/pgmmers/radar/service/impl/dnn/TensorFlowEstimator.java @@ -0,0 +1,19 @@ +package com.pgmmers.radar.service.impl.dnn; + +import com.pgmmers.radar.service.dnn.Estimator; +import com.pgmmers.radar.service.model.MoldService; +import org.springframework.stereotype.Component; + +import javax.annotation.Resource; +import java.util.Map; + +@Component +public class TensorFlowEstimator implements Estimator { + @Resource + private MoldService moldService; + + @Override + public double predict(Long modelId, Map> data) { + return 0; + } +} diff --git a/radar-service-impl/src/main/java/com/pgmmers/radar/service/impl/model/MoldServiceImpl.java b/radar-service-impl/src/main/java/com/pgmmers/radar/service/impl/model/MoldServiceImpl.java new file mode 100644 index 0000000..971c68e --- /dev/null +++ b/radar-service-impl/src/main/java/com/pgmmers/radar/service/impl/model/MoldServiceImpl.java @@ -0,0 +1,19 @@ +package com.pgmmers.radar.service.impl.model; + +import com.pgmmers.radar.dal.model.MoldDal; +import com.pgmmers.radar.service.model.MoldService; +import com.pgmmers.radar.vo.model.MoldVO; +import org.springframework.stereotype.Service; + +import javax.annotation.Resource; + +@Service +public class MoldServiceImpl implements MoldService { + @Resource + private MoldDal moldDal; + + @Override + public MoldVO get(Long id) { + return moldDal.get(id); + } +} diff --git a/radar-service/src/main/java/com/pgmmers/radar/service/dnn/Estimator.java b/radar-service/src/main/java/com/pgmmers/radar/service/dnn/Estimator.java new file mode 100644 index 0000000..5fb3026 --- /dev/null +++ b/radar-service/src/main/java/com/pgmmers/radar/service/dnn/Estimator.java @@ -0,0 +1,23 @@ +package com.pgmmers.radar.service.dnn; + +import java.util.Map; + +/** + *

+ * 机器学习模型执行器接口 + *

+ *

+ * 该接口内置TensorFlow实现,目前版本只考虑输入层为离散值的情况,不考虑词嵌入和融入卷积层,其中 + * 离散值通过表达式取数从前置流程传递过来,模型的预测结果为事件评分。 + *

+ *

+ * 模型抽象:y=f(x) + *

+ * + * @author guor + * @date 2019/11/28 + */ +public interface Estimator { + + double predict(Long modelId, Map> data); +} diff --git a/radar-service/src/main/java/com/pgmmers/radar/service/model/MoldService.java b/radar-service/src/main/java/com/pgmmers/radar/service/model/MoldService.java new file mode 100644 index 0000000..e7c7884 --- /dev/null +++ b/radar-service/src/main/java/com/pgmmers/radar/service/model/MoldService.java @@ -0,0 +1,7 @@ +package com.pgmmers.radar.service.model; + +import com.pgmmers.radar.vo.model.MoldVO; + +public interface MoldService { + MoldVO get(Long id); +} From 235de065546616a9584d01524bb67fb14f7e5504 Mon Sep 17 00:00:00 2001 From: "feihu.wang" Date: Mon, 18 Nov 2019 14:41:34 +0800 Subject: [PATCH 2/7] feats: update version. Signed-off-by: feihu.wang --- pom.xml | 2 +- radar-admin/pom.xml | 2 +- radar-commons/pom.xml | 2 +- radar-dal/pom.xml | 2 +- radar-dao/pom.xml | 2 +- radar-engine/pom.xml | 2 +- radar-service-impl/pom.xml | 2 +- radar-service/pom.xml | 2 +- 8 files changed, 8 insertions(+), 8 deletions(-) diff --git a/pom.xml b/pom.xml index 2c16c0a..a271035 100644 --- a/pom.xml +++ b/pom.xml @@ -19,7 +19,7 @@ com.pgmmers radar - 1.0.2-SNAPSHOT + 1.1.0-SNAPSHOT radar Demo project for Spring Boot pom diff --git a/radar-admin/pom.xml b/radar-admin/pom.xml index 4738f27..ae098b7 100644 --- a/radar-admin/pom.xml +++ b/radar-admin/pom.xml @@ -5,7 +5,7 @@ radar com.pgmmers - 1.0.2-SNAPSHOT + 1.1.0-SNAPSHOT 4.0.0 radar-admin diff --git a/radar-commons/pom.xml b/radar-commons/pom.xml index 6eb123f..56373f9 100644 --- a/radar-commons/pom.xml +++ b/radar-commons/pom.xml @@ -6,7 +6,7 @@ radar com.pgmmers - 1.0.2-SNAPSHOT + 1.1.0-SNAPSHOT 4.0.0 diff --git a/radar-dal/pom.xml b/radar-dal/pom.xml index 91b7cd2..9a64c0f 100644 --- a/radar-dal/pom.xml +++ b/radar-dal/pom.xml @@ -5,7 +5,7 @@ radar com.pgmmers - 1.0.2-SNAPSHOT + 1.1.0-SNAPSHOT 4.0.0 diff --git a/radar-dao/pom.xml b/radar-dao/pom.xml index 590db74..685b38e 100644 --- a/radar-dao/pom.xml +++ b/radar-dao/pom.xml @@ -5,7 +5,7 @@ radar com.pgmmers - 1.0.2-SNAPSHOT + 1.1.0-SNAPSHOT 4.0.0 radar-dao diff --git a/radar-engine/pom.xml b/radar-engine/pom.xml index ef4f7db..1be15ab 100644 --- a/radar-engine/pom.xml +++ b/radar-engine/pom.xml @@ -5,7 +5,7 @@ radar com.pgmmers - 1.0.2-SNAPSHOT + 1.1.0-SNAPSHOT 4.0.0 diff --git a/radar-service-impl/pom.xml b/radar-service-impl/pom.xml index d0f01c1..aa2c500 100644 --- a/radar-service-impl/pom.xml +++ b/radar-service-impl/pom.xml @@ -5,7 +5,7 @@ radar com.pgmmers - 1.0.2-SNAPSHOT + 1.1.0-SNAPSHOT 4.0.0 diff --git a/radar-service/pom.xml b/radar-service/pom.xml index 502ce75..06c42f3 100644 --- a/radar-service/pom.xml +++ b/radar-service/pom.xml @@ -5,7 +5,7 @@ radar com.pgmmers - 1.0.2-SNAPSHOT + 1.1.0-SNAPSHOT 4.0.0 From fa05b3dc6d86adec07d8d039275e89104e74fcb7 Mon Sep 17 00:00:00 2001 From: "feihu.wang" Date: Tue, 19 Nov 2019 14:25:37 +0800 Subject: [PATCH 3/7] =?UTF-8?q?fix:=20=E4=BF=AE=E5=A4=8D=E8=A7=84=E5=88=99?= =?UTF-8?q?=E9=A2=91=E7=B9=81=E4=BF=AE=E6=94=B9=E5=8F=AF=E8=83=BD=E4=BC=9A?= =?UTF-8?q?=E5=AF=BC=E8=87=B4=E5=86=85=E5=AD=98=E6=B3=84=E6=BC=8F=E7=9A=84?= =?UTF-8?q?=E9=A3=8E=E9=99=A9.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: feihu.wang --- README.md | 6 +++--- .../pgmmers/radar/controller/RuleApiController.java | 2 -- .../java/com/pgmmers/radar/util/GroovyScriptUtil.java | 8 +++++++- .../service/impl/model/AbstractionServiceImpl.java | 10 +++++++++- .../radar/service/impl/model/RuleServiceImpl.java | 8 ++++++++ 5 files changed, 27 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 0ce61e9..41dd942 100644 --- a/README.md +++ b/README.md @@ -17,8 +17,8 @@ The project code called Radar, like the code, monitor the transaction at the bac * 配置简单,开箱即用! ## 相关站点 - Github: https://github.com/wfh45678/radar Gitee: https://gitee.com/freshday/radar // 码云为镜像网站,贡献代码请提交到 github + Github: https://github.com/wfh45678/radar 官网: http://radar.pgmmer.top Wiki: https://gitee.com/freshday/radar/wikis/home @@ -56,7 +56,7 @@ The project code called Radar, like the code, monitor the transaction at the bac * Mysql : 本项目中关系数据库,主要用于存放 风险模型的元信息。 -* MongoDB: 用于存放事件JSON, 提供基本统计学计算(例如:max, min, sum, avg, ), +* MongoDB: 用于存放事件JSON, 提供基本统计学计算(例如:max, min, sum, avg,), 复杂的统计学概念(sd,variance, etc...)在内存中计算。 * Redis: 提供缓存支持,Engine 利用发布订阅特性监听管理端相关配置的更新 @@ -104,7 +104,7 @@ https://gitee.com/freshday/radar/wikis/manual?sort_id=1637446 建议大家自行注册用户,避免使用同样的测试账号受干扰. ## 未完待续 - +[Release Note:](https://gitee.com/freshday/radar/wikis/release%20note?sort_id=1723765) https://gitee.com/freshday/radar/wikis/release%20note?sort_id=1723765 ### 重大特性 * 支持机器学习 * 数据分析平台 diff --git a/radar-admin/src/main/java/com/pgmmers/radar/controller/RuleApiController.java b/radar-admin/src/main/java/com/pgmmers/radar/controller/RuleApiController.java index 11e0009..f83043f 100644 --- a/radar-admin/src/main/java/com/pgmmers/radar/controller/RuleApiController.java +++ b/radar-admin/src/main/java/com/pgmmers/radar/controller/RuleApiController.java @@ -47,8 +47,6 @@ public class RuleApiController { @PutMapping public CommonResult save(@RequestBody RuleVO rule, HttpServletRequest request) { -// HttpSession session = request.getSession(); -// UserVO user = (UserVO) session.getAttribute("user"); return ruleService.save(rule, contextHolder.getContext().getUsername()); } diff --git a/radar-commons/src/main/java/com/pgmmers/radar/util/GroovyScriptUtil.java b/radar-commons/src/main/java/com/pgmmers/radar/util/GroovyScriptUtil.java index 87c5b80..f9c72e1 100644 --- a/radar-commons/src/main/java/com/pgmmers/radar/util/GroovyScriptUtil.java +++ b/radar-commons/src/main/java/com/pgmmers/radar/util/GroovyScriptUtil.java @@ -3,7 +3,6 @@ package com.pgmmers.radar.util; import groovy.lang.GroovyClassLoader; import groovy.lang.GroovyObject; import org.springframework.beans.factory.annotation.AutowiredAnnotationBeanPostProcessor; - import java.util.HashMap; import java.util.Map; @@ -58,4 +57,11 @@ public class GroovyScriptUtil { return null; } } + + /** + * 删除不在使用的脚本关联的groovy object, 不然内存有溢出风险。 + */ + public static void removeInactiveScript(String script){ + passedClassMap.remove(script.hashCode() + ""); + } } diff --git a/radar-service-impl/src/main/java/com/pgmmers/radar/service/impl/model/AbstractionServiceImpl.java b/radar-service-impl/src/main/java/com/pgmmers/radar/service/impl/model/AbstractionServiceImpl.java index 6e8b51d..dac6160 100644 --- a/radar-service-impl/src/main/java/com/pgmmers/radar/service/impl/model/AbstractionServiceImpl.java +++ b/radar-service-impl/src/main/java/com/pgmmers/radar/service/impl/model/AbstractionServiceImpl.java @@ -9,6 +9,7 @@ import com.pgmmers.radar.service.cache.CacheService; import com.pgmmers.radar.service.cache.SubscribeHandle; import com.pgmmers.radar.service.common.CommonResult; import com.pgmmers.radar.service.model.AbstractionService; +import com.pgmmers.radar.util.GroovyScriptUtil; import com.pgmmers.radar.vo.model.AbstractionVO; import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; @@ -79,10 +80,17 @@ public class AbstractionServiceImpl implements AbstractionService, SubscribeHand @Override public CommonResult save(AbstractionVO abstraction) { CommonResult result = new CommonResult(); + if (abstraction.getId() != null) { + AbstractionVO oldAbs = abstractionDal.get(abstraction.getId()); + // 如果规则有更新,以前的编译好的groovy 对象用不到了,应该删除。 + if (!oldAbs.getRuleScript().equals(abstraction.getRuleScript())) { + GroovyScriptUtil.removeInactiveScript(oldAbs.getRuleScript()); + } + } int count = abstractionDal.save(abstraction); if (count > 0) { if(StringUtils.isEmpty(abstraction.getName())){ - abstraction.setName("abstraction_"+abstraction.getId()); + abstraction.setName("abstraction_" + abstraction.getId()); abstractionDal.save(abstraction); } diff --git a/radar-service-impl/src/main/java/com/pgmmers/radar/service/impl/model/RuleServiceImpl.java b/radar-service-impl/src/main/java/com/pgmmers/radar/service/impl/model/RuleServiceImpl.java index 223f11c..7c60577 100644 --- a/radar-service-impl/src/main/java/com/pgmmers/radar/service/impl/model/RuleServiceImpl.java +++ b/radar-service-impl/src/main/java/com/pgmmers/radar/service/impl/model/RuleServiceImpl.java @@ -14,6 +14,7 @@ import com.pgmmers.radar.service.cache.SubscribeHandle; import com.pgmmers.radar.service.common.CommonResult; import com.pgmmers.radar.service.model.RuleService; import com.pgmmers.radar.service.search.SearchEngineService; +import com.pgmmers.radar.util.GroovyScriptUtil; import com.pgmmers.radar.vo.model.*; import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; @@ -89,6 +90,13 @@ public class RuleServiceImpl implements RuleService, SubscribeHandle { @Override public CommonResult save(RuleVO rule,String merchantCode) { CommonResult result = new CommonResult(); + if (rule.getId() != null) { + RuleVO oldRule = ruleDal.get(rule.getId()); + // 如果规则有更新,以前的编译好的groovy 对象用不到了,应该删除。 + if (!oldRule.getScripts().equals(rule.getScripts())) { + GroovyScriptUtil.removeInactiveScript(oldRule.getScripts()); + } + } int count = ruleDal.save(rule); if (count > 0) { if(StringUtils.isEmpty(rule.getName())){ From 95b4b5dd308350fe896d69e831ef049d142f3376 Mon Sep 17 00:00:00 2001 From: trent <1206861220@qq.com> Date: Tue, 19 Nov 2019 22:09:12 +0800 Subject: [PATCH 4/7] =?UTF-8?q?1.=20=E4=BF=AE=E6=94=B9pom.xml=20=E6=96=87?= =?UTF-8?q?=E4=BB=B6=E7=BB=93=E6=9E=84=EF=BC=8C=E5=AF=B9=E4=BE=9D=E8=B5=96?= =?UTF-8?q?=E7=89=88=E6=9C=AC=E8=BF=9B=E8=A1=8C=E9=9B=86=E4=B8=AD=E7=AE=A1?= =?UTF-8?q?=E7=90=86=202.=20=E8=A7=A3=E5=86=B3=E4=BA=86=E4=B8=80=E5=A4=84?= =?UTF-8?q?=E7=89=88=E6=9C=AC=E5=86=B2=E7=AA=81(admin=20=E4=B8=ADHdrHistog?= =?UTF-8?q?ram)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pom.xml | 142 +++++++++++++++++++++++++++++-------- radar-admin/pom.xml | 66 ----------------- radar-commons/pom.xml | 15 ---- radar-dal/pom.xml | 22 +----- radar-dao/pom.xml | 38 ---------- radar-engine/pom.xml | 42 ----------- radar-service-impl/pom.xml | 19 ----- radar-service/pom.xml | 4 -- 8 files changed, 112 insertions(+), 236 deletions(-) diff --git a/pom.xml b/pom.xml index a271035..21f361b 100644 --- a/pom.xml +++ b/pom.xml @@ -24,8 +24,13 @@ Demo project for Spring Boot pom - UTF-8 1.8 + UTF-8 + + 4.0.0 + 1.3.7 + 5.1.47 + 2.1.7.RELEASE 8.5.37 @@ -35,7 +40,7 @@ org.springframework.boot spring-boot-dependencies - 2.1.7.RELEASE + ${springboot.version} pom import @@ -43,52 +48,42 @@ org.springframework.boot spring-boot-devtools - 2.1.7.RELEASE + ${springboot.version} runtime true org.springframework.boot spring-boot-starter-web - 2.1.7.RELEASE + ${springboot.version} org.springframework.boot spring-boot-starter-test - 2.1.7.RELEASE + ${springboot.version} test org.mybatis.generator mybatis-generator-core - 1.3.7 + ${mybatisGenerator.version} - - com.github.pagehelper - pagehelper - 5.1.10 - - - + tk.mybatis mapper - 4.0.0 + ${mapper.version} + tk.mybatis mapper-spring-boot-starter 2.1.5 - - org.apache.poi - poi - 3.9 - org.apache.poi poi-ooxml @@ -128,31 +123,29 @@ com.google.guava guava + + org.hdrhistogram + HdrHistogram + org.springframework.boot spring-boot-starter-thymeleaf - 2.1.7.RELEASE + ${springboot.version} org.springframework.boot spring-boot-starter-actuator - 2.1.7.RELEASE - - - - org.springframework.data - spring-data-redis - 2.1.10.RELEASE + ${springboot.version} org.springframework.boot spring-boot-starter-data-redis - 2.1.7.RELEASE + ${springboot.version} @@ -164,13 +157,13 @@ mysql mysql-connector-java - 5.1.47 + ${mysql.version} org.springframework.boot spring-boot-starter-aop - 2.1.7.RELEASE + ${springboot.version} @@ -202,11 +195,98 @@ 28.0-jre + + + io.jsonwebtoken + jjwt + 0.9.1 + + + commons-io + commons-io + 2.6 + + + + + org.codehaus.groovy + groovy-all + 2.4.6 + + + commons-lang + commons-lang + 2.6 + + + + + org.mybatis + mybatis + 3.4.6 + - + + + + org.springframework.boot + spring-boot-maven-plugin + ${springboot.version} + + + + repackage + + + + + + org.apache.maven.plugins + maven-compiler-plugin + 3.5.1 + + 1.8 + 1.8 + UTF-8 + + + + org.mybatis.generator + mybatis-generator-maven-plugin + ${mybatisGenerator.version} + + + ${basedir}/src/main/resources/generator/generatorConfig.xml + + true + true + + + + mysql + mysql-connector-java + ${mysql.version} + + + tk.mybatis + mapper + ${mapper.version} + + + com.pgmmers + radar-commons + ${project.version} + + + + + diff --git a/radar-admin/pom.xml b/radar-admin/pom.xml index ae098b7..297e842 100644 --- a/radar-admin/pom.xml +++ b/radar-admin/pom.xml @@ -21,10 +21,6 @@ org.springframework.boot spring-boot-starter-web - - org.springframework.boot - spring-boot-starter-jdbc - mysql @@ -33,43 +29,21 @@ org.springframework.boot spring-boot-starter-test - test com.alibaba druid-spring-boot-starter - - junit - junit - commons-io commons-io - 2.6 - - - tk.mybatis - mapper - - tk.mybatis mapper-spring-boot-starter - - com.alibaba - fastjson - - - - org.apache.commons - commons-lang3 - - org.springframework.boot @@ -79,60 +53,28 @@ org.springframework.boot spring-boot-starter-actuator - - com.pgmmers - radar-service - ${project.version} - com.pgmmers radar-service-impl ${project.version} - - com.github.pagehelper - pagehelper-spring-boot-starter - - - - com.github.pagehelper - pagehelper - - org.springframework.boot spring-boot-starter-data-redis - - org.springframework.data - spring-data-redis - - - org.springframework.boot - spring-boot-starter-cache - org.springframework.boot spring-boot-starter-aop - - io.springfox - springfox-swagger2 - io.springfox springfox-swagger-ui - - com.google.guava - guava - io.jsonwebtoken jjwt - 0.9.1 @@ -142,14 +84,6 @@ org.springframework.boot spring-boot-maven-plugin - 2.1.7.RELEASE - - - - repackage - - - diff --git a/radar-commons/pom.xml b/radar-commons/pom.xml index 56373f9..d135a00 100644 --- a/radar-commons/pom.xml +++ b/radar-commons/pom.xml @@ -21,10 +21,6 @@ org.springframework spring-context-support - - org.apache.poi - poi - org.apache.poi poi-ooxml @@ -33,32 +29,27 @@ org.mybatis.generator mybatis-generator-core - 1.3.7 org.codehaus.groovy groovy-all - 2.4.6 org.apache.commons commons-lang3 - 3.9 commons-lang commons-lang - 2.6 junit junit - 4.12 @@ -67,12 +58,6 @@ org.apache.maven.plugins maven-compiler-plugin - 3.5.1 - - 1.8 - 1.8 - UTF-8 - diff --git a/radar-dal/pom.xml b/radar-dal/pom.xml index 9a64c0f..e6aca35 100644 --- a/radar-dal/pom.xml +++ b/radar-dal/pom.xml @@ -18,23 +18,10 @@ radar-dao ${project.version} - - org.springframework.boot - spring-boot-starter - - - org.mybatis - mybatis - 3.4.6 - - - tk.mybatis - mapper - com.github.pagehelper - pagehelper + pagehelper-spring-boot-starter com.fasterxml.jackson.core @@ -44,18 +31,11 @@ - org.apache.maven.plugins maven-compiler-plugin - 3.5.1 - - 1.8 - 1.8 - UTF-8 - diff --git a/radar-dao/pom.xml b/radar-dao/pom.xml index 685b38e..05d8947 100644 --- a/radar-dao/pom.xml +++ b/radar-dao/pom.xml @@ -28,16 +28,9 @@ ${project.version} - - org.mybatis.generator - mybatis-generator-core - 1.3.7 - - tk.mybatis mapper - 4.0.0 @@ -46,42 +39,11 @@ org.apache.maven.plugins maven-compiler-plugin - 3.5.1 - - 1.8 - 1.8 - UTF-8 - org.mybatis.generator mybatis-generator-maven-plugin - 1.3.6 - - - ${basedir}/src/main/resources/generator/generatorConfig.xml - - true - true - - - - mysql - mysql-connector-java - 5.1.47 - - - tk.mybatis - mapper - 4.0.0 - - - com.pgmmers - radar-commons - ${project.version} - - diff --git a/radar-engine/pom.xml b/radar-engine/pom.xml index 1be15ab..92ccb41 100644 --- a/radar-engine/pom.xml +++ b/radar-engine/pom.xml @@ -27,11 +27,6 @@ spring-boot-starter-web - - org.springframework.boot - spring-boot-starter - - org.springframework.boot spring-boot-starter-test @@ -42,12 +37,6 @@ spring-boot-starter-actuator - - com.pgmmers - radar-service - ${project.version} - - com.pgmmers radar-service-impl @@ -64,44 +53,21 @@ mysql-connector-java - - tk.mybatis - mapper - - tk.mybatis mapper-spring-boot-starter - - org.springframework.data - spring-data-redis - - org.springframework.boot spring-boot-starter-data-redis - - junit - junit - - - - io.springfox - springfox-swagger2 - io.springfox springfox-swagger-ui - - com.google.guava - guava - @@ -111,14 +77,6 @@ org.springframework.boot spring-boot-maven-plugin - 2.1.7.RELEASE - - - - repackage - - - diff --git a/radar-service-impl/pom.xml b/radar-service-impl/pom.xml index aa2c500..e8c6178 100644 --- a/radar-service-impl/pom.xml +++ b/radar-service-impl/pom.xml @@ -13,29 +13,14 @@ - - org.springframework - spring-context-support - - org.mongodb mongo-java-driver - - com.alibaba - fastjson - - org.apache.httpcomponents httpclient - 4.5.9 - - - org.apache.commons - commons-lang3 com.pgmmers @@ -57,10 +42,6 @@ ip2region - - org.elasticsearch - elasticsearch - org.springframework spring-web diff --git a/radar-service/pom.xml b/radar-service/pom.xml index 06c42f3..7306667 100644 --- a/radar-service/pom.xml +++ b/radar-service/pom.xml @@ -19,10 +19,6 @@ - - junit - junit - com.pgmmers radar-dal From 523f7bbc975b6ec5e396d22d706fcc6122e0ecd0 Mon Sep 17 00:00:00 2001 From: "feihu.wang" Date: Wed, 20 Nov 2019 11:54:26 +0800 Subject: [PATCH 5/7] =?UTF-8?q?prefs:=20engine=20=E7=AB=AF=E5=A2=9E?= =?UTF-8?q?=E5=8A=A0=E7=B3=BB=E7=BB=9F=E5=90=AF=E5=8A=A8=E9=BB=98=E8=AE=A4?= =?UTF-8?q?=E9=A1=B5=EF=BC=8C=E5=A2=9E=E5=8A=A0=E5=8F=8B=E5=A5=BD=E6=8F=90?= =?UTF-8?q?=E7=A4=BA=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: feihu.wang --- .../pgmmers/radar/config/SwaggerConfig.java | 2 +- .../radar/controller/IndexController.java | 22 +++++++++++++++++++ .../radar/service/common/CommonResult.java | 11 ++++++++++ 3 files changed, 34 insertions(+), 1 deletion(-) create mode 100644 radar-engine/src/main/java/com/pgmmers/radar/controller/IndexController.java diff --git a/radar-engine/src/main/java/com/pgmmers/radar/config/SwaggerConfig.java b/radar-engine/src/main/java/com/pgmmers/radar/config/SwaggerConfig.java index b53d1ab..ff76574 100644 --- a/radar-engine/src/main/java/com/pgmmers/radar/config/SwaggerConfig.java +++ b/radar-engine/src/main/java/com/pgmmers/radar/config/SwaggerConfig.java @@ -21,7 +21,7 @@ public class SwaggerConfig { .apiInfo(buildApiInf()) .select() .apis(RequestHandlerSelectors.basePackage("com.pgmmers.radar.controller")) - .paths(PathSelectors.any()) + .paths(PathSelectors.regex("/services/.*")) .build(); } diff --git a/radar-engine/src/main/java/com/pgmmers/radar/controller/IndexController.java b/radar-engine/src/main/java/com/pgmmers/radar/controller/IndexController.java new file mode 100644 index 0000000..84856fb --- /dev/null +++ b/radar-engine/src/main/java/com/pgmmers/radar/controller/IndexController.java @@ -0,0 +1,22 @@ +package com.pgmmers.radar.controller; + +import com.pgmmers.radar.service.common.CommonResult; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.RestController; + +import javax.servlet.http.HttpServletRequest; + +/** + * engine 启动首页. + * @author wangfeihu + */ +@RestController +public class IndexController { + + @GetMapping(value = {"/", ""}) + public CommonResult index(HttpServletRequest request) { + CommonResult result = new CommonResult(Boolean.TRUE, "100", "Engine is running"); + result.getData().put("swagger url:", request.getRequestURL() + "swagger-ui.html"); + return result; + } +} diff --git a/radar-service/src/main/java/com/pgmmers/radar/service/common/CommonResult.java b/radar-service/src/main/java/com/pgmmers/radar/service/common/CommonResult.java index 8952878..6e5ffa0 100644 --- a/radar-service/src/main/java/com/pgmmers/radar/service/common/CommonResult.java +++ b/radar-service/src/main/java/com/pgmmers/radar/service/common/CommonResult.java @@ -58,6 +58,17 @@ public class CommonResult implements Serializable{ this.data = data; } + + public CommonResult () { + + } + + public CommonResult(boolean success, String code, String msg) { + this.success = success; + this.code = code; + this.msg = msg; + } + @Override public String toString() { return "CommonResult [success=" + success + ", msg=" + msg + ", code=" + code + "]"; From 2398e7bb7580f542be2e627d9eb1dfa4058955b2 Mon Sep 17 00:00:00 2001 From: guor Date: Thu, 28 Nov 2019 18:08:39 +0800 Subject: [PATCH 6/7] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E6=9C=BA=E5=99=A8?= =?UTF-8?q?=E5=AD=A6=E4=B9=A0=E6=A8=A1=E5=9E=8B=E7=9A=84=E5=BC=95=E5=85=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../com.pgmmers.radar/dal/model/MoldDal.java | 7 ++ .../dal/model/impl/MoldDalImpl.java | 44 ++++++++++ .../vo/model/MoldParamVO.java | 24 +++++ .../com.pgmmers.radar/vo/model/MoldVO.java | 88 +++++++++++++++++++ .../com/pgmmers/radar/mapper/MoldMapper.java | 7 ++ .../pgmmers/radar/mapper/MoldParamMapper.java | 7 ++ .../java/com/pgmmers/radar/model/MoldPO.java | 28 ++++++ .../com/pgmmers/radar/model/MoldParamPO.java | 17 ++++ .../service/impl/dnn/TensorFlowEstimator.java | 19 ++++ .../service/impl/model/MoldServiceImpl.java | 19 ++++ .../pgmmers/radar/service/dnn/Estimator.java | 23 +++++ .../radar/service/model/MoldService.java | 7 ++ 12 files changed, 290 insertions(+) create mode 100644 radar-dal/src/main/java/com.pgmmers.radar/dal/model/MoldDal.java create mode 100644 radar-dal/src/main/java/com.pgmmers.radar/dal/model/impl/MoldDalImpl.java create mode 100644 radar-dal/src/main/java/com.pgmmers.radar/vo/model/MoldParamVO.java create mode 100644 radar-dal/src/main/java/com.pgmmers.radar/vo/model/MoldVO.java create mode 100644 radar-dao/src/main/java/com/pgmmers/radar/mapper/MoldMapper.java create mode 100644 radar-dao/src/main/java/com/pgmmers/radar/mapper/MoldParamMapper.java create mode 100644 radar-dao/src/main/java/com/pgmmers/radar/model/MoldPO.java create mode 100644 radar-dao/src/main/java/com/pgmmers/radar/model/MoldParamPO.java create mode 100644 radar-service-impl/src/main/java/com/pgmmers/radar/service/impl/dnn/TensorFlowEstimator.java create mode 100644 radar-service-impl/src/main/java/com/pgmmers/radar/service/impl/model/MoldServiceImpl.java create mode 100644 radar-service/src/main/java/com/pgmmers/radar/service/dnn/Estimator.java create mode 100644 radar-service/src/main/java/com/pgmmers/radar/service/model/MoldService.java diff --git a/radar-dal/src/main/java/com.pgmmers.radar/dal/model/MoldDal.java b/radar-dal/src/main/java/com.pgmmers.radar/dal/model/MoldDal.java new file mode 100644 index 0000000..642a973 --- /dev/null +++ b/radar-dal/src/main/java/com.pgmmers.radar/dal/model/MoldDal.java @@ -0,0 +1,7 @@ +package com.pgmmers.radar.dal.model; + +import com.pgmmers.radar.vo.model.MoldVO; + +public interface MoldDal { + MoldVO get(Long id); +} diff --git a/radar-dal/src/main/java/com.pgmmers.radar/dal/model/impl/MoldDalImpl.java b/radar-dal/src/main/java/com.pgmmers.radar/dal/model/impl/MoldDalImpl.java new file mode 100644 index 0000000..0103200 --- /dev/null +++ b/radar-dal/src/main/java/com.pgmmers.radar/dal/model/impl/MoldDalImpl.java @@ -0,0 +1,44 @@ +package com.pgmmers.radar.dal.model.impl; + +import com.pgmmers.radar.dal.model.MoldDal; +import com.pgmmers.radar.mapper.MoldMapper; +import com.pgmmers.radar.mapper.MoldParamMapper; +import com.pgmmers.radar.model.MoldPO; +import com.pgmmers.radar.model.MoldParamPO; +import com.pgmmers.radar.vo.model.MoldParamVO; +import com.pgmmers.radar.vo.model.MoldVO; +import org.springframework.beans.BeanUtils; +import org.springframework.stereotype.Service; +import tk.mybatis.mapper.entity.Example; + +import javax.annotation.Resource; +import java.util.List; +import java.util.stream.Collectors; + +@Service +public class MoldDalImpl implements MoldDal { + @Resource + private MoldMapper moldMapper; + @Resource + private MoldParamMapper moldParamMapper; + + @Override + public MoldVO get(Long id) { + MoldVO vo = new MoldVO(); + MoldPO moldPO = moldMapper.selectByPrimaryKey(id); + if (moldPO != null) { + Example example = new Example(MoldParamPO.class); + example.createCriteria().andEqualTo("moldId", id); + List moldParamList = moldParamMapper.selectByExample(example); + List list = moldParamList.stream().map(moldParamPO -> { + MoldParamVO moldParamVO = new MoldParamVO(); + BeanUtils.copyProperties(moldParamPO, moldParamVO); + return moldParamVO; + }).collect(Collectors.toList()); + BeanUtils.copyProperties(moldPO, vo); + vo.setParams(list); + return vo; + } + return null; + } +} diff --git a/radar-dal/src/main/java/com.pgmmers.radar/vo/model/MoldParamVO.java b/radar-dal/src/main/java/com.pgmmers.radar/vo/model/MoldParamVO.java new file mode 100644 index 0000000..27cf7f6 --- /dev/null +++ b/radar-dal/src/main/java/com.pgmmers.radar/vo/model/MoldParamVO.java @@ -0,0 +1,24 @@ +package com.pgmmers.radar.vo.model; + +import java.io.Serializable; + +/** + *

+ * 机器学习模型配置,目前只考虑输入层为离散值的情况,不考虑需要词嵌入和融入卷积层,其中 + * 离散值通过表达式取数从前置流程传递过来. + *

+ * + * @author guor + * @date 2019/11/28 + */ +public class MoldParamVO implements Serializable { + private Long id; + /** + * 取数表达式 + */ + private String exp; + /** + * 参数顺序,显式配置,强制有序 + */ + private int order; +} diff --git a/radar-dal/src/main/java/com.pgmmers.radar/vo/model/MoldVO.java b/radar-dal/src/main/java/com.pgmmers.radar/vo/model/MoldVO.java new file mode 100644 index 0000000..444f64e --- /dev/null +++ b/radar-dal/src/main/java/com.pgmmers.radar/vo/model/MoldVO.java @@ -0,0 +1,88 @@ +package com.pgmmers.radar.vo.model; + +import java.io.Serializable; +import java.util.Date; +import java.util.List; + +/** + *

+ * 机器学习模型配置,定义模型文件路径和参数 + *

+ * + * @author guor + * @date 2019/11/28 + */ +public class MoldVO implements Serializable { + /** + * 自增ID,主键 + */ + private Long id; + /** + * 模型名称 + */ + private String name; + /** + * 输入层参数的key + */ + private String feed; + /** + * 模型参数定义,需要注意参数顺序,改变顺序会使模型调用出错或者失效 + */ + private List params; + /** + * 模型文件路径 + */ + private String path; + /** + * 模型更新时间 + */ + private Date updateDate; + + public Long getId() { + return id; + } + + public void setId(Long id) { + this.id = id; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public String getFeed() { + return feed; + } + + public void setFeed(String feed) { + this.feed = feed; + } + + public List getParams() { + return params; + } + + public void setParams(List params) { + this.params = params; + } + + public String getPath() { + return path; + } + + public void setPath(String path) { + this.path = path; + } + + public Date getUpdateDate() { + return updateDate; + } + + public void setUpdateDate(Date updateDate) { + this.updateDate = updateDate; + } +} diff --git a/radar-dao/src/main/java/com/pgmmers/radar/mapper/MoldMapper.java b/radar-dao/src/main/java/com/pgmmers/radar/mapper/MoldMapper.java new file mode 100644 index 0000000..22930bd --- /dev/null +++ b/radar-dao/src/main/java/com/pgmmers/radar/mapper/MoldMapper.java @@ -0,0 +1,7 @@ +package com.pgmmers.radar.mapper; + +import com.pgmmers.radar.model.MoldPO; +import tk.mybatis.mapper.common.Mapper; + +public interface MoldMapper extends Mapper { +} diff --git a/radar-dao/src/main/java/com/pgmmers/radar/mapper/MoldParamMapper.java b/radar-dao/src/main/java/com/pgmmers/radar/mapper/MoldParamMapper.java new file mode 100644 index 0000000..583d663 --- /dev/null +++ b/radar-dao/src/main/java/com/pgmmers/radar/mapper/MoldParamMapper.java @@ -0,0 +1,7 @@ +package com.pgmmers.radar.mapper; + +import com.pgmmers.radar.model.MoldParamPO; +import tk.mybatis.mapper.common.Mapper; + +public interface MoldParamMapper extends Mapper { +} diff --git a/radar-dao/src/main/java/com/pgmmers/radar/model/MoldPO.java b/radar-dao/src/main/java/com/pgmmers/radar/model/MoldPO.java new file mode 100644 index 0000000..55f7974 --- /dev/null +++ b/radar-dao/src/main/java/com/pgmmers/radar/model/MoldPO.java @@ -0,0 +1,28 @@ +package com.pgmmers.radar.model; + +import javax.persistence.Table; +import java.util.Date; + +@Table(name = "engine_mold") +public class MoldPO { + /** + * 自增ID,主键 + */ + private Long id; + /** + * 模型名称 + */ + private String name; + /** + * 输入层参数的key + */ + private String feed; + /** + * 模型文件路径 + */ + private String path; + /** + * 模型更新时间 + */ + private Date updateDate; +} diff --git a/radar-dao/src/main/java/com/pgmmers/radar/model/MoldParamPO.java b/radar-dao/src/main/java/com/pgmmers/radar/model/MoldParamPO.java new file mode 100644 index 0000000..5ea76d8 --- /dev/null +++ b/radar-dao/src/main/java/com/pgmmers/radar/model/MoldParamPO.java @@ -0,0 +1,17 @@ +package com.pgmmers.radar.model; + +import javax.persistence.Table; + +@Table(name = "engine_mold_param") +public class MoldParamPO { + private Long id; + private Long moldId; + /** + * 取数表达式 + */ + private String exp; + /** + * 参数顺序,显式配置,强制有序 + */ + private int order; +} diff --git a/radar-service-impl/src/main/java/com/pgmmers/radar/service/impl/dnn/TensorFlowEstimator.java b/radar-service-impl/src/main/java/com/pgmmers/radar/service/impl/dnn/TensorFlowEstimator.java new file mode 100644 index 0000000..28a4010 --- /dev/null +++ b/radar-service-impl/src/main/java/com/pgmmers/radar/service/impl/dnn/TensorFlowEstimator.java @@ -0,0 +1,19 @@ +package com.pgmmers.radar.service.impl.dnn; + +import com.pgmmers.radar.service.dnn.Estimator; +import com.pgmmers.radar.service.model.MoldService; +import org.springframework.stereotype.Component; + +import javax.annotation.Resource; +import java.util.Map; + +@Component +public class TensorFlowEstimator implements Estimator { + @Resource + private MoldService moldService; + + @Override + public double predict(Long modelId, Map> data) { + return 0; + } +} diff --git a/radar-service-impl/src/main/java/com/pgmmers/radar/service/impl/model/MoldServiceImpl.java b/radar-service-impl/src/main/java/com/pgmmers/radar/service/impl/model/MoldServiceImpl.java new file mode 100644 index 0000000..971c68e --- /dev/null +++ b/radar-service-impl/src/main/java/com/pgmmers/radar/service/impl/model/MoldServiceImpl.java @@ -0,0 +1,19 @@ +package com.pgmmers.radar.service.impl.model; + +import com.pgmmers.radar.dal.model.MoldDal; +import com.pgmmers.radar.service.model.MoldService; +import com.pgmmers.radar.vo.model.MoldVO; +import org.springframework.stereotype.Service; + +import javax.annotation.Resource; + +@Service +public class MoldServiceImpl implements MoldService { + @Resource + private MoldDal moldDal; + + @Override + public MoldVO get(Long id) { + return moldDal.get(id); + } +} diff --git a/radar-service/src/main/java/com/pgmmers/radar/service/dnn/Estimator.java b/radar-service/src/main/java/com/pgmmers/radar/service/dnn/Estimator.java new file mode 100644 index 0000000..5fb3026 --- /dev/null +++ b/radar-service/src/main/java/com/pgmmers/radar/service/dnn/Estimator.java @@ -0,0 +1,23 @@ +package com.pgmmers.radar.service.dnn; + +import java.util.Map; + +/** + *

+ * 机器学习模型执行器接口 + *

+ *

+ * 该接口内置TensorFlow实现,目前版本只考虑输入层为离散值的情况,不考虑词嵌入和融入卷积层,其中 + * 离散值通过表达式取数从前置流程传递过来,模型的预测结果为事件评分。 + *

+ *

+ * 模型抽象:y=f(x) + *

+ * + * @author guor + * @date 2019/11/28 + */ +public interface Estimator { + + double predict(Long modelId, Map> data); +} diff --git a/radar-service/src/main/java/com/pgmmers/radar/service/model/MoldService.java b/radar-service/src/main/java/com/pgmmers/radar/service/model/MoldService.java new file mode 100644 index 0000000..e7c7884 --- /dev/null +++ b/radar-service/src/main/java/com/pgmmers/radar/service/model/MoldService.java @@ -0,0 +1,7 @@ +package com.pgmmers.radar.service.model; + +import com.pgmmers.radar.vo.model.MoldVO; + +public interface MoldService { + MoldVO get(Long id); +} From 60272a4a1c411d135b169ad30d46555a44895afa Mon Sep 17 00:00:00 2001 From: guor Date: Tue, 3 Dec 2019 21:57:08 +0800 Subject: [PATCH 7/7] =?UTF-8?q?=E5=AE=8C=E5=96=84=E6=9C=BA=E5=99=A8?= =?UTF-8?q?=E5=AD=A6=E4=B9=A0=E6=A8=A1=E5=9E=8B=E7=9A=84=E8=B0=83=E7=94=A8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pom.xml | 6 ++ .../com.pgmmers.radar/dal/model/MoldDal.java | 1 + .../dal/model/impl/MoldDalImpl.java | 30 ++++-- .../vo/model/MoldParamVO.java | 32 +++++- .../com.pgmmers.radar/vo/model/MoldVO.java | 54 +++++++--- .../java/com/pgmmers/radar/model/MoldPO.java | 75 ++++++++++++- .../com/pgmmers/radar/model/MoldParamPO.java | 40 ++++++- radar-service-impl/pom.xml | 5 + .../service/impl/dnn/EstimatorContainer.java | 39 +++++++ .../service/impl/dnn/TensorDnnEstimator.java | 100 ++++++++++++++++++ .../service/impl/dnn/TensorFlowEstimator.java | 19 ---- .../service/impl/model/MoldServiceImpl.java | 5 + .../pgmmers/radar/service/dnn/Estimator.java | 10 ++ .../radar/service/model/MoldService.java | 1 + 14 files changed, 364 insertions(+), 53 deletions(-) create mode 100644 radar-service-impl/src/main/java/com/pgmmers/radar/service/impl/dnn/EstimatorContainer.java create mode 100644 radar-service-impl/src/main/java/com/pgmmers/radar/service/impl/dnn/TensorDnnEstimator.java delete mode 100644 radar-service-impl/src/main/java/com/pgmmers/radar/service/impl/dnn/TensorFlowEstimator.java diff --git a/pom.xml b/pom.xml index 21f361b..139a7ce 100644 --- a/pom.xml +++ b/pom.xml @@ -32,6 +32,7 @@ 5.1.47 2.1.7.RELEASE 8.5.37 + 1.12.0 @@ -228,6 +229,11 @@ mybatis 3.4.6
+ + org.tensorflow + tensorflow + ${tensorflow.version} +
diff --git a/radar-dal/src/main/java/com.pgmmers.radar/dal/model/MoldDal.java b/radar-dal/src/main/java/com.pgmmers.radar/dal/model/MoldDal.java index 642a973..76ee223 100644 --- a/radar-dal/src/main/java/com.pgmmers.radar/dal/model/MoldDal.java +++ b/radar-dal/src/main/java/com.pgmmers.radar/dal/model/MoldDal.java @@ -4,4 +4,5 @@ import com.pgmmers.radar.vo.model.MoldVO; public interface MoldDal { MoldVO get(Long id); + MoldVO getByModelId(Long modelId); } diff --git a/radar-dal/src/main/java/com.pgmmers.radar/dal/model/impl/MoldDalImpl.java b/radar-dal/src/main/java/com.pgmmers.radar/dal/model/impl/MoldDalImpl.java index 0103200..f039bd8 100644 --- a/radar-dal/src/main/java/com.pgmmers.radar/dal/model/impl/MoldDalImpl.java +++ b/radar-dal/src/main/java/com.pgmmers.radar/dal/model/impl/MoldDalImpl.java @@ -24,21 +24,39 @@ public class MoldDalImpl implements MoldDal { @Override public MoldVO get(Long id) { - MoldVO vo = new MoldVO(); MoldPO moldPO = moldMapper.selectByPrimaryKey(id); + return convert(moldPO); + } + + @Override + public MoldVO getByModelId(Long modelId) { + Example example = new Example(MoldPO.class); + example.createCriteria().andEqualTo("modelId", modelId); + MoldPO moldPO = moldMapper.selectOneByExample(example); + return convert(moldPO); + } + + private MoldVO convert(MoldPO moldPO) { + MoldVO vo = null; if (moldPO != null) { + vo = new MoldVO(); + BeanUtils.copyProperties(moldPO, vo); + fitParams(vo); + } + return vo; + } + + private void fitParams(MoldVO mold) { + if (mold != null) { Example example = new Example(MoldParamPO.class); - example.createCriteria().andEqualTo("moldId", id); + example.createCriteria().andEqualTo("moldId", mold.getId()); List moldParamList = moldParamMapper.selectByExample(example); List list = moldParamList.stream().map(moldParamPO -> { MoldParamVO moldParamVO = new MoldParamVO(); BeanUtils.copyProperties(moldParamPO, moldParamVO); return moldParamVO; }).collect(Collectors.toList()); - BeanUtils.copyProperties(moldPO, vo); - vo.setParams(list); - return vo; + mold.setParams(list); } - return null; } } diff --git a/radar-dal/src/main/java/com.pgmmers.radar/vo/model/MoldParamVO.java b/radar-dal/src/main/java/com.pgmmers.radar/vo/model/MoldParamVO.java index 27cf7f6..cfb6740 100644 --- a/radar-dal/src/main/java/com.pgmmers.radar/vo/model/MoldParamVO.java +++ b/radar-dal/src/main/java/com.pgmmers.radar/vo/model/MoldParamVO.java @@ -14,11 +14,35 @@ import java.io.Serializable; public class MoldParamVO implements Serializable { private Long id; /** - * 取数表达式 + * 参数的key */ - private String exp; + private String feed; /** - * 参数顺序,显式配置,强制有序 + * 取数表达式,英文逗号分隔fields.deviceId,abstractions.log_uid_ip_1_day_qty */ - private int order; + private String expressions; + + public Long getId() { + return id; + } + + public void setId(Long id) { + this.id = id; + } + + public String getFeed() { + return feed; + } + + public void setFeed(String feed) { + this.feed = feed; + } + + public String getExpressions() { + return expressions; + } + + public void setExpressions(String expressions) { + this.expressions = expressions; + } } diff --git a/radar-dal/src/main/java/com.pgmmers.radar/vo/model/MoldVO.java b/radar-dal/src/main/java/com.pgmmers.radar/vo/model/MoldVO.java index 444f64e..d0994e9 100644 --- a/radar-dal/src/main/java/com.pgmmers.radar/vo/model/MoldVO.java +++ b/radar-dal/src/main/java/com.pgmmers.radar/vo/model/MoldVO.java @@ -21,23 +21,29 @@ public class MoldVO implements Serializable { * 模型名称 */ private String name; - /** - * 输入层参数的key - */ - private String feed; - /** - * 模型参数定义,需要注意参数顺序,改变顺序会使模型调用出错或者失效 - */ - private List params; /** * 模型文件路径 */ private String path; + /** + * tensorflow框架保存模型时设置的tag,非tensorflow模型此字段为空 + */ + private String tag; + /** + * 参数列表 + */ + private List params; + /** + * 模型输出操作名称,predict_Y = tf.nn.softmax(softmax_before, name='predict') + */ + private String operation; /** * 模型更新时间 */ private Date updateDate; + private String type; + public Long getId() { return id; } @@ -54,14 +60,6 @@ public class MoldVO implements Serializable { this.name = name; } - public String getFeed() { - return feed; - } - - public void setFeed(String feed) { - this.feed = feed; - } - public List getParams() { return params; } @@ -85,4 +83,28 @@ public class MoldVO implements Serializable { public void setUpdateDate(Date updateDate) { this.updateDate = updateDate; } + + public String getTag() { + return tag; + } + + public void setTag(String tag) { + this.tag = tag; + } + + public String getOperation() { + return operation; + } + + public void setOperation(String operation) { + this.operation = operation; + } + + public String getType() { + return type; + } + + public void setType(String type) { + this.type = type; + } } diff --git a/radar-dao/src/main/java/com/pgmmers/radar/model/MoldPO.java b/radar-dao/src/main/java/com/pgmmers/radar/model/MoldPO.java index 55f7974..4a6800d 100644 --- a/radar-dao/src/main/java/com/pgmmers/radar/model/MoldPO.java +++ b/radar-dao/src/main/java/com/pgmmers/radar/model/MoldPO.java @@ -9,20 +9,87 @@ public class MoldPO { * 自增ID,主键 */ private Long id; + private Long modelId; /** * 模型名称 */ private String name; - /** - * 输入层参数的key - */ - private String feed; /** * 模型文件路径 */ private String path; + private String tag; + private String operation; /** * 模型更新时间 */ private Date updateDate; + /** + * 模型类型 + */ + private String type; + + public Long getId() { + return id; + } + + public void setId(Long id) { + this.id = id; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public String getPath() { + return path; + } + + public void setPath(String path) { + this.path = path; + } + + public Date getUpdateDate() { + return updateDate; + } + + public void setUpdateDate(Date updateDate) { + this.updateDate = updateDate; + } + + public String getType() { + return type; + } + + public void setType(String type) { + this.type = type; + } + + public Long getModelId() { + return modelId; + } + + public void setModelId(Long modelId) { + this.modelId = modelId; + } + + public String getTag() { + return tag; + } + + public void setTag(String tag) { + this.tag = tag; + } + + public String getOperation() { + return operation; + } + + public void setOperation(String operation) { + this.operation = operation; + } } diff --git a/radar-dao/src/main/java/com/pgmmers/radar/model/MoldParamPO.java b/radar-dao/src/main/java/com/pgmmers/radar/model/MoldParamPO.java index 5ea76d8..22f6701 100644 --- a/radar-dao/src/main/java/com/pgmmers/radar/model/MoldParamPO.java +++ b/radar-dao/src/main/java/com/pgmmers/radar/model/MoldParamPO.java @@ -7,11 +7,43 @@ public class MoldParamPO { private Long id; private Long moldId; /** - * 取数表达式 + * 参数的key */ - private String exp; + private String feed; /** - * 参数顺序,显式配置,强制有序 + * 取数表达式,英文逗号分隔fields.deviceId,abstractions.log_uid_ip_1_day_qty */ - private int order; + private String expressions; + + public Long getId() { + return id; + } + + public void setId(Long id) { + this.id = id; + } + + public Long getMoldId() { + return moldId; + } + + public void setMoldId(Long moldId) { + this.moldId = moldId; + } + + public String getFeed() { + return feed; + } + + public void setFeed(String feed) { + this.feed = feed; + } + + public String getExpressions() { + return expressions; + } + + public void setExpressions(String expressions) { + this.expressions = expressions; + } } diff --git a/radar-service-impl/pom.xml b/radar-service-impl/pom.xml index e8c6178..74ae025 100644 --- a/radar-service-impl/pom.xml +++ b/radar-service-impl/pom.xml @@ -47,5 +47,10 @@ spring-web compile
+ + + org.tensorflow + tensorflow +
\ No newline at end of file diff --git a/radar-service-impl/src/main/java/com/pgmmers/radar/service/impl/dnn/EstimatorContainer.java b/radar-service-impl/src/main/java/com/pgmmers/radar/service/impl/dnn/EstimatorContainer.java new file mode 100644 index 0000000..b2d990d --- /dev/null +++ b/radar-service-impl/src/main/java/com/pgmmers/radar/service/impl/dnn/EstimatorContainer.java @@ -0,0 +1,39 @@ +package com.pgmmers.radar.service.impl.dnn; + +import com.pgmmers.radar.service.dnn.Estimator; +import com.pgmmers.radar.service.model.MoldService; +import com.pgmmers.radar.vo.model.MoldVO; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; + +import javax.annotation.Resource; +import java.util.HashMap; +import java.util.Map; + +@Component +public class EstimatorContainer { + + private Map estimatorMap = new HashMap<>(); + + @Resource + private MoldService moldService; + + @Autowired + public void set(Estimator[] estimators) { + for (Estimator estimator : estimators) { + estimatorMap.put(estimator.getType(), estimator); + } + } + + public Estimator getByType(String type) { + return estimatorMap.get(type); + } + + public Estimator getByModelId(Long modelId) { + MoldVO mold = moldService.getByModelId(modelId); + if (mold == null) { + return null; + } + return getByType(mold.getType()); + } +} diff --git a/radar-service-impl/src/main/java/com/pgmmers/radar/service/impl/dnn/TensorDnnEstimator.java b/radar-service-impl/src/main/java/com/pgmmers/radar/service/impl/dnn/TensorDnnEstimator.java new file mode 100644 index 0000000..c6ee0db --- /dev/null +++ b/radar-service-impl/src/main/java/com/pgmmers/radar/service/impl/dnn/TensorDnnEstimator.java @@ -0,0 +1,100 @@ +package com.pgmmers.radar.service.impl.dnn; + +import com.pgmmers.radar.service.dnn.Estimator; +import com.pgmmers.radar.service.model.MoldService; +import com.pgmmers.radar.vo.model.MoldParamVO; +import com.pgmmers.radar.vo.model.MoldVO; +import org.apache.commons.lang3.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.stereotype.Component; +import org.tensorflow.SavedModelBundle; +import org.tensorflow.Session; +import org.tensorflow.Tensor; + +import javax.annotation.Resource; +import java.io.File; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +@Component +public class TensorDnnEstimator implements Estimator { + private static final Logger LOGGER = LoggerFactory.getLogger(TensorDnnEstimator.class); + @Resource + private MoldService moldService; + private Map modelBundleMap = new HashMap<>(); + + @Override + public double predict(Long modelId, Map> data) { + MoldVO mold = moldService.getByModelId(modelId); + if (mold == null) { + LOGGER.debug("没有找到模型配置,ModelId:{}", modelId); + return 0; + } + SavedModelBundle modelBundle = loadAndCacheModel(mold); + if (modelBundle == null) { + LOGGER.warn("模型文件不存在或加载失败,ModelId:{}", modelId); + return 0; + } + Session tfSession = modelBundle.session(); + try { + List params = mold.getParams(); + Session.Runner runner = tfSession.runner(); + for (MoldParamVO moldParam : params) { + runner.feed(moldParam.getFeed(), convert2Tensor(moldParam, data)); + } + Tensor output = runner.fetch(mold.getOperation()).run().get(0); + double[] results = new double[1]; + output.copyTo(results); + return results[0]; + } catch (Exception e) { + LOGGER.error("模型调用失败,ModelId:" + modelId, e); + } finally { + tfSession.close(); + } + return 0; + } + + private Tensor convert2Tensor(MoldParamVO moldParam, Map> data) { + String expressions = moldParam.getExpressions(); + if (StringUtils.isEmpty(expressions)) { + return Tensor.create(new double[1][1]); + } + String[] expList = expressions.split(","); + double[][] vec = new double[expList.length][1]; + int a = 0; + for (String s : expList) { + double xn = 0; + String[] ss = s.split("\\.");//fields.deviceId,abstractions.log_uid_ip_1_day_qty + Map stringMap = data.get(ss[0]); + if (stringMap != null) { + xn = (Double) stringMap.get(ss[1]); + } + vec[a++][0] = xn; + } + return Tensor.create(vec); + } + + private synchronized SavedModelBundle loadAndCacheModel(MoldVO mold) { + SavedModelBundle modelBundle = modelBundleMap.get(mold.getId()); + if (modelBundle == null) { + File file = new File(mold.getPath()); + if (file.exists() && file.isDirectory()) { + // 模型加载,比较耗时 + try { + modelBundle = SavedModelBundle.load(mold.getPath(), mold.getTag()); + modelBundleMap.put(mold.getId(), modelBundle); + } catch (Exception e) { + LOGGER.warn("模型加载失败,MoldId:{}", mold.getId()); + } + } + } + return modelBundle; + } + + @Override + public String getType() { + return Estimator.TYPE_TENSOR_DNN; + } +} diff --git a/radar-service-impl/src/main/java/com/pgmmers/radar/service/impl/dnn/TensorFlowEstimator.java b/radar-service-impl/src/main/java/com/pgmmers/radar/service/impl/dnn/TensorFlowEstimator.java deleted file mode 100644 index 28a4010..0000000 --- a/radar-service-impl/src/main/java/com/pgmmers/radar/service/impl/dnn/TensorFlowEstimator.java +++ /dev/null @@ -1,19 +0,0 @@ -package com.pgmmers.radar.service.impl.dnn; - -import com.pgmmers.radar.service.dnn.Estimator; -import com.pgmmers.radar.service.model.MoldService; -import org.springframework.stereotype.Component; - -import javax.annotation.Resource; -import java.util.Map; - -@Component -public class TensorFlowEstimator implements Estimator { - @Resource - private MoldService moldService; - - @Override - public double predict(Long modelId, Map> data) { - return 0; - } -} diff --git a/radar-service-impl/src/main/java/com/pgmmers/radar/service/impl/model/MoldServiceImpl.java b/radar-service-impl/src/main/java/com/pgmmers/radar/service/impl/model/MoldServiceImpl.java index 971c68e..a3e9ef6 100644 --- a/radar-service-impl/src/main/java/com/pgmmers/radar/service/impl/model/MoldServiceImpl.java +++ b/radar-service-impl/src/main/java/com/pgmmers/radar/service/impl/model/MoldServiceImpl.java @@ -16,4 +16,9 @@ public class MoldServiceImpl implements MoldService { public MoldVO get(Long id) { return moldDal.get(id); } + + @Override + public MoldVO getByModelId(Long modelId) { + return moldDal.getByModelId(modelId); + } } diff --git a/radar-service/src/main/java/com/pgmmers/radar/service/dnn/Estimator.java b/radar-service/src/main/java/com/pgmmers/radar/service/dnn/Estimator.java index 5fb3026..45eac4e 100644 --- a/radar-service/src/main/java/com/pgmmers/radar/service/dnn/Estimator.java +++ b/radar-service/src/main/java/com/pgmmers/radar/service/dnn/Estimator.java @@ -18,6 +18,16 @@ import java.util.Map; * @date 2019/11/28 */ public interface Estimator { + /** + * 线性回归模型 + */ + String TYPE_REGRESSION = "REGRESSION"; + /** + * 基于TensorFlow实现的神经网络模型 + */ + String TYPE_TENSOR_DNN = "TENSOR_DNN"; double predict(Long modelId, Map> data); + + String getType(); } diff --git a/radar-service/src/main/java/com/pgmmers/radar/service/model/MoldService.java b/radar-service/src/main/java/com/pgmmers/radar/service/model/MoldService.java index e7c7884..6ffaf1e 100644 --- a/radar-service/src/main/java/com/pgmmers/radar/service/model/MoldService.java +++ b/radar-service/src/main/java/com/pgmmers/radar/service/model/MoldService.java @@ -4,4 +4,5 @@ import com.pgmmers.radar.vo.model.MoldVO; public interface MoldService { MoldVO get(Long id); + MoldVO getByModelId(Long modelId); }