feats: 机器学习配置相关功能。

Signed-off-by: feihu.wang <wfh45678@163.com>
This commit is contained in:
feihu.wang
2020-01-11 11:41:19 +08:00
parent 4681b7e8ee
commit 4927dbd791
21 changed files with 545 additions and 25 deletions

View File

@@ -99,6 +99,22 @@ public class ActivationApiController {
return result;
}
@GetMapping("/absColumns/{modelId}")
public CommonResult getAbstractionColumns(@PathVariable Long modelId) {
CommonResult result = new CommonResult();
result.setSuccess(true);
List<AbstractionVO> listAbstract = abstractionService.listAbstraction(modelId);
DataColumnInfo ds = new DataColumnInfo(DataType.ABSTRACTIONS.getDesc(), DataType.ABSTRACTIONS.getName());
if (listAbstract != null) {
for (AbstractionVO abs : listAbstract) {
ds.addChildren(abs.getLabel(), abs.getName(), FieldType.DOUBLE.name());
}
}
result.getData().put("columns", ds.getChildren());
return result;
}
@GetMapping("/rulecolumns/{modelId}")
public CommonResult getRuleColumns(@PathVariable Long modelId) {
List<DataColumnInfo> list = new ArrayList<>();

View File

@@ -1,21 +1,28 @@
package com.pgmmers.radar.controller;
import com.alibaba.excel.util.IoUtils;
import com.pgmmers.radar.enums.FieldType;
import com.pgmmers.radar.enums.PluginType;
import com.pgmmers.radar.service.common.CommonResult;
import com.pgmmers.radar.util.RandomValidateCode;
import com.pgmmers.radar.util.ZipUtils;
import com.pgmmers.radar.vo.common.PluginVO;
import io.swagger.annotations.Api;
import io.swagger.annotations.ApiParam;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.util.StringUtils;
import org.springframework.web.bind.annotation.*;
import org.springframework.web.multipart.MultipartFile;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.LinkedHashMap;
@@ -28,6 +35,8 @@ public class CommonApiController {
public static Logger logger = LoggerFactory.getLogger(CommonApiController.class);
@Value("${sys.conf.workdir}")
public String workDir;
@GetMapping("/plugins")
public CommonResult plugins() {
@@ -44,10 +53,10 @@ public class CommonApiController {
@GetMapping("/fieldtypes")
public CommonResult fieldTypes() {
CommonResult result = new CommonResult();
List<HashMap<String,Object>> fields = new ArrayList<HashMap<String,Object>>();
List<HashMap<String,Object>> fields = new ArrayList<>();
for (FieldType ft : FieldType.values()) {
//fields.add(ft.name());
HashMap<String,Object> map=new LinkedHashMap<String,Object>();
HashMap<String,Object> map=new LinkedHashMap<>();
map.put("name", ft.name());
map.put("desc", ft.getDesc());
fields.add(map);
@@ -71,6 +80,26 @@ public class CommonApiController {
logger.error("get captcha error", e);
}
}
@PostMapping(value = "/upload")
public CommonResult upload(@ApiParam(value = "file") @RequestPart("file") MultipartFile file, @RequestParam(defaultValue = "") String key) {
CommonResult result = new CommonResult();
String fileName = file.getOriginalFilename();
try {
String path = workDir + "/" + fileName;
String decomposePath = path.substring(0, path.lastIndexOf("."));
FileOutputStream fos = new FileOutputStream(new File(workDir + "/" + fileName));
IoUtils.copy(file.getInputStream(), fos);
fos.flush();
fos.close();
if (!StringUtils.isEmpty(key) && key.equals("machine")) {
ZipUtils.unZipIt(path, decomposePath);
}
} catch (IOException e) {
e.printStackTrace();
} finally {
}
return result;
}
}

View File

@@ -0,0 +1,63 @@
package com.pgmmers.radar.controller;
import com.pgmmers.radar.service.common.CommonResult;
import com.pgmmers.radar.service.model.ModelConfService;
import com.pgmmers.radar.vo.model.ModelConfVO;
import io.swagger.annotations.Api;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.util.StringUtils;
import org.springframework.web.bind.annotation.*;
/**
* 机器学习配置 api.
* @author wangfeihu
*/
@RestController
@RequestMapping("/services/v1/modelConfig")
@Api(value = "ModelConfigApi", description = "模型机器学习配置相关操作", tags = {"机器学习配置API"})
public class ModelConfigApiController {
@Autowired
private ModelConfService modelConfService;
@GetMapping("/list/{modelId}")
public CommonResult list(@PathVariable Long modelId) {
CommonResult result = new CommonResult();
ModelConfVO modelConfVO = modelConfService.getByModelId(modelId);
if (modelConfVO != null) {
result.getData().put("modelConfig", modelConfVO);
}
result.setSuccess(true);
return result;
}
@PutMapping
public CommonResult save(@RequestBody ModelConfVO modelConf) {
CommonResult result = new CommonResult();
if (!validate(modelConf)) {
result.setMsg("信息输入不完整");
return result;
}
if (modelConf.getId() == -1L) {
modelConf.setId(null);
}
modelConfService.save(modelConf);
result.setSuccess(true);
return result;
}
boolean validate(ModelConfVO modelConf) {
boolean result = true;
if (StringUtils.isEmpty(modelConf.getName())
|| StringUtils.isEmpty(modelConf.getOperation())
|| StringUtils.isEmpty(modelConf.getTag())
|| StringUtils.isEmpty(modelConf.getPath())
|| StringUtils.isEmpty(modelConf.getConfParam().getFeed())
|| StringUtils.isEmpty(modelConf.getConfParam().getExpressions())
) {
result = false;
}
return result;
}
}

View File

@@ -0,0 +1,46 @@
package com.pgmmers.radar.controller;
import com.pgmmers.radar.service.common.CommonResult;
import com.pgmmers.radar.service.model.ModelConfParamService;
import com.pgmmers.radar.service.model.ModelConfService;
import com.pgmmers.radar.vo.model.ModelConfParamVO;
import com.pgmmers.radar.vo.model.ModelConfVO;
import io.swagger.annotations.Api;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.web.bind.annotation.*;
/**
* 机器学习配置 api.
* @author wangfeihu
*/
@RestController
@RequestMapping("/services/v1/modelConfigParam")
@Api(value = "ModelConfigApi", description = "模型机器学习配置相关操作", tags = {"机器学习配置API"})
public class ModelConfigParamApiController {
@Autowired
private ModelConfParamService modelParamService;
@GetMapping("/{id}")
public CommonResult get(@PathVariable Long id) {
CommonResult result = new CommonResult();
ModelConfParamVO paramVO = modelParamService.get(id);
result.getData().put("param", paramVO);
result.setSuccess(true);
return result;
}
@PutMapping
public CommonResult save(@RequestBody ModelConfParamVO modelConf) {
CommonResult result = new CommonResult();
if (modelConf.getId() == -1L) {
modelConf.setId(null);
}
modelParamService.save(modelConf);
result.setSuccess(true);
return result;
}
}

View File

@@ -50,10 +50,10 @@ mongodb:
url: mongodb://localhost:27017/radar
mobile:
info:
path: D:/soft/moble_info.csv #手机号码归属地文件
path: D:/radar/moble_info.csv #手机号码归属地文件
ip2region:
db:
path: D:/soft/ip2region.db #IP地址库文件
path: D:/radar/ip2region.db #IP地址库文件
elasticsearch:
ip: localhost
port: 9300
@@ -71,4 +71,7 @@ sys:
conf:
app: admin # admin 或者 engine 根据启动的项目名称进行选择
entity-duplicate-insert: false # 事件是否允许重复插入
mongo-restore-days: 93 # 事件保存时间默认3个月
mongo-restore-days: 93 # 事件保存时间默认3个月
workdir: d:\\radar # 工作目录
server:
port: 8080

View File

@@ -0,0 +1,145 @@
package com.pgmmers.radar.util;
import java.io.*;
import java.util.Enumeration;
import java.util.zip.ZipEntry;
import java.util.zip.ZipException;
import java.util.zip.ZipFile;
import java.util.zip.ZipInputStream;
public class ZipUtils {
public static void main(String[] args) {
String zipFilePath = "/Users/xxx/Documents/temp/v6.zip";
String outputFolder = "/Users/xxx/Documents/temp/res/";
System.out.println(new File(zipFilePath).exists());
unZipIt(zipFilePath, outputFolder);
//测试2
try {
upZipFile(new File(zipFilePath), outputFolder);
} catch (IOException e) {
e.printStackTrace();
}
}
/**
* 解压文件
*
* @param zipFilePath 解压文件路径
* @param outputFolder 输出解压文件路径
*/
public static void unZipIt(String zipFilePath, String outputFolder) {
byte[] buffer = new byte[1024];
File folder = new File(outputFolder);
if (!folder.exists()) {
folder.mkdir();
}
try {
//get the zip file content
ZipInputStream zis = new ZipInputStream(new FileInputStream(zipFilePath));
ZipEntry ze = zis.getNextEntry();
while (ze != null) {
String fileName = ze.getName();
File newFile = new File(outputFolder + File.separator + fileName);
System.out.println("file unzip : " + newFile.getAbsoluteFile());
//create all non exists folders
//else you will hit FileNotFoundException for compressed folder
//大部分网络上的源码,这里没有判断子目录
if (ze.isDirectory()) {
newFile.mkdirs();
} else {
new File(newFile.getParent()).mkdirs();
FileOutputStream fos = new FileOutputStream(newFile);
int len;
while ((len = zis.read(buffer)) != -1) {
fos.write(buffer, 0, len);
}
fos.close();
}
ze = zis.getNextEntry();
}
zis.closeEntry();
zis.close();
System.out.println("Done");
} catch (IOException e) {
e.printStackTrace();
}
}
public static void unzip(File source, String out) throws IOException {
try (ZipInputStream zis = new ZipInputStream(new FileInputStream(source))) {
ZipEntry entry = zis.getNextEntry();
while (entry != null) {
File file = new File(out, entry.getName());
if (entry.isDirectory()) {
file.mkdirs();
} else {
File parent = file.getParentFile();
if (!parent.exists()) {
parent.mkdirs();
}
try (BufferedOutputStream bos = new BufferedOutputStream(new FileOutputStream(file))) {
byte[] buffer = new byte[Math.toIntExact(entry.getSize())];
int location;
while ((location = zis.read(buffer)) != -1) {
bos.write(buffer, 0, location);
}
}
}
entry = zis.getNextEntry();
}
}
}
/**
* 把所有文件都直接解压到指定目录(忽略子文件夹)
*
* @param zipFile
* @param folderPath
* @throws ZipException
* @throws IOException
*/
public static void upZipFile(File zipFile, String folderPath) throws ZipException, IOException {
File desDir = new File(folderPath);
if (!desDir.exists()) {
desDir.mkdirs();
}
ZipFile zf = new ZipFile(zipFile);
for (Enumeration<?> entries = zf.entries(); entries.hasMoreElements(); ) {
ZipEntry entry = ((ZipEntry) entries.nextElement());
InputStream in = zf.getInputStream(entry);
String str = folderPath;
// str = new String(str.getBytes("8859_1"), "GB2312");
File desFile = new File(str, java.net.URLEncoder.encode(entry.getName(), "UTF-8"));
if (!desFile.exists()) {
File fileParentDir = desFile.getParentFile();
if (!fileParentDir.exists()) {
fileParentDir.mkdirs();
}
}
OutputStream out = new FileOutputStream(desFile);
byte buffer[] = new byte[1024 * 1024];
int realLength = in.read(buffer);
while (realLength != -1) {
out.write(buffer, 0, realLength);
realLength = in.read(buffer);
}
out.close();
in.close();
}
}
}

View File

@@ -1,9 +1,16 @@
package com.pgmmers.radar.dal.model;
import com.pgmmers.radar.vo.model.ModelConfParamVO;
import com.pgmmers.radar.vo.model.ModelConfVO;
public interface ModelConfDal {
ModelConfVO get(Long id);
ModelConfVO getByModelId(Long modelId);
ModelConfVO save(ModelConfVO confVO);
ModelConfParamVO getParamById(Long id);
ModelConfParamVO saveParam(ModelConfParamVO paramVO);
}

View File

@@ -12,6 +12,7 @@ import org.springframework.stereotype.Service;
import tk.mybatis.mapper.entity.Example;
import javax.annotation.Resource;
import java.util.Date;
import java.util.List;
import java.util.stream.Collectors;
@@ -36,6 +37,49 @@ public class ModelConfDalImpl implements ModelConfDal {
return convert(modelConfPO);
}
@Override
public ModelConfVO save(ModelConfVO confVO) {
ModelConfPO conf = new ModelConfPO();
BeanUtils.copyProperties(confVO, conf);
if (conf.getId() == null) {
conf.setCreateTime(new Date());
conf.setUpdateDate(new Date());
modelConfMapper.insert(conf);
confVO.setId(conf.getId());
ModelConfParamPO paramPO = new ModelConfParamPO();
paramPO.setFeed(confVO.getConfParam().getFeed());
paramPO.setExpressions(confVO.getConfParam().getExpressions());
paramPO.setMoldId(conf.getId());
modelConfParamMapper.insert(paramPO);
confVO.getConfParam().setId(paramPO.getId());
} else {
modelConfMapper.updateByPrimaryKeySelective(conf);
}
return confVO;
}
@Override
public ModelConfParamVO getParamById(Long id) {
ModelConfParamVO modelConfParamVO = new ModelConfParamVO();
ModelConfParamPO po = modelConfParamMapper.selectByPrimaryKey(id);
BeanUtils.copyProperties(po,modelConfParamVO);
return modelConfParamVO;
}
@Override
public ModelConfParamVO saveParam(ModelConfParamVO paramVO) {
ModelConfParamPO po = new ModelConfParamPO();
BeanUtils.copyProperties(paramVO, po);
if (po.getId() == null) {
modelConfParamMapper.insert(po);
paramVO.setId(po.getId());
} else {
modelConfParamMapper.updateByPrimaryKeySelective(po);
}
return paramVO;
}
private ModelConfVO convert(ModelConfPO modelConfPO) {
ModelConfVO vo = null;
if (modelConfPO != null) {

View File

@@ -45,4 +45,13 @@ public class ModelConfParamVO implements Serializable {
public void setExpressions(String expressions) {
this.expressions = expressions;
}
@Override
public String toString() {
return "ModelConfParamVO{" +
"id=" + id +
", feed='" + feed + '\'' +
", expressions='" + expressions + '\'' +
'}';
}
}

View File

@@ -17,6 +17,8 @@ public class ModelConfVO implements Serializable {
* 自增ID主键
*/
private Long id;
private Long modelId;
/**
* 模型名称
*/
@@ -46,6 +48,8 @@ public class ModelConfVO implements Serializable {
private String comment;
private ModelConfParamVO confParam;
public Long getId() {
return id;
}
@@ -117,4 +121,36 @@ public class ModelConfVO implements Serializable {
public void setComment(String comment) {
this.comment = comment;
}
public Long getModelId() {
return modelId;
}
public void setModelId(Long modelId) {
this.modelId = modelId;
}
public ModelConfParamVO getConfParam() {
return confParam;
}
public void setConfParam(ModelConfParamVO confParam) {
this.confParam = confParam;
}
@Override
public String toString() {
return "ModelConfVO{" +
"id=" + id +
", modelId=" + modelId +
", name='" + name + '\'' +
", path='" + path + '\'' +
", tag='" + tag + '\'' +
", operation='" + operation + '\'' +
", updateDate=" + updateDate +
", type='" + type + '\'' +
", comment='" + comment + '\'' +
", confParam=" + confParam.toString() +
'}';
}
}

View File

@@ -4,4 +4,4 @@ import com.pgmmers.radar.model.ModelConfParamPO;
import tk.mybatis.mapper.common.Mapper;
public interface ModelConfParamMapper extends Mapper<ModelConfParamPO> {
}
}

View File

@@ -7,10 +7,10 @@ import javax.persistence.*;
public class ModelConfPO {
@Id
@GeneratedValue(generator = "JDBC")
private Integer id;
private Long id;
@Column(name = "model_id")
private Integer modelId;
private Long modelId;
private String name;
@@ -27,31 +27,34 @@ public class ModelConfPO {
private String comment;
@Column(name = "create_time")
private Date createTime;
/**
* @return id
*/
public Integer getId() {
public Long getId() {
return id;
}
/**
* @param id
*/
public void setId(Integer id) {
public void setId(Long id) {
this.id = id;
}
/**
* @return model_id
*/
public Integer getModelId() {
public Long getModelId() {
return modelId;
}
/**
* @param modelId
*/
public void setModelId(Integer modelId) {
public void setModelId(Long modelId) {
this.modelId = modelId;
}
@@ -152,4 +155,18 @@ public class ModelConfPO {
public void setComment(String comment) {
this.comment = comment;
}
/**
* @return create_time
*/
public Date getCreateTime() {
return createTime;
}
/**
* @param createTime
*/
public void setCreateTime(Date createTime) {
this.createTime = createTime;
}
}

View File

@@ -1,49 +1,73 @@
package com.pgmmers.radar.model;
import javax.persistence.Table;
import javax.persistence.*;
@Table(name = "engine_model_conf_param")
public class ModelConfParamPO {
@Id
@GeneratedValue(generator = "JDBC")
private Long id;
@Column(name = "mold_id")
private Long moldId;
/**
* 参数的key
*/
private String feed;
/**
* 取数表达式英文逗号分隔fields.deviceIdabstractions.log_uid_ip_1_day_qty
*/
private String expressions;
/**
* @return id
*/
public Long getId() {
return id;
}
/**
* @param id
*/
public void setId(Long id) {
this.id = id;
}
/**
* @return mold_id
*/
public Long getMoldId() {
return moldId;
}
/**
* @param moldId
*/
public void setMoldId(Long moldId) {
this.moldId = moldId;
}
/**
* @return feed
*/
public String getFeed() {
return feed;
}
/**
* @param feed
*/
public void setFeed(String feed) {
this.feed = feed;
}
/**
* @return expressions
*/
public String getExpressions() {
return expressions;
}
/**
* @param expressions
*/
public void setExpressions(String expressions) {
this.expressions = expressions;
}
}
}

View File

@@ -90,5 +90,8 @@
<table tableName="ENGINE_MODEL_CONF" domainObjectName="ModelConf" selectByExampleQueryId="false">
<generatedKey column="ID" sqlStatement="JDBC" />
</table>
<table tableName="ENGINE_MODEL_CONF_PARAM" domainObjectName="ModelConfParam" selectByExampleQueryId="false">
<generatedKey column="ID" sqlStatement="JDBC" />
</table>
</context>
</generatorConfiguration>

View File

@@ -0,0 +1,19 @@
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN" "http://mybatis.org/dtd/mybatis-3-mapper.dtd">
<mapper namespace="com.pgmmers.radar.mapper.ModelConfMapper">
<resultMap id="BaseResultMap" type="com.pgmmers.radar.model.ModelConfPO">
<!--
WARNING - @mbg.generated
-->
<id column="id" jdbcType="BIGINT" property="id" />
<result column="model_id" jdbcType="BIGINT" property="modelId" />
<result column="name" jdbcType="VARCHAR" property="name" />
<result column="path" jdbcType="VARCHAR" property="path" />
<result column="tag" jdbcType="VARCHAR" property="tag" />
<result column="operation" jdbcType="VARCHAR" property="operation" />
<result column="update_date" jdbcType="TIMESTAMP" property="updateDate" />
<result column="type" jdbcType="VARCHAR" property="type" />
<result column="comment" jdbcType="VARCHAR" property="comment" />
<result column="create_time" jdbcType="TIMESTAMP" property="createTime" />
</resultMap>
</mapper>

View File

@@ -0,0 +1,13 @@
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN" "http://mybatis.org/dtd/mybatis-3-mapper.dtd">
<mapper namespace="com.pgmmers.radar.mapper.ModelConfParamMapper">
<resultMap id="BaseResultMap" type="com.pgmmers.radar.model.ModelConfParamPO">
<!--
WARNING - @mbg.generated
-->
<id column="id" jdbcType="BIGINT" property="id" />
<result column="mold_id" jdbcType="BIGINT" property="moldId" />
<result column="feed" jdbcType="VARCHAR" property="feed" />
<result column="expressions" jdbcType="VARCHAR" property="expressions" />
</resultMap>
</mapper>

View File

@@ -68,6 +68,6 @@ sys:
app: engine # admin 或者 engine 根据启动的项目名称进行选择
entity-duplicate-insert: false # 事件是否允许重复插入
mongo-restore-days: 93 # 事件保存时间默认3个月
workdir: d:\\radar
workdir: d:\\radar # 工作目录
server:
port: 9090

View File

@@ -0,0 +1,27 @@
package com.pgmmers.radar.service.impl.model;
import com.pgmmers.radar.dal.model.ModelConfDal;
import com.pgmmers.radar.service.model.ModelConfParamService;
import com.pgmmers.radar.vo.model.ModelConfParamVO;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
@Service
public class ModelConfParamServiceImpl implements ModelConfParamService {
@Autowired
private ModelConfDal modelConfDal;
@Override
public ModelConfParamVO get(Long id) {
ModelConfParamVO paramVO;
paramVO = modelConfDal.getParamById(id);
return paramVO;
}
@Override
public ModelConfParamVO save(ModelConfParamVO modelConfParam) {
return modelConfDal.saveParam(modelConfParam);
}
}

View File

@@ -21,4 +21,9 @@ public class ModelConfServiceImpl implements ModelConfService {
public ModelConfVO getByModelId(Long modelId) {
return modelConfDal.getByModelId(modelId);
}
@Override
public ModelConfVO save(ModelConfVO modelConf) {
return modelConfDal.save(modelConf);
}
}

View File

@@ -0,0 +1,11 @@
package com.pgmmers.radar.service.model;
import com.pgmmers.radar.vo.model.ModelConfParamVO;
public interface ModelConfParamService {
ModelConfParamVO get(Long id);
ModelConfParamVO save(ModelConfParamVO modelConfParam);
}

View File

@@ -6,4 +6,7 @@ public interface ModelConfService {
ModelConfVO get(Long id);
ModelConfVO getByModelId(Long modelId);
ModelConfVO save(ModelConfVO modelConf);
}