wol/radar
1
0
Fork 0
mirror of https://gitee.com/freshday/radar.git synced 2025-12-26 07:16:26 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #34 from CatCccC/develop

Browse Source 
access mongo  by spring data api
This commit is contained in:
feihu.wang 2020-03-10 17:41:56 +08:00 committed by GitHub
commit a94d89dd4a
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
14 changed files with 249 additions and 168 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
radar-admin/pom.xml
View File

@ -11,7 +11,29 @@
<artifactId>radar-admin</artifactId>
<name>radar-admin</name>
<packaging>jar</packaging>
<profiles>
<profile>
<id>local</id>
<properties>
<profileActive>local</profileActive>
</properties>
<activation>
<activeByDefault>true</activeByDefault>
</activation>
</profile>
<profile>
<id>dev</id>
<properties>
<profileActive>dev</profileActive>
</properties>
</profile>
<profile>
<id>prod</id>
<properties>
<profileActive>prod</profileActive>
</properties>
</profile>
</profiles>
<properties>
</properties>

12
radar-admin/src/main/java/com/pgmmers/AdminApplication.java
View File

@ -2,16 +2,16 @@ package com.pgmmers;
import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;
import org.springframework.boot.autoconfigure.data.mongo.MongoDataAutoConfiguration;
import org.springframework.boot.autoconfigure.elasticsearch.rest.RestClientAutoConfiguration;
import org.springframework.boot.autoconfigure.mongo.MongoAutoConfiguration;
import org.springframework.context.annotation.ComponentScan;
import tk.mybatis.spring.annotation.MapperScan;
@SpringBootApplication(exclude = {MongoAutoConfiguration.class, MongoDataAutoConfiguration.class, RestClientAutoConfiguration.class})
@SpringBootApplication(exclude = {RestClientAutoConfiguration.class})
@MapperScan("com.pgmmers.radar.mapper")
public class AdminApplication
{
public static void main( String[] args ){
@ComponentScan(basePackages = {"com.pgmmers.radar"})
public class AdminApplication {
public static void main(String[] args) {
System.setProperty("es.set.netty.runtime.available.processors", "false");
SpringApplication.run(AdminApplication.class, args);
}

36
radar-admin/src/main/resources/application-dev.yml Normal file
View File

@ -0,0 +1,36 @@
spring:
data:
mongodb:
uri: mongodb://admin:123456@127.0.0.1:27017/radar
datasource:
url: jdbc:mysql://127.0.0.1:3306/radar?useUnicode=true&characterEncoding=UTF-8&serverTimezone=GMT%2B8&useSSL=false
username: root
password: 123456
redis:
host: 127.0.0.1
password: 123456
timeout: 2000ms
database: 1 #默认有16个db(集群模式用hash槽代替)，这里配置具体使用的db，默认是0
mobile:
info:
path: /radar/resources/moble_info.csv #手机号码归属地文件
ip2region:
db:
path: /radar/resources/ip2region.db #IP地址库文件
elasticsearch:
ip: 192.168.1.60
port: 9300
pool-size: 5
cluster:
name: elasticsearch
url: http://127.0.0.1:9200
sys:
conf:
app: admin # admin 或者 engine， 根据启动的项目名称进行选择
entity-duplicate-insert: false # 事件是否允许重复插入
mongo-restore-days: 93 # 事件保存时间，默认3个月
workdir: /radar/workdir # 工作目录
server:
port: 18080

34
radar-admin/src/main/resources/application.yml
View File

@ -1,8 +1,7 @@
spring:
profiles:
active: @profileActive@
datasource:
url: jdbc:mysql://localhost:3306/radar?useUnicode=true&characterEncoding=UTF-8&serverTimezone=GMT%2B8&useSSL=false
username: root
password: 456132
driver-class-name: com.mysql.jdbc.Driver
type: com.alibaba.druid.pool.DruidDataSource
druid:
@ -29,10 +28,6 @@ spring:
cache: false
prefix: classpath:/templates
redis:
host: 127.0.0.1
#password:
timeout: 2000ms
database: 0 #默认有16个db(集群模式用hash槽代替)，这里配置具体使用的db，默认是0
lettuce:
pool:
max-active: 30 #最大连接数（使用负值表示没有限制） 默认 8
@ -46,32 +41,13 @@ mapper:
mappers:
- tk.mybatis.mapper.common.Mapper
not-empty: false
mongodb:
url: mongodb://localhost:27017/radar
mobile:
info:
path: D:/radar/moble_info.csv #手机号码归属地文件
ip2region:
db:
path: D:/radar/ip2region.db #IP地址库文件
elasticsearch:
ip: localhost
port: 9300
pool-size: 5
cluster:
name: elasticsearch
url: http://localhost:9200
logging:
level:
root: info
com.pgmmers.radar: info
com.pgmmers.radar.mapper: debug
org.elasticsearch: info
sys:
conf:
app: admin # admin 或者 engine， 根据启动的项目名称进行选择
entity-duplicate-insert: false # 事件是否允许重复插入
mongo-restore-days: 93 # 事件保存时间，默认3个月
workdir: d:\\radar # 工作目录
server:
port: 8080
port: 18080

24
radar-engine/pom.xml
View File

@ -14,7 +14,29 @@
<name>radar-engine</name>
<profiles>
<profile>
<id>local</id>
<properties>
<profileActive>local</profileActive>
</properties>
<activation>
<activeByDefault>true</activeByDefault>
</activation>
</profile>
<profile>
<id>dev</id>
<properties>
<profileActive>dev</profileActive>
</properties>
</profile>
<profile>
<id>prod</id>
<properties>
<profileActive>prod</profileActive>
</properties>
</profile>
</profiles>
<properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>

4
radar-engine/src/main/java/com/pgmmers/radar/EngineApplication.java
View File

@ -2,12 +2,10 @@ package com.pgmmers.radar;
import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;
import org.springframework.boot.autoconfigure.data.mongo.MongoDataAutoConfiguration;
import org.springframework.boot.autoconfigure.elasticsearch.rest.RestClientAutoConfiguration;
import org.springframework.boot.autoconfigure.mongo.MongoAutoConfiguration;
import tk.mybatis.spring.annotation.MapperScan;
@SpringBootApplication(exclude = {MongoAutoConfiguration.class, MongoDataAutoConfiguration.class, RestClientAutoConfiguration.class})
@SpringBootApplication(exclude = {RestClientAutoConfiguration.class})
@MapperScan("com.pgmmers.radar.mapper")
public class EngineApplication {

40
radar-engine/src/main/resources/application-dev.yml Normal file
View File

@ -0,0 +1,40 @@
spring:
data:
mongodb:
uri: mongodb://admin:123456@127.0.0.1:27017/radar
datasource:
url: jdbc:mysql://127.0.0.1:3306/radar?useUnicode=true&characterEncoding=UTF-8&serverTimezone=GMT%2B8&useSSL=false
username: root
password: 123456
druid:
stat-view-servlet:
enabled: true
login-username: admin
login-password: 123456
redis:
host: 127.0.0.1
password: 123456
timeout: 2000ms
database: 1 #默认有16个db(集群模式用hash槽代替)，这里配置具体使用的db，默认是0
mobile:
info:
path: /radar/resources/moble_info.csv # 手机号码归属地文件
ip2region:
db:
path: /radar/resources/ip2region.db # IP地址库文件
elasticsearch:
ip: 127.0.0.1
port: 9300
pool-size: 5
cluster:
name: elasticsearch
url: http://127.0.0.1:9200
sys:
conf:
app: engine # admin 或者 engine， 根据启动的项目名称进行选择
entity-duplicate-insert: false # 事件是否允许重复插入
mongo-restore-days: 93 # 事件保存时间，默认3个月
workdir: /radar/workdir # 工作目录
server:
port: 9090

4
radar-engine/src/main/resources/application.yml
View File

@ -1,4 +1,6 @@
spring:
profiles:
active: @profileActive@
datasource:
url: jdbc:mysql://localhost:3306/radar?useUnicode=true&characterEncoding=UTF-8&serverTimezone=GMT%2B8&useSSL=false
username: root
@ -70,4 +72,4 @@ sys:
mongo-restore-days: 93 # 事件保存时间，默认3个月
workdir: d:\\radar # 工作目录
server:
port: 9090
port: 9090

6
radar-service-impl/pom.xml
View File

@ -14,8 +14,8 @@
<dependencies>
<dependency>
<groupId>org.mongodb</groupId>
<artifactId>mongo-java-driver</artifactId>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-data-mongodb</artifactId>
</dependency>
<dependency>
@ -57,4 +57,4 @@
<artifactId>elasticsearch</artifactId>
</dependency>
</dependencies>
</project>
</project>

2
radar-service-impl/src/main/java/com/pgmmers/radar/service/impl/engine/AggregateCommandImpl.java
View File

@ -22,7 +22,7 @@ import java.util.List;
@Service
public class AggregateCommandImpl implements AggregateCommand {
@Value("${mongodb.url}")
@Value("${spring.data.mongodb.uri}")
private String url;
@Override

4
radar-service-impl/src/main/java/com/pgmmers/radar/service/impl/model/EntityServiceImpl.java
View File

@ -23,12 +23,12 @@ public class EntityServiceImpl implements EntityService, SubscribeHandle {
private Logger logger = LoggerFactory.getLogger(EntityServiceImpl.class);
@Value("${mongodb.url}")
@Value("${spring.data.mongodb.uri}")
private String url;
@Autowired
private ModelDal modelDal;
@Autowired
private CacheService cacheService;

117
radar-service-impl/src/main/java/com/pgmmers/radar/service/impl/model/ModelServiceImpl.java
View File

@ -2,9 +2,6 @@ package com.pgmmers.radar.service.impl.model;
import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONObject;
import com.mongodb.MongoClient;
import com.mongodb.MongoClientURI;
import com.mongodb.client.model.IndexModel;
import com.mongodb.client.model.IndexOptions;
import com.pgmmers.radar.dal.bean.ModelQuery;
@ -22,6 +19,11 @@ import com.pgmmers.radar.service.search.SearchEngineService;
import com.pgmmers.radar.vo.model.FieldVO;
import com.pgmmers.radar.vo.model.ModelVO;
import com.pgmmers.radar.vo.model.PreItemVO;
import java.util.ArrayList;
import java.util.List;
import java.util.UUID;
import java.util.concurrent.TimeUnit;
import javax.annotation.PostConstruct;
import org.apache.commons.lang3.StringUtils;
import org.bson.Document;
import org.slf4j.Logger;
@ -30,14 +32,9 @@ import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Service;
import javax.annotation.PostConstruct;
import java.util.ArrayList;
import java.util.List;
import java.util.UUID;
import java.util.concurrent.TimeUnit;
@Service
public class ModelServiceImpl implements ModelService, SubscribeHandle {
public static Logger logger = LoggerFactory
.getLogger(ModelServiceImpl.class);
@ -47,7 +44,7 @@ public class ModelServiceImpl implements ModelService, SubscribeHandle {
@Autowired
private CacheService cacheService;
@Value("${mongodb.url}")
@Value("${spring.data.mongodb.uri}")
private String url;
@Value("${sys.conf.mongo-restore-days}")
@ -136,11 +133,11 @@ public class ModelServiceImpl implements ModelService, SubscribeHandle {
}
int count = modelDal.save(model);
if (count > 0) {
if(StringUtils.isEmpty(model.getModelName())){
model.setModelName("model" + model.getId());
modelDal.save(model);
}
if (StringUtils.isEmpty(model.getModelName())) {
model.setModelName("model" + model.getId());
modelDal.save(model);
}
result.getData().put("id", model.getId());
result.setSuccess(true);
// 通知更新
@ -175,15 +172,10 @@ public class ModelServiceImpl implements ModelService, SubscribeHandle {
List<FieldVO> fields = modelDal.listField(id);
List<PreItemVO> items = modelDal.listPreItem(id, null);
String tempUrl = url + ".entity_" + id;
MongoClientURI uri = new MongoClientURI(tempUrl);
MongoClient client = MongodbUtil.getClient(tempUrl);
client.getDatabase(uri.getDatabase())
.getCollection(uri.getCollection()).drop();
client.getDatabase(uri.getDatabase()).createCollection(
uri.getCollection());
MongodbUtil.mongoTemplate.getCollection(tempUrl).drop();
MongodbUtil.mongoTemplate.createCollection(tempUrl);
List<IndexModel> indexes = new ArrayList<>();
if (fields == null) {
result.setMsg("请先为模型配置字段");
return result;
@ -197,26 +189,26 @@ public class ModelServiceImpl implements ModelService, SubscribeHandle {
indexes.add(index);
}
}
Document ttlKeys = new Document();
ttlKeys.put("radar_ref_datetime", 1);
IndexOptions options = new IndexOptions();
options.expireAfter((long)eventExpireDays, TimeUnit.DAYS);
options.expireAfter((long) eventExpireDays, TimeUnit.DAYS);
IndexModel ttlIndex = new IndexModel(ttlKeys, options);
indexes.add(ttlIndex);
client.getDatabase(uri.getDatabase()).getCollection(uri.getCollection()).createIndexes(indexes);
MongodbUtil.getCollection(tempUrl).createIndexes(indexes);
//
// 重建es index
JSONObject total = buildEsMappingJson(fields, items);
// execute
boolean isCreated;
isCreated = searchService.createIndex(modelVO.getGuid().toLowerCase(), modelVO.getModelName().toLowerCase(), "radar", total.toJSONString());
isCreated = searchService
.createIndex(modelVO.getGuid().toLowerCase(), modelVO.getModelName().toLowerCase(),
"radar", total.toJSONString());
logger.info("index mapping:{} is create {}", total.toJSONString(), isCreated);
if (isCreated) {
modelVO.setStatus(StatusType.INACTIVE.getKey());
@ -230,9 +222,6 @@ public class ModelServiceImpl implements ModelService, SubscribeHandle {
/**
* recreate elastic mapping
* @param fields
* @param items
* @return
*/
private JSONObject buildEsMappingJson(List<FieldVO> fields, List<PreItemVO> items) {
//
@ -256,7 +245,7 @@ public class ModelServiceImpl implements ModelService, SubscribeHandle {
String columns = plugin.getMeta();
if (columns == null) {
String fieldType = plugin.getType();
if(fieldType.equals("JSON")) {
if (fieldType.equals("JSON")) {
//TODO: json类型需要另外处理
} else {
String elaType = convertFieldType2ElasticType(fieldType);
@ -295,14 +284,14 @@ public class ModelServiceImpl implements ModelService, SubscribeHandle {
}
@Override
public CommonResult copy(Long id,String merchantCode,String name,String label) {
ModelVO model = modelDal.getModelById(id);
model.setModelName(name);
model.setLabel(label);
model.setCode(merchantCode);
CommonResult result = new CommonResult();
//检查是否重复
public CommonResult copy(Long id, String merchantCode, String name, String label) {
ModelVO model = modelDal.getModelById(id);
model.setModelName(name);
model.setLabel(label);
model.setCode(merchantCode);
CommonResult result = new CommonResult();
//检查是否重复
ModelQuery query = new ModelQuery();
query.setMerchantCode(model.getCode());
query.setLabel(model.getLabel());
@ -317,41 +306,41 @@ public class ModelServiceImpl implements ModelService, SubscribeHandle {
}
}
int count = modelDal.copy(model);
if (count > 0) {
if(StringUtils.isEmpty(model.getModelName())){
model.setModelName("model_" + model.getId());
modelDal.save(model);
}
if (StringUtils.isEmpty(model.getModelName())) {
model.setModelName("model_" + model.getId());
modelDal.save(model);
}
result.getData().put("id", model.getId());
result.setSuccess(true);
// 通知更新
cacheService.publishModel(model);
}
return result;
}
}
public String convertFieldType2ElasticType(String fieldType) {
public String convertFieldType2ElasticType(String fieldType) {
FieldType type = Enum.valueOf(FieldType.class, fieldType);
String tmp;
switch (type) {
case STRING:
tmp = "keyword";
break;
case INTEGER:
tmp = "integer";
break;
case LONG:
tmp = "long";
break;
case DOUBLE:
tmp = "double";
break;
default:
tmp = "text";
break;
case STRING:
tmp = "keyword";
break;
case INTEGER:
tmp = "integer";
break;
case LONG:
tmp = "long";
break;
case DOUBLE:
tmp = "double";
break;
default:
tmp = "text";
break;
}
return tmp;
}

34
radar-service-impl/src/main/java/com/pgmmers/radar/service/impl/util/BeanUtils.java Normal file
View File

@ -0,0 +1,34 @@
package com.pgmmers.radar.service.impl.util;
import org.springframework.context.ApplicationContext;
import org.springframework.context.ApplicationContextAware;
import org.springframework.stereotype.Component;
@Component
public class BeanUtils implements ApplicationContextAware {
private static ApplicationContext applicationContext;
@Override
public void setApplicationContext(ApplicationContext applicationContext) {
if (BeanUtils.applicationContext == null) {
BeanUtils.applicationContext = applicationContext;
}
}
public static ApplicationContext getApplicationContext() {
return applicationContext;
}
public static Object getBean(String name) {
return getApplicationContext().getBean(name);
}
public static <T> T getBean(Class<T> clazz) {
return getApplicationContext().getBean(clazz);
}
public static <T> T getBean(String name, Class<T> clazz) {
return getApplicationContext().getBean(name, clazz);
}
}

76
radar-service-impl/src/main/java/com/pgmmers/radar/service/impl/util/MongodbUtil.java
View File

@ -1,70 +1,47 @@
package com.pgmmers.radar.service.impl.util;
import com.mongodb.MongoClient;
import com.mongodb.MongoClientURI;
import com.mongodb.client.AggregateIterable;
import com.mongodb.client.FindIterable;
import com.mongodb.client.MongoCollection;
import java.util.Iterator;
import java.util.List;
import java.util.Objects;
import org.bson.BsonValue;
import org.bson.Document;
import org.bson.conversions.Bson;
import org.springframework.beans.factory.InitializingBean;
import org.springframework.data.mongodb.core.MongoTemplate;
import org.springframework.stereotype.Component;
import java.util.Iterator;
import java.util.List;
@Component
public class MongodbUtil implements InitializingBean {
public class MongodbUtil {
public static MongoTemplate mongoTemplate;
private static MongoClient client = null;
static MongoClient initClient(String host, int port) {
if (client == null) {
client = new MongoClient(host, port);
}
return client;
}
static MongoClient initClient(String uri) {
if (client == null) {
client = new MongoClient(new MongoClientURI(uri));
}
return client;
}
public static void closeClient(MongoClient client) {
client.close();
client = null;
@Override
public void afterPropertiesSet() {
mongoTemplate = (MongoTemplate) BeanUtils.getBean("mongoTemplate");
}
public static MongoCollection<Document> getCollection(String uri) {
MongoClientURI clientURI = new MongoClientURI(uri);
return client.getDatabase(clientURI.getDatabase()).getCollection(clientURI.getCollection());
return mongoTemplate.getCollection(Objects.requireNonNull(clientURI.getCollection()));
}
public static void insert(String url, Document doc) {
MongoClientURI uri = new MongoClientURI(url);
if (client == null) {
client = new MongoClient(uri);
}
MongoCollection<Document> collection = client.getDatabase(uri.getDatabase()).getCollection(uri.getCollection());
MongoCollection<Document> collection = getCollection(url);
collection.insertOne(doc);
}
public static long count(String url, Bson filter) {
MongoClientURI uri = new MongoClientURI(url);
if (client == null) {
client = new MongoClient(uri);
}
MongoCollection<Document> collection = client.getDatabase(uri.getDatabase()).getCollection(uri.getCollection());
MongoCollection<Document> collection = getCollection(url);
long count = collection.count(filter);
return count;
}
public static long distinctCount(String url, Bson filter, String fieldName) {
MongoClientURI uri = new MongoClientURI(url);
if (client == null) {
client = new MongoClient(uri);
}
MongoCollection<Document> collection = client.getDatabase(uri.getDatabase()).getCollection(uri.getCollection());
MongoCollection<Document> collection =getCollection(url);
long count = 0;
Iterator<BsonValue> it = collection.distinct(fieldName, filter, BsonValue.class).iterator();
while (it.hasNext()) {
@ -75,30 +52,15 @@ public class MongodbUtil {
}
public static AggregateIterable<Document> aggregate(String url, List<Bson> pipeline) {
MongoClientURI uri = new MongoClientURI(url);
if (client == null) {
client = new MongoClient(uri);
}
MongoCollection<Document> collection = client.getDatabase(uri.getDatabase()).getCollection(uri.getCollection());
MongoCollection<Document> collection = getCollection(url);
AggregateIterable<Document> it = collection.aggregate(pipeline);
return it;
}
public static FindIterable<Document> find(String url, Bson filter) {
MongoClientURI uri = new MongoClientURI(url);
if (client == null) {
client = new MongoClient(uri);
}
MongoCollection<Document> collection = client.getDatabase(uri.getDatabase()).getCollection(uri.getCollection());
MongoCollection<Document> collection = getCollection(url);
FindIterable<Document> it = collection.find(filter);
return it;
}
public static MongoClient getClient(String uri) {
if (client == null) {
client = new MongoClient(new MongoClientURI(uri));
}
return client;
}
}