From bd1dddece700e6830031079da04d7be5c40f1d2c Mon Sep 17 00:00:00 2001 From: "github@wangccc.win" Date: Mon, 9 Mar 2020 22:03:00 +0800 Subject: [PATCH] feat:use spring boot mongodb ,optimization config --- radar-admin/pom.xml | 24 +++- .../java/com/pgmmers/AdminApplication.java | 12 +- .../src/main/resources/application-dev.yml | 36 ++++++ .../src/main/resources/application.yml | 34 +---- radar-engine/pom.xml | 24 +++- .../com/pgmmers/radar/EngineApplication.java | 4 +- .../src/main/resources/application-dev.yml | 40 ++++++ .../src/main/resources/application.yml | 4 +- radar-service-impl/pom.xml | 6 +- .../impl/engine/AggregateCommandImpl.java | 2 +- .../service/impl/model/EntityServiceImpl.java | 4 +- .../service/impl/model/ModelServiceImpl.java | 117 ++++++++---------- .../radar/service/impl/util/BeanUtils.java | 34 +++++ .../radar/service/impl/util/MongodbUtil.java | 76 +++--------- 14 files changed, 249 insertions(+), 168 deletions(-) create mode 100644 radar-admin/src/main/resources/application-dev.yml create mode 100644 radar-engine/src/main/resources/application-dev.yml create mode 100644 radar-service-impl/src/main/java/com/pgmmers/radar/service/impl/util/BeanUtils.java diff --git a/radar-admin/pom.xml b/radar-admin/pom.xml index 1ddfbde..aa5b311 100644 --- a/radar-admin/pom.xml +++ b/radar-admin/pom.xml @@ -11,7 +11,29 @@ radar-admin radar-admin jar - + + + local + + local + + + true + + + + dev + + dev + + + + prod + + prod + + + diff --git a/radar-admin/src/main/java/com/pgmmers/AdminApplication.java b/radar-admin/src/main/java/com/pgmmers/AdminApplication.java index 15ac835..8e39f13 100644 --- a/radar-admin/src/main/java/com/pgmmers/AdminApplication.java +++ b/radar-admin/src/main/java/com/pgmmers/AdminApplication.java @@ -2,16 +2,16 @@ package com.pgmmers; import org.springframework.boot.SpringApplication; import org.springframework.boot.autoconfigure.SpringBootApplication; -import org.springframework.boot.autoconfigure.data.mongo.MongoDataAutoConfiguration; import org.springframework.boot.autoconfigure.elasticsearch.rest.RestClientAutoConfiguration; -import org.springframework.boot.autoconfigure.mongo.MongoAutoConfiguration; +import org.springframework.context.annotation.ComponentScan; import tk.mybatis.spring.annotation.MapperScan; -@SpringBootApplication(exclude = {MongoAutoConfiguration.class, MongoDataAutoConfiguration.class, RestClientAutoConfiguration.class}) +@SpringBootApplication(exclude = {RestClientAutoConfiguration.class}) @MapperScan("com.pgmmers.radar.mapper") -public class AdminApplication -{ - public static void main( String[] args ){ +@ComponentScan(basePackages = {"com.pgmmers.radar"}) +public class AdminApplication { + + public static void main(String[] args) { System.setProperty("es.set.netty.runtime.available.processors", "false"); SpringApplication.run(AdminApplication.class, args); } diff --git a/radar-admin/src/main/resources/application-dev.yml b/radar-admin/src/main/resources/application-dev.yml new file mode 100644 index 0000000..43b04d2 --- /dev/null +++ b/radar-admin/src/main/resources/application-dev.yml @@ -0,0 +1,36 @@ +spring: + data: + mongodb: + uri: mongodb://admin:123456@127.0.0.1:27017/radar + datasource: + url: jdbc:mysql://127.0.0.1:3306/radar?useUnicode=true&characterEncoding=UTF-8&serverTimezone=GMT%2B8&useSSL=false + username: root + password: 123456 + redis: + host: 127.0.0.1 + password: 123456 + timeout: 2000ms + database: 1 #默认有16个db(集群模式用hash槽代替),这里配置具体使用的db,默认是0 + +mobile: + info: + path: /radar/resources/moble_info.csv #手机号码归属地文件 +ip2region: + db: + path: /radar/resources/ip2region.db #IP地址库文件 +elasticsearch: + ip: 192.168.1.60 + port: 9300 + pool-size: 5 + cluster: + name: elasticsearch + url: http://127.0.0.1:9200 + +sys: + conf: + app: admin # admin 或者 engine, 根据启动的项目名称进行选择 + entity-duplicate-insert: false # 事件是否允许重复插入 + mongo-restore-days: 93 # 事件保存时间,默认3个月 + workdir: /radar/workdir # 工作目录 +server: + port: 18080 diff --git a/radar-admin/src/main/resources/application.yml b/radar-admin/src/main/resources/application.yml index 9351b52..bb139b5 100644 --- a/radar-admin/src/main/resources/application.yml +++ b/radar-admin/src/main/resources/application.yml @@ -1,8 +1,7 @@ spring: + profiles: + active: @profileActive@ datasource: - url: jdbc:mysql://localhost:3306/radar?useUnicode=true&characterEncoding=UTF-8&serverTimezone=GMT%2B8&useSSL=false - username: root - password: 456132 driver-class-name: com.mysql.jdbc.Driver type: com.alibaba.druid.pool.DruidDataSource druid: @@ -29,10 +28,6 @@ spring: cache: false prefix: classpath:/templates redis: - host: 127.0.0.1 - #password: - timeout: 2000ms - database: 0 #默认有16个db(集群模式用hash槽代替),这里配置具体使用的db,默认是0 lettuce: pool: max-active: 30 #最大连接数(使用负值表示没有限制) 默认 8 @@ -46,32 +41,13 @@ mapper: mappers: - tk.mybatis.mapper.common.Mapper not-empty: false -mongodb: - url: mongodb://localhost:27017/radar -mobile: - info: - path: D:/radar/moble_info.csv #手机号码归属地文件 -ip2region: - db: - path: D:/radar/ip2region.db #IP地址库文件 -elasticsearch: - ip: localhost - port: 9300 - pool-size: 5 - cluster: - name: elasticsearch - url: http://localhost:9200 + logging: level: root: info com.pgmmers.radar: info com.pgmmers.radar.mapper: debug org.elasticsearch: info -sys: - conf: - app: admin # admin 或者 engine, 根据启动的项目名称进行选择 - entity-duplicate-insert: false # 事件是否允许重复插入 - mongo-restore-days: 93 # 事件保存时间,默认3个月 - workdir: d:\\radar # 工作目录 + server: - port: 8080 \ No newline at end of file + port: 18080 diff --git a/radar-engine/pom.xml b/radar-engine/pom.xml index 84eb44c..7687fea 100644 --- a/radar-engine/pom.xml +++ b/radar-engine/pom.xml @@ -14,7 +14,29 @@ radar-engine - + + + local + + local + + + true + + + + dev + + dev + + + + prod + + prod + + + UTF-8 diff --git a/radar-engine/src/main/java/com/pgmmers/radar/EngineApplication.java b/radar-engine/src/main/java/com/pgmmers/radar/EngineApplication.java index ffff561..5af5ab4 100644 --- a/radar-engine/src/main/java/com/pgmmers/radar/EngineApplication.java +++ b/radar-engine/src/main/java/com/pgmmers/radar/EngineApplication.java @@ -2,12 +2,10 @@ package com.pgmmers.radar; import org.springframework.boot.SpringApplication; import org.springframework.boot.autoconfigure.SpringBootApplication; -import org.springframework.boot.autoconfigure.data.mongo.MongoDataAutoConfiguration; import org.springframework.boot.autoconfigure.elasticsearch.rest.RestClientAutoConfiguration; -import org.springframework.boot.autoconfigure.mongo.MongoAutoConfiguration; import tk.mybatis.spring.annotation.MapperScan; -@SpringBootApplication(exclude = {MongoAutoConfiguration.class, MongoDataAutoConfiguration.class, RestClientAutoConfiguration.class}) +@SpringBootApplication(exclude = {RestClientAutoConfiguration.class}) @MapperScan("com.pgmmers.radar.mapper") public class EngineApplication { diff --git a/radar-engine/src/main/resources/application-dev.yml b/radar-engine/src/main/resources/application-dev.yml new file mode 100644 index 0000000..68931e1 --- /dev/null +++ b/radar-engine/src/main/resources/application-dev.yml @@ -0,0 +1,40 @@ +spring: + data: + mongodb: + uri: mongodb://admin:123456@127.0.0.1:27017/radar + datasource: + url: jdbc:mysql://127.0.0.1:3306/radar?useUnicode=true&characterEncoding=UTF-8&serverTimezone=GMT%2B8&useSSL=false + username: root + password: 123456 + druid: + stat-view-servlet: + enabled: true + login-username: admin + login-password: 123456 + redis: + host: 127.0.0.1 + password: 123456 + timeout: 2000ms + database: 1 #默认有16个db(集群模式用hash槽代替),这里配置具体使用的db,默认是0 +mobile: + info: + path: /radar/resources/moble_info.csv # 手机号码归属地文件 +ip2region: + db: + path: /radar/resources/ip2region.db # IP地址库文件 +elasticsearch: + ip: 127.0.0.1 + port: 9300 + pool-size: 5 + cluster: + name: elasticsearch + url: http://127.0.0.1:9200 + +sys: + conf: + app: engine # admin 或者 engine, 根据启动的项目名称进行选择 + entity-duplicate-insert: false # 事件是否允许重复插入 + mongo-restore-days: 93 # 事件保存时间,默认3个月 + workdir: /radar/workdir # 工作目录 +server: + port: 9090 diff --git a/radar-engine/src/main/resources/application.yml b/radar-engine/src/main/resources/application.yml index db54636..0479730 100644 --- a/radar-engine/src/main/resources/application.yml +++ b/radar-engine/src/main/resources/application.yml @@ -1,4 +1,6 @@ spring: + profiles: + active: @profileActive@ datasource: url: jdbc:mysql://localhost:3306/radar?useUnicode=true&characterEncoding=UTF-8&serverTimezone=GMT%2B8&useSSL=false username: root @@ -70,4 +72,4 @@ sys: mongo-restore-days: 93 # 事件保存时间,默认3个月 workdir: d:\\radar # 工作目录 server: - port: 9090 \ No newline at end of file + port: 9090 diff --git a/radar-service-impl/pom.xml b/radar-service-impl/pom.xml index f62924d..fa5b080 100644 --- a/radar-service-impl/pom.xml +++ b/radar-service-impl/pom.xml @@ -14,8 +14,8 @@ - org.mongodb - mongo-java-driver + org.springframework.boot + spring-boot-starter-data-mongodb @@ -57,4 +57,4 @@ elasticsearch - \ No newline at end of file + diff --git a/radar-service-impl/src/main/java/com/pgmmers/radar/service/impl/engine/AggregateCommandImpl.java b/radar-service-impl/src/main/java/com/pgmmers/radar/service/impl/engine/AggregateCommandImpl.java index d1bd182..f425df4 100644 --- a/radar-service-impl/src/main/java/com/pgmmers/radar/service/impl/engine/AggregateCommandImpl.java +++ b/radar-service-impl/src/main/java/com/pgmmers/radar/service/impl/engine/AggregateCommandImpl.java @@ -22,7 +22,7 @@ import java.util.List; @Service public class AggregateCommandImpl implements AggregateCommand { - @Value("${mongodb.url}") + @Value("${spring.data.mongodb.uri}") private String url; @Override diff --git a/radar-service-impl/src/main/java/com/pgmmers/radar/service/impl/model/EntityServiceImpl.java b/radar-service-impl/src/main/java/com/pgmmers/radar/service/impl/model/EntityServiceImpl.java index 1eac111..8b1742e 100644 --- a/radar-service-impl/src/main/java/com/pgmmers/radar/service/impl/model/EntityServiceImpl.java +++ b/radar-service-impl/src/main/java/com/pgmmers/radar/service/impl/model/EntityServiceImpl.java @@ -23,12 +23,12 @@ public class EntityServiceImpl implements EntityService, SubscribeHandle { private Logger logger = LoggerFactory.getLogger(EntityServiceImpl.class); - @Value("${mongodb.url}") + @Value("${spring.data.mongodb.uri}") private String url; @Autowired private ModelDal modelDal; - + @Autowired private CacheService cacheService; diff --git a/radar-service-impl/src/main/java/com/pgmmers/radar/service/impl/model/ModelServiceImpl.java b/radar-service-impl/src/main/java/com/pgmmers/radar/service/impl/model/ModelServiceImpl.java index 6bec1a4..44d03c2 100644 --- a/radar-service-impl/src/main/java/com/pgmmers/radar/service/impl/model/ModelServiceImpl.java +++ b/radar-service-impl/src/main/java/com/pgmmers/radar/service/impl/model/ModelServiceImpl.java @@ -2,9 +2,6 @@ package com.pgmmers.radar.service.impl.model; import com.alibaba.fastjson.JSON; import com.alibaba.fastjson.JSONObject; - -import com.mongodb.MongoClient; -import com.mongodb.MongoClientURI; import com.mongodb.client.model.IndexModel; import com.mongodb.client.model.IndexOptions; import com.pgmmers.radar.dal.bean.ModelQuery; @@ -22,6 +19,11 @@ import com.pgmmers.radar.service.search.SearchEngineService; import com.pgmmers.radar.vo.model.FieldVO; import com.pgmmers.radar.vo.model.ModelVO; import com.pgmmers.radar.vo.model.PreItemVO; +import java.util.ArrayList; +import java.util.List; +import java.util.UUID; +import java.util.concurrent.TimeUnit; +import javax.annotation.PostConstruct; import org.apache.commons.lang3.StringUtils; import org.bson.Document; import org.slf4j.Logger; @@ -30,14 +32,9 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; -import javax.annotation.PostConstruct; -import java.util.ArrayList; -import java.util.List; -import java.util.UUID; -import java.util.concurrent.TimeUnit; - @Service public class ModelServiceImpl implements ModelService, SubscribeHandle { + public static Logger logger = LoggerFactory .getLogger(ModelServiceImpl.class); @@ -47,7 +44,7 @@ public class ModelServiceImpl implements ModelService, SubscribeHandle { @Autowired private CacheService cacheService; - @Value("${mongodb.url}") + @Value("${spring.data.mongodb.uri}") private String url; @Value("${sys.conf.mongo-restore-days}") @@ -136,11 +133,11 @@ public class ModelServiceImpl implements ModelService, SubscribeHandle { } int count = modelDal.save(model); if (count > 0) { - if(StringUtils.isEmpty(model.getModelName())){ - model.setModelName("model" + model.getId()); - modelDal.save(model); - } - + if (StringUtils.isEmpty(model.getModelName())) { + model.setModelName("model" + model.getId()); + modelDal.save(model); + } + result.getData().put("id", model.getId()); result.setSuccess(true); // 通知更新 @@ -175,15 +172,10 @@ public class ModelServiceImpl implements ModelService, SubscribeHandle { List fields = modelDal.listField(id); List items = modelDal.listPreItem(id, null); String tempUrl = url + ".entity_" + id; - MongoClientURI uri = new MongoClientURI(tempUrl); - MongoClient client = MongodbUtil.getClient(tempUrl); - client.getDatabase(uri.getDatabase()) - .getCollection(uri.getCollection()).drop(); - client.getDatabase(uri.getDatabase()).createCollection( - uri.getCollection()); + MongodbUtil.mongoTemplate.getCollection(tempUrl).drop(); + MongodbUtil.mongoTemplate.createCollection(tempUrl); List indexes = new ArrayList<>(); - if (fields == null) { result.setMsg("请先为模型配置字段"); return result; @@ -197,26 +189,26 @@ public class ModelServiceImpl implements ModelService, SubscribeHandle { indexes.add(index); } } - + Document ttlKeys = new Document(); ttlKeys.put("radar_ref_datetime", 1); IndexOptions options = new IndexOptions(); - options.expireAfter((long)eventExpireDays, TimeUnit.DAYS); + options.expireAfter((long) eventExpireDays, TimeUnit.DAYS); IndexModel ttlIndex = new IndexModel(ttlKeys, options); indexes.add(ttlIndex); - client.getDatabase(uri.getDatabase()).getCollection(uri.getCollection()).createIndexes(indexes); - - - + MongodbUtil.getCollection(tempUrl).createIndexes(indexes); +// // 重建es index JSONObject total = buildEsMappingJson(fields, items); // execute boolean isCreated; - isCreated = searchService.createIndex(modelVO.getGuid().toLowerCase(), modelVO.getModelName().toLowerCase(), "radar", total.toJSONString()); + isCreated = searchService + .createIndex(modelVO.getGuid().toLowerCase(), modelVO.getModelName().toLowerCase(), + "radar", total.toJSONString()); logger.info("index mapping:{} is create {}", total.toJSONString(), isCreated); if (isCreated) { modelVO.setStatus(StatusType.INACTIVE.getKey()); @@ -230,9 +222,6 @@ public class ModelServiceImpl implements ModelService, SubscribeHandle { /** * recreate elastic mapping - * @param fields - * @param items - * @return */ private JSONObject buildEsMappingJson(List fields, List items) { // @@ -256,7 +245,7 @@ public class ModelServiceImpl implements ModelService, SubscribeHandle { String columns = plugin.getMeta(); if (columns == null) { String fieldType = plugin.getType(); - if(fieldType.equals("JSON")) { + if (fieldType.equals("JSON")) { //TODO: json类型需要另外处理 } else { String elaType = convertFieldType2ElasticType(fieldType); @@ -295,14 +284,14 @@ public class ModelServiceImpl implements ModelService, SubscribeHandle { } @Override - public CommonResult copy(Long id,String merchantCode,String name,String label) { - ModelVO model = modelDal.getModelById(id); - model.setModelName(name); - model.setLabel(label); - model.setCode(merchantCode); - - CommonResult result = new CommonResult(); - //检查是否重复 + public CommonResult copy(Long id, String merchantCode, String name, String label) { + ModelVO model = modelDal.getModelById(id); + model.setModelName(name); + model.setLabel(label); + model.setCode(merchantCode); + + CommonResult result = new CommonResult(); + //检查是否重复 ModelQuery query = new ModelQuery(); query.setMerchantCode(model.getCode()); query.setLabel(model.getLabel()); @@ -317,41 +306,41 @@ public class ModelServiceImpl implements ModelService, SubscribeHandle { } } - + int count = modelDal.copy(model); if (count > 0) { - if(StringUtils.isEmpty(model.getModelName())){ - model.setModelName("model_" + model.getId()); - modelDal.save(model); - } - + if (StringUtils.isEmpty(model.getModelName())) { + model.setModelName("model_" + model.getId()); + modelDal.save(model); + } + result.getData().put("id", model.getId()); result.setSuccess(true); // 通知更新 cacheService.publishModel(model); } return result; - } + } - public String convertFieldType2ElasticType(String fieldType) { + public String convertFieldType2ElasticType(String fieldType) { FieldType type = Enum.valueOf(FieldType.class, fieldType); String tmp; switch (type) { - case STRING: - tmp = "keyword"; - break; - case INTEGER: - tmp = "integer"; - break; - case LONG: - tmp = "long"; - break; - case DOUBLE: - tmp = "double"; - break; - default: - tmp = "text"; - break; + case STRING: + tmp = "keyword"; + break; + case INTEGER: + tmp = "integer"; + break; + case LONG: + tmp = "long"; + break; + case DOUBLE: + tmp = "double"; + break; + default: + tmp = "text"; + break; } return tmp; } diff --git a/radar-service-impl/src/main/java/com/pgmmers/radar/service/impl/util/BeanUtils.java b/radar-service-impl/src/main/java/com/pgmmers/radar/service/impl/util/BeanUtils.java new file mode 100644 index 0000000..3682e9d --- /dev/null +++ b/radar-service-impl/src/main/java/com/pgmmers/radar/service/impl/util/BeanUtils.java @@ -0,0 +1,34 @@ +package com.pgmmers.radar.service.impl.util; + +import org.springframework.context.ApplicationContext; +import org.springframework.context.ApplicationContextAware; +import org.springframework.stereotype.Component; + +@Component +public class BeanUtils implements ApplicationContextAware { + + private static ApplicationContext applicationContext; + + @Override + public void setApplicationContext(ApplicationContext applicationContext) { + if (BeanUtils.applicationContext == null) { + BeanUtils.applicationContext = applicationContext; + } + } + + public static ApplicationContext getApplicationContext() { + return applicationContext; + } + + public static Object getBean(String name) { + return getApplicationContext().getBean(name); + } + + public static T getBean(Class clazz) { + return getApplicationContext().getBean(clazz); + } + + public static T getBean(String name, Class clazz) { + return getApplicationContext().getBean(name, clazz); + } +} diff --git a/radar-service-impl/src/main/java/com/pgmmers/radar/service/impl/util/MongodbUtil.java b/radar-service-impl/src/main/java/com/pgmmers/radar/service/impl/util/MongodbUtil.java index f2c64f4..e2173f9 100644 --- a/radar-service-impl/src/main/java/com/pgmmers/radar/service/impl/util/MongodbUtil.java +++ b/radar-service-impl/src/main/java/com/pgmmers/radar/service/impl/util/MongodbUtil.java @@ -1,70 +1,47 @@ package com.pgmmers.radar.service.impl.util; -import com.mongodb.MongoClient; import com.mongodb.MongoClientURI; import com.mongodb.client.AggregateIterable; import com.mongodb.client.FindIterable; import com.mongodb.client.MongoCollection; +import java.util.Iterator; +import java.util.List; +import java.util.Objects; import org.bson.BsonValue; import org.bson.Document; import org.bson.conversions.Bson; +import org.springframework.beans.factory.InitializingBean; +import org.springframework.data.mongodb.core.MongoTemplate; +import org.springframework.stereotype.Component; -import java.util.Iterator; -import java.util.List; +@Component +public class MongodbUtil implements InitializingBean { -public class MongodbUtil { + public static MongoTemplate mongoTemplate; - private static MongoClient client = null; - - static MongoClient initClient(String host, int port) { - if (client == null) { - client = new MongoClient(host, port); - } - return client; - } - - static MongoClient initClient(String uri) { - if (client == null) { - client = new MongoClient(new MongoClientURI(uri)); - } - return client; - } - - public static void closeClient(MongoClient client) { - client.close(); - client = null; + @Override + public void afterPropertiesSet() { + mongoTemplate = (MongoTemplate) BeanUtils.getBean("mongoTemplate"); } public static MongoCollection getCollection(String uri) { MongoClientURI clientURI = new MongoClientURI(uri); - return client.getDatabase(clientURI.getDatabase()).getCollection(clientURI.getCollection()); + return mongoTemplate.getCollection(Objects.requireNonNull(clientURI.getCollection())); } public static void insert(String url, Document doc) { - MongoClientURI uri = new MongoClientURI(url); - if (client == null) { - client = new MongoClient(uri); - } - MongoCollection collection = client.getDatabase(uri.getDatabase()).getCollection(uri.getCollection()); + MongoCollection collection = getCollection(url); collection.insertOne(doc); } public static long count(String url, Bson filter) { - MongoClientURI uri = new MongoClientURI(url); - if (client == null) { - client = new MongoClient(uri); - } - MongoCollection collection = client.getDatabase(uri.getDatabase()).getCollection(uri.getCollection()); + MongoCollection collection = getCollection(url); long count = collection.count(filter); return count; } public static long distinctCount(String url, Bson filter, String fieldName) { - MongoClientURI uri = new MongoClientURI(url); - if (client == null) { - client = new MongoClient(uri); - } - MongoCollection collection = client.getDatabase(uri.getDatabase()).getCollection(uri.getCollection()); + MongoCollection collection =getCollection(url); long count = 0; Iterator it = collection.distinct(fieldName, filter, BsonValue.class).iterator(); while (it.hasNext()) { @@ -75,30 +52,15 @@ public class MongodbUtil { } public static AggregateIterable aggregate(String url, List pipeline) { - MongoClientURI uri = new MongoClientURI(url); - if (client == null) { - client = new MongoClient(uri); - } - MongoCollection collection = client.getDatabase(uri.getDatabase()).getCollection(uri.getCollection()); + MongoCollection collection = getCollection(url); AggregateIterable it = collection.aggregate(pipeline); return it; } - + public static FindIterable find(String url, Bson filter) { - MongoClientURI uri = new MongoClientURI(url); - if (client == null) { - client = new MongoClient(uri); - } - MongoCollection collection = client.getDatabase(uri.getDatabase()).getCollection(uri.getCollection()); + MongoCollection collection = getCollection(url); FindIterable it = collection.find(filter); return it; } - - public static MongoClient getClient(String uri) { - if (client == null) { - client = new MongoClient(new MongoClientURI(uri)); - } - return client; - } }