From 2f5213076279774ebd370e19fb815e89ebe029da Mon Sep 17 00:00:00 2001 From: zhangzhiqiang Date: Fri, 10 Feb 2023 17:28:42 +0800 Subject: [PATCH] =?UTF-8?q?add:es=E6=97=A5=E5=BF=97?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- nladmin-system/nlsso-server/pom.xml | 19 +-- .../main/java/org/nl/config/DruidFilter.java | 95 +++++++++++++++ .../monitor/controller/log/LogController.java | 37 ++++++ .../org/nl/monitor/server/LogService.java | 26 ++++ .../nl/monitor/server/log/dto/LogQuery.java | 46 +++++++ .../server/log/dto/LogRepositoryDTO.java | 28 +++++ .../server/log/impl/LogServiceImpl.java | 115 ++++++++++++++++++ .../server/log/repository/LogRepository.java | 14 +++ .../META-INF/druid-filter.properties | 1 + .../main/resources/config/application-dev.yml | 16 +++ .../src/main/resources/logback-spring.xml | 46 ++++--- 11 files changed, 411 insertions(+), 32 deletions(-) create mode 100644 nladmin-system/nlsso-server/src/main/java/org/nl/config/DruidFilter.java create mode 100644 nladmin-system/nlsso-server/src/main/java/org/nl/monitor/controller/log/LogController.java create mode 100644 nladmin-system/nlsso-server/src/main/java/org/nl/monitor/server/LogService.java create mode 100644 nladmin-system/nlsso-server/src/main/java/org/nl/monitor/server/log/dto/LogQuery.java create mode 100644 nladmin-system/nlsso-server/src/main/java/org/nl/monitor/server/log/dto/LogRepositoryDTO.java create mode 100644 nladmin-system/nlsso-server/src/main/java/org/nl/monitor/server/log/impl/LogServiceImpl.java create mode 100644 nladmin-system/nlsso-server/src/main/java/org/nl/monitor/server/log/repository/LogRepository.java create mode 100644 nladmin-system/nlsso-server/src/main/resources/META-INF/druid-filter.properties diff --git a/nladmin-system/nlsso-server/pom.xml b/nladmin-system/nlsso-server/pom.xml index bde513f..acdf66d 100644 --- a/nladmin-system/nlsso-server/pom.xml +++ b/nladmin-system/nlsso-server/pom.xml @@ -33,20 +33,11 @@ + - org.elasticsearch.client - elasticsearch-rest-high-level-client - 7.6.1 - - - org.elasticsearch.client - elasticsearch-rest-client - 7.6.1 - - - org.elasticsearch - elasticsearch - 7.6.1 + org.springframework.boot + spring-boot-starter-data-elasticsearch + 2.7.5 @@ -271,7 +262,7 @@ mysql mysql-connector-java - runtime + 8.0.20 diff --git a/nladmin-system/nlsso-server/src/main/java/org/nl/config/DruidFilter.java b/nladmin-system/nlsso-server/src/main/java/org/nl/config/DruidFilter.java new file mode 100644 index 0000000..9e483c8 --- /dev/null +++ b/nladmin-system/nlsso-server/src/main/java/org/nl/config/DruidFilter.java @@ -0,0 +1,95 @@ +package org.nl.config; + +import com.alibaba.druid.filter.FilterChain; +import com.alibaba.druid.filter.FilterEventAdapter; +import com.alibaba.druid.proxy.jdbc.JdbcParameter; +import com.alibaba.druid.proxy.jdbc.PreparedStatementProxy; +import com.alibaba.druid.proxy.jdbc.ResultSetProxy; +import com.alibaba.druid.proxy.jdbc.StatementProxy; +import com.alibaba.druid.sql.SQLUtils; +import com.alibaba.druid.util.JdbcUtils; +import com.mysql.cj.jdbc.result.ResultSetImpl; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; +import org.slf4j.MDC; + +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; + +/* + * @author ZZQ + * @Date 2023/2/10 11:27 上午 + */ +@Slf4j +public class DruidFilter extends FilterEventAdapter { + + @Override + public int preparedStatement_executeUpdate(FilterChain chain, PreparedStatementProxy statement) throws SQLException { + + return super.preparedStatement_executeUpdate(chain, statement); + } + + @Override + public int statement_executeUpdate(FilterChain chain, StatementProxy statement, String sql) throws SQLException { + + return super.statement_executeUpdate(chain, statement, sql); + } + + @Override + protected void statementExecuteAfter(StatementProxy statement, String sql, boolean result) { + String traceId = MDC.get("traceId"); + int size = statement.getParametersSize(); + String executeSql = sql; + int count = 0; + try { + count=statement.getUpdateCount(); + }catch (Exception ex){ } + if (StringUtils.isNotEmpty(traceId) && count>0) { + if (size > 0) { + Collection values = statement.getParameters().values(); + List params = new ArrayList<>(); + for (JdbcParameter value : values) { + params.add(value.getValue()); + } + executeSql = SQLUtils.format(executeSql, JdbcUtils.MYSQL, params); + } + log.info("[----SQL----][update][ SQL: {} ]", executeSql); + } + super.statementExecuteAfter(statement, sql, result); + } + @Override + public ResultSetProxy statement_getResultSet(FilterChain chain, StatementProxy statement) throws SQLException { + ResultSetProxy rs = super.statement_getResultSet(chain, statement); + String executeSql = statement.getLastExecuteSql(); + String traceId = MDC.get("traceId"); + if (StringUtils.isNotEmpty(traceId)){ + int result = 0; + if (rs != null) { + ResultSetImpl rss = rs.getResultSetRaw().unwrap(ResultSetImpl.class); + result = rss.getRows().size(); + } + try { + int size = statement.getParametersSize(); + if (size>0){ + Collection values = statement.getParameters().values(); + List params = new ArrayList<>(); + for (JdbcParameter value : values) { + params.add(value.getValue()); + } + executeSql = SQLUtils.format(executeSql, JdbcUtils.MYSQL, params); + } + }catch (Exception ex){ + log.warn("[-SQL解析异常-][{}]",ex.getMessage()); + } + log.info("[----SQL----][select][执行结果:{}][ SQL: {} ]",result, executeSql); + } + return rs; + } + + + + + +} diff --git a/nladmin-system/nlsso-server/src/main/java/org/nl/monitor/controller/log/LogController.java b/nladmin-system/nlsso-server/src/main/java/org/nl/monitor/controller/log/LogController.java new file mode 100644 index 0000000..f8fc6b4 --- /dev/null +++ b/nladmin-system/nlsso-server/src/main/java/org/nl/monitor/controller/log/LogController.java @@ -0,0 +1,37 @@ +package org.nl.monitor.controller.log; + +import io.swagger.annotations.ApiOperation; +import lombok.RequiredArgsConstructor; +import org.nl.monitor.server.LogService; +import org.nl.monitor.server.log.dto.LogQuery; +import org.springframework.http.HttpStatus; +import org.springframework.http.ResponseEntity; +import org.springframework.web.bind.annotation.*; + +/** + * @author ldjun + * @version 1.0 + * @date 2023年01月29日 18:55 + * @desc desc + */ + +@RestController +@RequiredArgsConstructor +@RequestMapping("/api/esLog") +public class LogController { + private final LogService esLogService; + + + @GetMapping("/labels/{type}") + @ApiOperation("获取标签") + public ResponseEntity labelsValues(@PathVariable String type) { + return new ResponseEntity<>(esLogService.getLabelsValues(type), HttpStatus.OK); + } + + @PostMapping("/query") + @ApiOperation("日志查询") + public ResponseEntity queryAll(@RequestBody LogQuery query) { + return new ResponseEntity<>(esLogService.query(query), HttpStatus.OK); + } + +} diff --git a/nladmin-system/nlsso-server/src/main/java/org/nl/monitor/server/LogService.java b/nladmin-system/nlsso-server/src/main/java/org/nl/monitor/server/LogService.java new file mode 100644 index 0000000..5147a08 --- /dev/null +++ b/nladmin-system/nlsso-server/src/main/java/org/nl/monitor/server/LogService.java @@ -0,0 +1,26 @@ +package org.nl.monitor.server; + +import com.alibaba.fastjson.JSONArray; +import com.baomidou.mybatisplus.extension.plugins.pagination.Page; +import org.nl.monitor.server.log.dto.LogQuery; + +/** + * @author ldjun + * @version 1.0 + * @date 2023年02月07日 14:34 + * @desc desc + */ +public interface LogService { + /** + * 获取labels和values树 + * @return + */ + JSONArray getLabelsValues(String type); + + /** + * 日志查询 + * @param logQuery + * @return + */ + Page query(LogQuery logQuery); +} diff --git a/nladmin-system/nlsso-server/src/main/java/org/nl/monitor/server/log/dto/LogQuery.java b/nladmin-system/nlsso-server/src/main/java/org/nl/monitor/server/log/dto/LogQuery.java new file mode 100644 index 0000000..96b479e --- /dev/null +++ b/nladmin-system/nlsso-server/src/main/java/org/nl/monitor/server/log/dto/LogQuery.java @@ -0,0 +1,46 @@ +package org.nl.monitor.server.log.dto; + +import lombok.Data; + +import java.util.Date; + +/* + * @author ZZQ + * @Date 2023/2/8 5:18 下午 + */ +@Data +public class LogQuery { + /** + * 创建时间范围查询 + */ + private Date startTime; + private Date endTime; + /** + * 追踪id + */ + private String traceId; + /** + * 日志内容模糊匹配 + */ + private String message; + /** + * 日志级别 + */ + private String logLevel; + /** + * 系统标签 + */ + private String system; + /** + * 是否只查询Http相关请求 + */ + private Boolean isRequest = Boolean.TRUE; + /** + * 是否过滤wql日志 + */ + private Boolean filterSql = Boolean.TRUE; + + private Integer size = 20; + + private Integer page = 1; +} diff --git a/nladmin-system/nlsso-server/src/main/java/org/nl/monitor/server/log/dto/LogRepositoryDTO.java b/nladmin-system/nlsso-server/src/main/java/org/nl/monitor/server/log/dto/LogRepositoryDTO.java new file mode 100644 index 0000000..6f56781 --- /dev/null +++ b/nladmin-system/nlsso-server/src/main/java/org/nl/monitor/server/log/dto/LogRepositoryDTO.java @@ -0,0 +1,28 @@ +package org.nl.monitor.server.log.dto; + +import lombok.Data; +import org.springframework.data.annotation.Id; +import org.springframework.data.elasticsearch.annotations.Document; + +/* + * @author ZZQ + * @Date 2023/2/8 4:06 下午 + */ +@Document(indexName = "lms_log", type = "lms_log") +@Data +public class LogRepositoryDTO { + + private String message; + private String host; + private String logLevel; + private String logger; + private String requestTime; + private String requestIp; + @Id + private String id; + private String traceId; + private String requestMethod; + private String thread; + private String system; + +} diff --git a/nladmin-system/nlsso-server/src/main/java/org/nl/monitor/server/log/impl/LogServiceImpl.java b/nladmin-system/nlsso-server/src/main/java/org/nl/monitor/server/log/impl/LogServiceImpl.java new file mode 100644 index 0000000..a6b5eb8 --- /dev/null +++ b/nladmin-system/nlsso-server/src/main/java/org/nl/monitor/server/log/impl/LogServiceImpl.java @@ -0,0 +1,115 @@ +package org.nl.monitor.server.log.impl; + +import com.alibaba.fastjson.JSONArray; +import com.alibaba.fastjson.JSONObject; +import com.baomidou.mybatisplus.extension.plugins.pagination.Page; +import lombok.RequiredArgsConstructor; +import org.apache.commons.lang3.StringUtils; +import org.apache.poi.ss.formula.functions.T; +import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.index.query.BoolQueryBuilder; +import org.elasticsearch.index.query.QueryBuilders; +import org.elasticsearch.script.Script; +import org.elasticsearch.search.aggregations.AggregationBuilders; +import org.elasticsearch.search.aggregations.Aggregations; +import org.elasticsearch.search.aggregations.bucket.terms.Terms; +import org.nl.monitor.server.LogService; +import org.nl.monitor.server.log.dto.LogQuery; +import org.nl.monitor.server.log.dto.LogRepositoryDTO; +import org.nl.monitor.server.log.repository.LogRepository; +import org.springframework.data.domain.PageRequest; +import org.springframework.data.domain.Sort; +import org.springframework.data.elasticsearch.core.ElasticsearchRestTemplate; +import org.springframework.data.elasticsearch.core.aggregation.impl.AggregatedPageImpl; +import org.springframework.data.elasticsearch.core.query.FetchSourceFilter; +import org.springframework.data.elasticsearch.core.query.NativeSearchQueryBuilder; +import org.springframework.stereotype.Service; +import org.springframework.util.CollectionUtils; + +import java.util.List; + +/** + * @author ldjun + * @version 1.0 + * @date 2023年02月07日 14:35 + * @desc desc + */ +@Service +@RequiredArgsConstructor +public class LogServiceImpl implements LogService { + + private final LogRepository esLogRepository; + + private final ElasticsearchRestTemplate elasticsearchRestTemplate; + + @Override + public Page query(LogQuery logQuery){ + Page page = new Page<>(); + if (logQuery != null){ + BoolQueryBuilder query = QueryBuilders.boolQuery(); //requestMethod + extractedParam(logQuery, query); + Iterable all = esLogRepository.search(query, PageRequest.of(logQuery.getPage()-1,logQuery.getSize(), Sort.by("@timestamp").descending())); + page.setRecords(((AggregatedPageImpl) all).getContent()); + page.setTotal(((AggregatedPageImpl) all).getTotalElements()); + page.setPages(logQuery.getPage()); + page.setSize(logQuery.getSize()); + } + return page; + } + + + private void extractedParam(LogQuery logQuery, BoolQueryBuilder query) { + if (StringUtils.isNotEmpty(logQuery.getLogLevel())){ + query.must().add(QueryBuilders.matchQuery("logLevel", logQuery.getLogLevel())); + } + if (StringUtils.isNotEmpty(logQuery.getSystem())){ + query.must().add(QueryBuilders.matchQuery("system", logQuery.getSystem())); + } + if (logQuery.getIsRequest()){ + query.must().add(QueryBuilders.existsQuery("requestMethod")); + } + if (logQuery.getFilterSql()){ + query.mustNot().add(QueryBuilders.wildcardQuery("logger","org.nl.modules.wql.core.engine.*")); + } + query.mustNot().add(QueryBuilders.matchPhraseQuery("logger","org.elasticsearch.client.RestClient")); + if (StringUtils.isNotEmpty(logQuery.getTraceId())){ + query.must().add(QueryBuilders.matchQuery("traceId", logQuery.getTraceId())); + } + if (StringUtils.isNotEmpty(logQuery.getMessage())){ + query.must().add(QueryBuilders.matchQuery("message", logQuery.getMessage()).minimumShouldMatch("80%")); + } + if (logQuery.getEndTime()!=null ){ + String script = "doc['@timestamp'].value.millis < " + logQuery.getEndTime().getTime() + "L"; + query.must().add(QueryBuilders.scriptQuery(new Script(script))); + } + if (logQuery.getStartTime()!=null){ + String script = "doc['@timestamp'].value.millis > " + logQuery.getStartTime().getTime() + "L"; + query.must().add(QueryBuilders.scriptQuery(new Script(script))); + } + } + + + + @Override + public JSONArray getLabelsValues(String type) { + JSONArray result = new JSONArray(); + FetchSourceFilter fetchSourceFilter = new FetchSourceFilter(new String[]{type}, null); + NativeSearchQueryBuilder queryBuilder = new NativeSearchQueryBuilder(); + queryBuilder.withCollapseField(type+".keyword"); + queryBuilder.withSourceFilter(fetchSourceFilter); + queryBuilder.addAggregation(AggregationBuilders.terms(type).field(type+".keyword").size(100)); + Aggregations agg = elasticsearchRestTemplate.query(queryBuilder.build(), SearchResponse::getAggregations); + Terms terms = agg.get(type); + List buckets = terms.getBuckets(); + if (!CollectionUtils.isEmpty(buckets)){ + buckets.stream().map(Terms.Bucket::getKeyAsString).forEach(v-> { + JSONObject item = new JSONObject(); + item.put("label", v); + item.put("value", v); + result.add(item); + }); + } + return result; + } + +} diff --git a/nladmin-system/nlsso-server/src/main/java/org/nl/monitor/server/log/repository/LogRepository.java b/nladmin-system/nlsso-server/src/main/java/org/nl/monitor/server/log/repository/LogRepository.java new file mode 100644 index 0000000..dbed650 --- /dev/null +++ b/nladmin-system/nlsso-server/src/main/java/org/nl/monitor/server/log/repository/LogRepository.java @@ -0,0 +1,14 @@ +package org.nl.monitor.server.log.repository; + +import org.nl.monitor.server.log.dto.LogRepositoryDTO; +import org.springframework.data.elasticsearch.repository.ElasticsearchRepository; +import org.springframework.stereotype.Repository; + +/* + * @author ZZQ + * @Date 2023/2/8 4:11 下午 + */ +@Repository +public interface LogRepository extends ElasticsearchRepository { + +} diff --git a/nladmin-system/nlsso-server/src/main/resources/META-INF/druid-filter.properties b/nladmin-system/nlsso-server/src/main/resources/META-INF/druid-filter.properties new file mode 100644 index 0000000..9b025c1 --- /dev/null +++ b/nladmin-system/nlsso-server/src/main/resources/META-INF/druid-filter.properties @@ -0,0 +1 @@ +druid.filters.DruidFilter=org.nl.config.DruidFilter diff --git a/nladmin-system/nlsso-server/src/main/resources/config/application-dev.yml b/nladmin-system/nlsso-server/src/main/resources/config/application-dev.yml index a6d6377..aa669af 100644 --- a/nladmin-system/nlsso-server/src/main/resources/config/application-dev.yml +++ b/nladmin-system/nlsso-server/src/main/resources/config/application-dev.yml @@ -7,6 +7,22 @@ server: min-spare-threads: 50 #配置数据源 spring: + data: + elasticsearch: + repositories: + enabled: true + client: + reactive: + #endpoints: 172.31.185.110:9200,172.31.154.9:9200 #内网 + # endpoints: 47.96.133.178:8200 #外网 + endpoints: http://10.1.3.90:9200 #外网 + elasticsearch: + rest: + #uris: 172.31.185.110:9200,172.31.154.9:9200 #内网 + # uris: 47.96.133.178:8200 #外网 + uris: http://10.1.3.90:9200 #外网 +# username: elastic +# password: 123456 datasource: druid: db-type: com.alibaba.druid.pool.DruidDataSource diff --git a/nladmin-system/nlsso-server/src/main/resources/logback-spring.xml b/nladmin-system/nlsso-server/src/main/resources/logback-spring.xml index 5d4de0a..042287f 100644 --- a/nladmin-system/nlsso-server/src/main/resources/logback-spring.xml +++ b/nladmin-system/nlsso-server/src/main/resources/logback-spring.xml @@ -55,10 +55,10 @@ https://juejin.cn/post/6844903775631572999 - - http://47.111.78.178:27017/_bulk - logs-%date{yyyy-MM-dd} - tester + + http://47.96.133.178:8200/_bulk + lms_log + lms_log es-logger es-error-logger 30000 @@ -72,32 +72,39 @@ https://juejin.cn/post/6844903775631572999 false false 20000 - + - host - ${HOSTNAME} - false + system + lms - severity + traceId + %X{traceId} + + + logLevel %level - thread - %thread + requestMethod + %X{requestMethod} - stacktrace - %ex + requestTime + %d{yyyy-MM-dd HH:mm:ss.SSS} - logger - %logger + requestIp + %X{requestIp} - traceId - %X{traceId} + thread + %thread + + + logger + %logger @@ -113,8 +120,11 @@ https://juejin.cn/post/6844903775631572999 - + + + +