From 2f5213076279774ebd370e19fb815e89ebe029da Mon Sep 17 00:00:00 2001
From: zhangzhiqiang <zhangzhiqiang@yoycol.com>
Date: Fri, 10 Feb 2023 17:28:42 +0800
Subject: [PATCH] =?UTF-8?q?add:es=E6=97=A5=E5=BF=97?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
---
nladmin-system/nlsso-server/pom.xml | 19 +--
.../main/java/org/nl/config/DruidFilter.java | 95 +++++++++++++++
.../monitor/controller/log/LogController.java | 37 ++++++
.../org/nl/monitor/server/LogService.java | 26 ++++
.../nl/monitor/server/log/dto/LogQuery.java | 46 +++++++
.../server/log/dto/LogRepositoryDTO.java | 28 +++++
.../server/log/impl/LogServiceImpl.java | 115 ++++++++++++++++++
.../server/log/repository/LogRepository.java | 14 +++
.../META-INF/druid-filter.properties | 1 +
.../main/resources/config/application-dev.yml | 16 +++
.../src/main/resources/logback-spring.xml | 46 ++++---
11 files changed, 411 insertions(+), 32 deletions(-)
create mode 100644 nladmin-system/nlsso-server/src/main/java/org/nl/config/DruidFilter.java
create mode 100644 nladmin-system/nlsso-server/src/main/java/org/nl/monitor/controller/log/LogController.java
create mode 100644 nladmin-system/nlsso-server/src/main/java/org/nl/monitor/server/LogService.java
create mode 100644 nladmin-system/nlsso-server/src/main/java/org/nl/monitor/server/log/dto/LogQuery.java
create mode 100644 nladmin-system/nlsso-server/src/main/java/org/nl/monitor/server/log/dto/LogRepositoryDTO.java
create mode 100644 nladmin-system/nlsso-server/src/main/java/org/nl/monitor/server/log/impl/LogServiceImpl.java
create mode 100644 nladmin-system/nlsso-server/src/main/java/org/nl/monitor/server/log/repository/LogRepository.java
create mode 100644 nladmin-system/nlsso-server/src/main/resources/META-INF/druid-filter.properties
diff --git a/nladmin-system/nlsso-server/pom.xml b/nladmin-system/nlsso-server/pom.xml
index bde513f..acdf66d 100644
--- a/nladmin-system/nlsso-server/pom.xml
+++ b/nladmin-system/nlsso-server/pom.xml
@@ -33,20 +33,11 @@
</properties>
<dependencies>
+ <!-- elasticsearch-->
<dependency>
- <groupId>org.elasticsearch.client</groupId>
- <artifactId>elasticsearch-rest-high-level-client</artifactId>
- <version>7.6.1</version>
- </dependency>
- <dependency>
- <groupId>org.elasticsearch.client</groupId>
- <artifactId>elasticsearch-rest-client</artifactId>
- <version>7.6.1</version>
- </dependency>
- <dependency>
- <groupId>org.elasticsearch</groupId>
- <artifactId>elasticsearch</artifactId>
- <version>7.6.1</version>
+ <groupId>org.springframework.boot</groupId>
+ <artifactId>spring-boot-starter-data-elasticsearch</artifactId>
+ <version>2.7.5</version>
</dependency>
<!-- logback appender日志-->
@@ -271,7 +262,7 @@
<dependency>
<groupId>mysql</groupId>
<artifactId>mysql-connector-java</artifactId>
- <scope>runtime</scope>
+ <version>8.0.20</version>
</dependency>
<!-- druid数据源驱动 -->
diff --git a/nladmin-system/nlsso-server/src/main/java/org/nl/config/DruidFilter.java b/nladmin-system/nlsso-server/src/main/java/org/nl/config/DruidFilter.java
new file mode 100644
index 0000000..9e483c8
--- /dev/null
+++ b/nladmin-system/nlsso-server/src/main/java/org/nl/config/DruidFilter.java
@@ -0,0 +1,95 @@
+package org.nl.config;
+
+import com.alibaba.druid.filter.FilterChain;
+import com.alibaba.druid.filter.FilterEventAdapter;
+import com.alibaba.druid.proxy.jdbc.JdbcParameter;
+import com.alibaba.druid.proxy.jdbc.PreparedStatementProxy;
+import com.alibaba.druid.proxy.jdbc.ResultSetProxy;
+import com.alibaba.druid.proxy.jdbc.StatementProxy;
+import com.alibaba.druid.sql.SQLUtils;
+import com.alibaba.druid.util.JdbcUtils;
+import com.mysql.cj.jdbc.result.ResultSetImpl;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.lang3.StringUtils;
+import org.slf4j.MDC;
+
+import java.sql.SQLException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+
+/*
+ * @author ZZQ
+ * @Date 2023/2/10 11:27 上午
+ */
+@Slf4j
+public class DruidFilter extends FilterEventAdapter {
+
+ @Override
+ public int preparedStatement_executeUpdate(FilterChain chain, PreparedStatementProxy statement) throws SQLException {
+
+ return super.preparedStatement_executeUpdate(chain, statement);
+ }
+
+ @Override
+ public int statement_executeUpdate(FilterChain chain, StatementProxy statement, String sql) throws SQLException {
+
+ return super.statement_executeUpdate(chain, statement, sql);
+ }
+
+ @Override
+ protected void statementExecuteAfter(StatementProxy statement, String sql, boolean result) {
+ String traceId = MDC.get("traceId");
+ int size = statement.getParametersSize();
+ String executeSql = sql;
+ int count = 0;
+ try {
+ count=statement.getUpdateCount();
+ }catch (Exception ex){ }
+ if (StringUtils.isNotEmpty(traceId) && count>0) {
+ if (size > 0) {
+ Collection<JdbcParameter> values = statement.getParameters().values();
+ List<Object> params = new ArrayList<>();
+ for (JdbcParameter value : values) {
+ params.add(value.getValue());
+ }
+ executeSql = SQLUtils.format(executeSql, JdbcUtils.MYSQL, params);
+ }
+ log.info("[----SQL----][update][ SQL: {} ]", executeSql);
+ }
+ super.statementExecuteAfter(statement, sql, result);
+ }
+ @Override
+ public ResultSetProxy statement_getResultSet(FilterChain chain, StatementProxy statement) throws SQLException {
+ ResultSetProxy rs = super.statement_getResultSet(chain, statement);
+ String executeSql = statement.getLastExecuteSql();
+ String traceId = MDC.get("traceId");
+ if (StringUtils.isNotEmpty(traceId)){
+ int result = 0;
+ if (rs != null) {
+ ResultSetImpl rss = rs.getResultSetRaw().unwrap(ResultSetImpl.class);
+ result = rss.getRows().size();
+ }
+ try {
+ int size = statement.getParametersSize();
+ if (size>0){
+ Collection<JdbcParameter> values = statement.getParameters().values();
+ List<Object> params = new ArrayList<>();
+ for (JdbcParameter value : values) {
+ params.add(value.getValue());
+ }
+ executeSql = SQLUtils.format(executeSql, JdbcUtils.MYSQL, params);
+ }
+ }catch (Exception ex){
+ log.warn("[-SQL解析异常-][{}]",ex.getMessage());
+ }
+ log.info("[----SQL----][select][执行结果:{}][ SQL: {} ]",result, executeSql);
+ }
+ return rs;
+ }
+
+
+
+
+
+}
diff --git a/nladmin-system/nlsso-server/src/main/java/org/nl/monitor/controller/log/LogController.java b/nladmin-system/nlsso-server/src/main/java/org/nl/monitor/controller/log/LogController.java
new file mode 100644
index 0000000..f8fc6b4
--- /dev/null
+++ b/nladmin-system/nlsso-server/src/main/java/org/nl/monitor/controller/log/LogController.java
@@ -0,0 +1,37 @@
+package org.nl.monitor.controller.log;
+
+import io.swagger.annotations.ApiOperation;
+import lombok.RequiredArgsConstructor;
+import org.nl.monitor.server.LogService;
+import org.nl.monitor.server.log.dto.LogQuery;
+import org.springframework.http.HttpStatus;
+import org.springframework.http.ResponseEntity;
+import org.springframework.web.bind.annotation.*;
+
+/**
+ * @author ldjun
+ * @version 1.0
+ * @date 2023年01月29日 18:55
+ * @desc desc
+ */
+
+@RestController
+@RequiredArgsConstructor
+@RequestMapping("/api/esLog")
+public class LogController {
+ private final LogService esLogService;
+
+
+ @GetMapping("/labels/{type}")
+ @ApiOperation("获取标签")
+ public ResponseEntity<Object> labelsValues(@PathVariable String type) {
+ return new ResponseEntity<>(esLogService.getLabelsValues(type), HttpStatus.OK);
+ }
+
+ @PostMapping("/query")
+ @ApiOperation("日志查询")
+ public ResponseEntity<Object> queryAll(@RequestBody LogQuery query) {
+ return new ResponseEntity<>(esLogService.query(query), HttpStatus.OK);
+ }
+
+}
diff --git a/nladmin-system/nlsso-server/src/main/java/org/nl/monitor/server/LogService.java b/nladmin-system/nlsso-server/src/main/java/org/nl/monitor/server/LogService.java
new file mode 100644
index 0000000..5147a08
--- /dev/null
+++ b/nladmin-system/nlsso-server/src/main/java/org/nl/monitor/server/LogService.java
@@ -0,0 +1,26 @@
+package org.nl.monitor.server;
+
+import com.alibaba.fastjson.JSONArray;
+import com.baomidou.mybatisplus.extension.plugins.pagination.Page;
+import org.nl.monitor.server.log.dto.LogQuery;
+
+/**
+ * @author ldjun
+ * @version 1.0
+ * @date 2023年02月07日 14:34
+ * @desc desc
+ */
+public interface LogService {
+ /**
+ * 获取labels和values树
+ * @return
+ */
+ JSONArray getLabelsValues(String type);
+
+ /**
+ * 日志查询
+ * @param logQuery
+ * @return
+ */
+ Page query(LogQuery logQuery);
+}
diff --git a/nladmin-system/nlsso-server/src/main/java/org/nl/monitor/server/log/dto/LogQuery.java b/nladmin-system/nlsso-server/src/main/java/org/nl/monitor/server/log/dto/LogQuery.java
new file mode 100644
index 0000000..96b479e
--- /dev/null
+++ b/nladmin-system/nlsso-server/src/main/java/org/nl/monitor/server/log/dto/LogQuery.java
@@ -0,0 +1,46 @@
+package org.nl.monitor.server.log.dto;
+
+import lombok.Data;
+
+import java.util.Date;
+
+/*
+ * @author ZZQ
+ * @Date 2023/2/8 5:18 下午
+ */
+@Data
+public class LogQuery {
+ /**
+ * 创建时间范围查询
+ */
+ private Date startTime;
+ private Date endTime;
+ /**
+ * 追踪id
+ */
+ private String traceId;
+ /**
+ * 日志内容模糊匹配
+ */
+ private String message;
+ /**
+ * 日志级别
+ */
+ private String logLevel;
+ /**
+ * 系统标签
+ */
+ private String system;
+ /**
+ * 是否只查询Http相关请求
+ */
+ private Boolean isRequest = Boolean.TRUE;
+ /**
+ * 是否过滤wql日志
+ */
+ private Boolean filterSql = Boolean.TRUE;
+
+ private Integer size = 20;
+
+ private Integer page = 1;
+}
diff --git a/nladmin-system/nlsso-server/src/main/java/org/nl/monitor/server/log/dto/LogRepositoryDTO.java b/nladmin-system/nlsso-server/src/main/java/org/nl/monitor/server/log/dto/LogRepositoryDTO.java
new file mode 100644
index 0000000..6f56781
--- /dev/null
+++ b/nladmin-system/nlsso-server/src/main/java/org/nl/monitor/server/log/dto/LogRepositoryDTO.java
@@ -0,0 +1,28 @@
+package org.nl.monitor.server.log.dto;
+
+import lombok.Data;
+import org.springframework.data.annotation.Id;
+import org.springframework.data.elasticsearch.annotations.Document;
+
+/*
+ * @author ZZQ
+ * @Date 2023/2/8 4:06 下午
+ */
+@Document(indexName = "lms_log", type = "lms_log")
+@Data
+public class LogRepositoryDTO {
+
+ private String message;
+ private String host;
+ private String logLevel;
+ private String logger;
+ private String requestTime;
+ private String requestIp;
+ @Id
+ private String id;
+ private String traceId;
+ private String requestMethod;
+ private String thread;
+ private String system;
+
+}
diff --git a/nladmin-system/nlsso-server/src/main/java/org/nl/monitor/server/log/impl/LogServiceImpl.java b/nladmin-system/nlsso-server/src/main/java/org/nl/monitor/server/log/impl/LogServiceImpl.java
new file mode 100644
index 0000000..a6b5eb8
--- /dev/null
+++ b/nladmin-system/nlsso-server/src/main/java/org/nl/monitor/server/log/impl/LogServiceImpl.java
@@ -0,0 +1,115 @@
+package org.nl.monitor.server.log.impl;
+
+import com.alibaba.fastjson.JSONArray;
+import com.alibaba.fastjson.JSONObject;
+import com.baomidou.mybatisplus.extension.plugins.pagination.Page;
+import lombok.RequiredArgsConstructor;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.poi.ss.formula.functions.T;
+import org.elasticsearch.action.search.SearchResponse;
+import org.elasticsearch.index.query.BoolQueryBuilder;
+import org.elasticsearch.index.query.QueryBuilders;
+import org.elasticsearch.script.Script;
+import org.elasticsearch.search.aggregations.AggregationBuilders;
+import org.elasticsearch.search.aggregations.Aggregations;
+import org.elasticsearch.search.aggregations.bucket.terms.Terms;
+import org.nl.monitor.server.LogService;
+import org.nl.monitor.server.log.dto.LogQuery;
+import org.nl.monitor.server.log.dto.LogRepositoryDTO;
+import org.nl.monitor.server.log.repository.LogRepository;
+import org.springframework.data.domain.PageRequest;
+import org.springframework.data.domain.Sort;
+import org.springframework.data.elasticsearch.core.ElasticsearchRestTemplate;
+import org.springframework.data.elasticsearch.core.aggregation.impl.AggregatedPageImpl;
+import org.springframework.data.elasticsearch.core.query.FetchSourceFilter;
+import org.springframework.data.elasticsearch.core.query.NativeSearchQueryBuilder;
+import org.springframework.stereotype.Service;
+import org.springframework.util.CollectionUtils;
+
+import java.util.List;
+
+/**
+ * @author ldjun
+ * @version 1.0
+ * @date 2023年02月07日 14:35
+ * @desc desc
+ */
+@Service
+@RequiredArgsConstructor
+public class LogServiceImpl implements LogService {
+
+ private final LogRepository esLogRepository;
+
+ private final ElasticsearchRestTemplate elasticsearchRestTemplate;
+
+ @Override
+ public Page query(LogQuery logQuery){
+ Page<T> page = new Page<>();
+ if (logQuery != null){
+ BoolQueryBuilder query = QueryBuilders.boolQuery(); //requestMethod
+ extractedParam(logQuery, query);
+ Iterable<LogRepositoryDTO> all = esLogRepository.search(query, PageRequest.of(logQuery.getPage()-1,logQuery.getSize(), Sort.by("@timestamp").descending()));
+ page.setRecords(((AggregatedPageImpl) all).getContent());
+ page.setTotal(((AggregatedPageImpl) all).getTotalElements());
+ page.setPages(logQuery.getPage());
+ page.setSize(logQuery.getSize());
+ }
+ return page;
+ }
+
+
+ private void extractedParam(LogQuery logQuery, BoolQueryBuilder query) {
+ if (StringUtils.isNotEmpty(logQuery.getLogLevel())){
+ query.must().add(QueryBuilders.matchQuery("logLevel", logQuery.getLogLevel()));
+ }
+ if (StringUtils.isNotEmpty(logQuery.getSystem())){
+ query.must().add(QueryBuilders.matchQuery("system", logQuery.getSystem()));
+ }
+ if (logQuery.getIsRequest()){
+ query.must().add(QueryBuilders.existsQuery("requestMethod"));
+ }
+ if (logQuery.getFilterSql()){
+ query.mustNot().add(QueryBuilders.wildcardQuery("logger","org.nl.modules.wql.core.engine.*"));
+ }
+ query.mustNot().add(QueryBuilders.matchPhraseQuery("logger","org.elasticsearch.client.RestClient"));
+ if (StringUtils.isNotEmpty(logQuery.getTraceId())){
+ query.must().add(QueryBuilders.matchQuery("traceId", logQuery.getTraceId()));
+ }
+ if (StringUtils.isNotEmpty(logQuery.getMessage())){
+ query.must().add(QueryBuilders.matchQuery("message", logQuery.getMessage()).minimumShouldMatch("80%"));
+ }
+ if (logQuery.getEndTime()!=null ){
+ String script = "doc['@timestamp'].value.millis < " + logQuery.getEndTime().getTime() + "L";
+ query.must().add(QueryBuilders.scriptQuery(new Script(script)));
+ }
+ if (logQuery.getStartTime()!=null){
+ String script = "doc['@timestamp'].value.millis > " + logQuery.getStartTime().getTime() + "L";
+ query.must().add(QueryBuilders.scriptQuery(new Script(script)));
+ }
+ }
+
+
+
+ @Override
+ public JSONArray getLabelsValues(String type) {
+ JSONArray result = new JSONArray();
+ FetchSourceFilter fetchSourceFilter = new FetchSourceFilter(new String[]{type}, null);
+ NativeSearchQueryBuilder queryBuilder = new NativeSearchQueryBuilder();
+ queryBuilder.withCollapseField(type+".keyword");
+ queryBuilder.withSourceFilter(fetchSourceFilter);
+ queryBuilder.addAggregation(AggregationBuilders.terms(type).field(type+".keyword").size(100));
+ Aggregations agg = elasticsearchRestTemplate.query(queryBuilder.build(), SearchResponse::getAggregations);
+ Terms terms = agg.get(type);
+ List<? extends Terms.Bucket> buckets = terms.getBuckets();
+ if (!CollectionUtils.isEmpty(buckets)){
+ buckets.stream().map(Terms.Bucket::getKeyAsString).forEach(v-> {
+ JSONObject item = new JSONObject();
+ item.put("label", v);
+ item.put("value", v);
+ result.add(item);
+ });
+ }
+ return result;
+ }
+
+}
diff --git a/nladmin-system/nlsso-server/src/main/java/org/nl/monitor/server/log/repository/LogRepository.java b/nladmin-system/nlsso-server/src/main/java/org/nl/monitor/server/log/repository/LogRepository.java
new file mode 100644
index 0000000..dbed650
--- /dev/null
+++ b/nladmin-system/nlsso-server/src/main/java/org/nl/monitor/server/log/repository/LogRepository.java
@@ -0,0 +1,14 @@
+package org.nl.monitor.server.log.repository;
+
+import org.nl.monitor.server.log.dto.LogRepositoryDTO;
+import org.springframework.data.elasticsearch.repository.ElasticsearchRepository;
+import org.springframework.stereotype.Repository;
+
+/*
+ * @author ZZQ
+ * @Date 2023/2/8 4:11 下午
+ */
+@Repository
+public interface LogRepository extends ElasticsearchRepository<LogRepositoryDTO, String> {
+
+}
diff --git a/nladmin-system/nlsso-server/src/main/resources/META-INF/druid-filter.properties b/nladmin-system/nlsso-server/src/main/resources/META-INF/druid-filter.properties
new file mode 100644
index 0000000..9b025c1
--- /dev/null
+++ b/nladmin-system/nlsso-server/src/main/resources/META-INF/druid-filter.properties
@@ -0,0 +1 @@
+druid.filters.DruidFilter=org.nl.config.DruidFilter
diff --git a/nladmin-system/nlsso-server/src/main/resources/config/application-dev.yml b/nladmin-system/nlsso-server/src/main/resources/config/application-dev.yml
index a6d6377..aa669af 100644
--- a/nladmin-system/nlsso-server/src/main/resources/config/application-dev.yml
+++ b/nladmin-system/nlsso-server/src/main/resources/config/application-dev.yml
@@ -7,6 +7,22 @@ server:
min-spare-threads: 50
#配置数据源
spring:
+ data:
+ elasticsearch:
+ repositories:
+ enabled: true
+ client:
+ reactive:
+ #endpoints: 172.31.185.110:9200,172.31.154.9:9200 #内网
+ # endpoints: 47.96.133.178:8200 #外网
+ endpoints: http://10.1.3.90:9200 #外网
+ elasticsearch:
+ rest:
+ #uris: 172.31.185.110:9200,172.31.154.9:9200 #内网
+ # uris: 47.96.133.178:8200 #外网
+ uris: http://10.1.3.90:9200 #外网
+# username: elastic
+# password: 123456
datasource:
druid:
db-type: com.alibaba.druid.pool.DruidDataSource
diff --git a/nladmin-system/nlsso-server/src/main/resources/logback-spring.xml b/nladmin-system/nlsso-server/src/main/resources/logback-spring.xml
index 5d4de0a..042287f 100644
--- a/nladmin-system/nlsso-server/src/main/resources/logback-spring.xml
+++ b/nladmin-system/nlsso-server/src/main/resources/logback-spring.xml
@@ -55,10 +55,10 @@ https://juejin.cn/post/6844903775631572999
<appender-ref ref="FILE"/>
</appender>
- <appender name="plumelog" class="com.internetitem.logback.elasticsearch.ElasticsearchAppender">
- <url>http://47.111.78.178:27017/_bulk</url>
- <index>logs-%date{yyyy-MM-dd}</index>
- <type>tester</type>
+ <appender name="esLogAppender" class="com.internetitem.logback.elasticsearch.ElasticsearchAppender">
+ <url>http://47.96.133.178:8200/_bulk</url>
+ <index>lms_log</index>
+ <type>lms_log</type>
<loggerName>es-logger</loggerName> <!-- optional -->
<errorLoggerName>es-error-logger</errorLoggerName> <!-- optional -->
<connectTimeout>30000</connectTimeout> <!-- optional (in ms, default 30000) -->
@@ -72,32 +72,39 @@ https://juejin.cn/post/6844903775631572999
<rawJsonMessage>false</rawJsonMessage> <!-- optional (default false) -->
<includeMdc>false</includeMdc> <!-- optional (default false) -->
<maxMessageSize>20000</maxMessageSize> <!-- optional (default -1 -->
-<!-- <authentication class="com.internetitem.logback.elasticsearch.config.BasicAuthentication" /> <!– optional –>-->
+ <!-- <authentication class="com.internetitem.logback.elasticsearch.config.BasicAuthentication" /> <!– optional –>-->
<properties>
<property>
- <name>host</name>
- <value>${HOSTNAME}</value>
- <allowEmpty>false</allowEmpty>
+ <name>system</name>
+ <value>lms</value>
</property>
<property>
- <name>severity</name>
+ <name>traceId</name>
+ <value>%X{traceId}</value>
+ </property>
+ <property>
+ <name>logLevel</name>
<value>%level</value>
</property>
<property>
- <name>thread</name>
- <value>%thread</value>
+ <name>requestMethod</name>
+ <value>%X{requestMethod}</value>
</property>
<property>
- <name>stacktrace</name>
- <value>%ex</value>
+ <name>requestTime</name>
+ <value>%d{yyyy-MM-dd HH:mm:ss.SSS}</value>
</property>
<property>
- <name>logger</name>
- <value>%logger</value>
+ <name>requestIp</name>
+ <value>%X{requestIp}</value>
</property>
<property>
- <name>traceId</name>
- <value>%X{traceId}</value>
+ <name>thread</name>
+ <value>%thread</value>
+ </property>
+ <property>
+ <name>logger</name>
+ <value>%logger</value>
</property>
</properties>
<headers>
@@ -113,8 +120,11 @@ https://juejin.cn/post/6844903775631572999
<springProfile name="dev">
<root level="debug">
<appender-ref ref="CONSOLE"/>
- <appender-ref ref="plumelog" />
+ <appender-ref ref="esLogAppender" />
</root>
+ <logger name="es-logger" level="warn" additivity="false">
+ <appender-ref ref="esLogAppender" />
+ </logger>
<!--<logger name="org.springframework" level="ERROR" additivity="false">
<appender-ref ref="CONSOLE"/>
</logger>