zhangzhiqiang
2 years ago
11 changed files with 411 additions and 32 deletions
@ -0,0 +1,95 @@ |
|||
package org.nl.config; |
|||
|
|||
import com.alibaba.druid.filter.FilterChain; |
|||
import com.alibaba.druid.filter.FilterEventAdapter; |
|||
import com.alibaba.druid.proxy.jdbc.JdbcParameter; |
|||
import com.alibaba.druid.proxy.jdbc.PreparedStatementProxy; |
|||
import com.alibaba.druid.proxy.jdbc.ResultSetProxy; |
|||
import com.alibaba.druid.proxy.jdbc.StatementProxy; |
|||
import com.alibaba.druid.sql.SQLUtils; |
|||
import com.alibaba.druid.util.JdbcUtils; |
|||
import com.mysql.cj.jdbc.result.ResultSetImpl; |
|||
import lombok.extern.slf4j.Slf4j; |
|||
import org.apache.commons.lang3.StringUtils; |
|||
import org.slf4j.MDC; |
|||
|
|||
import java.sql.SQLException; |
|||
import java.util.ArrayList; |
|||
import java.util.Collection; |
|||
import java.util.List; |
|||
|
|||
/* |
|||
* @author ZZQ |
|||
* @Date 2023/2/10 11:27 上午 |
|||
*/ |
|||
@Slf4j |
|||
public class DruidFilter extends FilterEventAdapter { |
|||
|
|||
@Override |
|||
public int preparedStatement_executeUpdate(FilterChain chain, PreparedStatementProxy statement) throws SQLException { |
|||
|
|||
return super.preparedStatement_executeUpdate(chain, statement); |
|||
} |
|||
|
|||
@Override |
|||
public int statement_executeUpdate(FilterChain chain, StatementProxy statement, String sql) throws SQLException { |
|||
|
|||
return super.statement_executeUpdate(chain, statement, sql); |
|||
} |
|||
|
|||
@Override |
|||
protected void statementExecuteAfter(StatementProxy statement, String sql, boolean result) { |
|||
String traceId = MDC.get("traceId"); |
|||
int size = statement.getParametersSize(); |
|||
String executeSql = sql; |
|||
int count = 0; |
|||
try { |
|||
count=statement.getUpdateCount(); |
|||
}catch (Exception ex){ } |
|||
if (StringUtils.isNotEmpty(traceId) && count>0) { |
|||
if (size > 0) { |
|||
Collection<JdbcParameter> values = statement.getParameters().values(); |
|||
List<Object> params = new ArrayList<>(); |
|||
for (JdbcParameter value : values) { |
|||
params.add(value.getValue()); |
|||
} |
|||
executeSql = SQLUtils.format(executeSql, JdbcUtils.MYSQL, params); |
|||
} |
|||
log.info("[----SQL----][update][ SQL: {} ]", executeSql); |
|||
} |
|||
super.statementExecuteAfter(statement, sql, result); |
|||
} |
|||
@Override |
|||
public ResultSetProxy statement_getResultSet(FilterChain chain, StatementProxy statement) throws SQLException { |
|||
ResultSetProxy rs = super.statement_getResultSet(chain, statement); |
|||
String executeSql = statement.getLastExecuteSql(); |
|||
String traceId = MDC.get("traceId"); |
|||
if (StringUtils.isNotEmpty(traceId)){ |
|||
int result = 0; |
|||
if (rs != null) { |
|||
ResultSetImpl rss = rs.getResultSetRaw().unwrap(ResultSetImpl.class); |
|||
result = rss.getRows().size(); |
|||
} |
|||
try { |
|||
int size = statement.getParametersSize(); |
|||
if (size>0){ |
|||
Collection<JdbcParameter> values = statement.getParameters().values(); |
|||
List<Object> params = new ArrayList<>(); |
|||
for (JdbcParameter value : values) { |
|||
params.add(value.getValue()); |
|||
} |
|||
executeSql = SQLUtils.format(executeSql, JdbcUtils.MYSQL, params); |
|||
} |
|||
}catch (Exception ex){ |
|||
log.warn("[-SQL解析异常-][{}]",ex.getMessage()); |
|||
} |
|||
log.info("[----SQL----][select][执行结果:{}][ SQL: {} ]",result, executeSql); |
|||
} |
|||
return rs; |
|||
} |
|||
|
|||
|
|||
|
|||
|
|||
|
|||
} |
|||
@ -0,0 +1,37 @@ |
|||
package org.nl.monitor.controller.log; |
|||
|
|||
import io.swagger.annotations.ApiOperation; |
|||
import lombok.RequiredArgsConstructor; |
|||
import org.nl.monitor.server.LogService; |
|||
import org.nl.monitor.server.log.dto.LogQuery; |
|||
import org.springframework.http.HttpStatus; |
|||
import org.springframework.http.ResponseEntity; |
|||
import org.springframework.web.bind.annotation.*; |
|||
|
|||
/** |
|||
* @author ldjun |
|||
* @version 1.0 |
|||
* @date 2023年01月29日 18:55 |
|||
* @desc desc |
|||
*/ |
|||
|
|||
@RestController |
|||
@RequiredArgsConstructor |
|||
@RequestMapping("/api/esLog") |
|||
public class LogController { |
|||
private final LogService esLogService; |
|||
|
|||
|
|||
@GetMapping("/labels/{type}") |
|||
@ApiOperation("获取标签") |
|||
public ResponseEntity<Object> labelsValues(@PathVariable String type) { |
|||
return new ResponseEntity<>(esLogService.getLabelsValues(type), HttpStatus.OK); |
|||
} |
|||
|
|||
@PostMapping("/query") |
|||
@ApiOperation("日志查询") |
|||
public ResponseEntity<Object> queryAll(@RequestBody LogQuery query) { |
|||
return new ResponseEntity<>(esLogService.query(query), HttpStatus.OK); |
|||
} |
|||
|
|||
} |
|||
@ -0,0 +1,26 @@ |
|||
package org.nl.monitor.server; |
|||
|
|||
import com.alibaba.fastjson.JSONArray; |
|||
import com.baomidou.mybatisplus.extension.plugins.pagination.Page; |
|||
import org.nl.monitor.server.log.dto.LogQuery; |
|||
|
|||
/** |
|||
* @author ldjun |
|||
* @version 1.0 |
|||
* @date 2023年02月07日 14:34 |
|||
* @desc desc |
|||
*/ |
|||
public interface LogService { |
|||
/** |
|||
* 获取labels和values树 |
|||
* @return |
|||
*/ |
|||
JSONArray getLabelsValues(String type); |
|||
|
|||
/** |
|||
* 日志查询 |
|||
* @param logQuery |
|||
* @return |
|||
*/ |
|||
Page query(LogQuery logQuery); |
|||
} |
|||
@ -0,0 +1,46 @@ |
|||
package org.nl.monitor.server.log.dto; |
|||
|
|||
import lombok.Data; |
|||
|
|||
import java.util.Date; |
|||
|
|||
/* |
|||
* @author ZZQ |
|||
* @Date 2023/2/8 5:18 下午 |
|||
*/ |
|||
@Data |
|||
public class LogQuery { |
|||
/** |
|||
* 创建时间范围查询 |
|||
*/ |
|||
private Date startTime; |
|||
private Date endTime; |
|||
/** |
|||
* 追踪id |
|||
*/ |
|||
private String traceId; |
|||
/** |
|||
* 日志内容模糊匹配 |
|||
*/ |
|||
private String message; |
|||
/** |
|||
* 日志级别 |
|||
*/ |
|||
private String logLevel; |
|||
/** |
|||
* 系统标签 |
|||
*/ |
|||
private String system; |
|||
/** |
|||
* 是否只查询Http相关请求 |
|||
*/ |
|||
private Boolean isRequest = Boolean.TRUE; |
|||
/** |
|||
* 是否过滤wql日志 |
|||
*/ |
|||
private Boolean filterSql = Boolean.TRUE; |
|||
|
|||
private Integer size = 20; |
|||
|
|||
private Integer page = 1; |
|||
} |
|||
@ -0,0 +1,28 @@ |
|||
package org.nl.monitor.server.log.dto; |
|||
|
|||
import lombok.Data; |
|||
import org.springframework.data.annotation.Id; |
|||
import org.springframework.data.elasticsearch.annotations.Document; |
|||
|
|||
/* |
|||
* @author ZZQ |
|||
* @Date 2023/2/8 4:06 下午 |
|||
*/ |
|||
@Document(indexName = "lms_log", type = "lms_log") |
|||
@Data |
|||
public class LogRepositoryDTO { |
|||
|
|||
private String message; |
|||
private String host; |
|||
private String logLevel; |
|||
private String logger; |
|||
private String requestTime; |
|||
private String requestIp; |
|||
@Id |
|||
private String id; |
|||
private String traceId; |
|||
private String requestMethod; |
|||
private String thread; |
|||
private String system; |
|||
|
|||
} |
|||
@ -0,0 +1,115 @@ |
|||
package org.nl.monitor.server.log.impl; |
|||
|
|||
import com.alibaba.fastjson.JSONArray; |
|||
import com.alibaba.fastjson.JSONObject; |
|||
import com.baomidou.mybatisplus.extension.plugins.pagination.Page; |
|||
import lombok.RequiredArgsConstructor; |
|||
import org.apache.commons.lang3.StringUtils; |
|||
import org.apache.poi.ss.formula.functions.T; |
|||
import org.elasticsearch.action.search.SearchResponse; |
|||
import org.elasticsearch.index.query.BoolQueryBuilder; |
|||
import org.elasticsearch.index.query.QueryBuilders; |
|||
import org.elasticsearch.script.Script; |
|||
import org.elasticsearch.search.aggregations.AggregationBuilders; |
|||
import org.elasticsearch.search.aggregations.Aggregations; |
|||
import org.elasticsearch.search.aggregations.bucket.terms.Terms; |
|||
import org.nl.monitor.server.LogService; |
|||
import org.nl.monitor.server.log.dto.LogQuery; |
|||
import org.nl.monitor.server.log.dto.LogRepositoryDTO; |
|||
import org.nl.monitor.server.log.repository.LogRepository; |
|||
import org.springframework.data.domain.PageRequest; |
|||
import org.springframework.data.domain.Sort; |
|||
import org.springframework.data.elasticsearch.core.ElasticsearchRestTemplate; |
|||
import org.springframework.data.elasticsearch.core.aggregation.impl.AggregatedPageImpl; |
|||
import org.springframework.data.elasticsearch.core.query.FetchSourceFilter; |
|||
import org.springframework.data.elasticsearch.core.query.NativeSearchQueryBuilder; |
|||
import org.springframework.stereotype.Service; |
|||
import org.springframework.util.CollectionUtils; |
|||
|
|||
import java.util.List; |
|||
|
|||
/** |
|||
* @author ldjun |
|||
* @version 1.0 |
|||
* @date 2023年02月07日 14:35 |
|||
* @desc desc |
|||
*/ |
|||
@Service |
|||
@RequiredArgsConstructor |
|||
public class LogServiceImpl implements LogService { |
|||
|
|||
private final LogRepository esLogRepository; |
|||
|
|||
private final ElasticsearchRestTemplate elasticsearchRestTemplate; |
|||
|
|||
@Override |
|||
public Page query(LogQuery logQuery){ |
|||
Page<T> page = new Page<>(); |
|||
if (logQuery != null){ |
|||
BoolQueryBuilder query = QueryBuilders.boolQuery(); //requestMethod
|
|||
extractedParam(logQuery, query); |
|||
Iterable<LogRepositoryDTO> all = esLogRepository.search(query, PageRequest.of(logQuery.getPage()-1,logQuery.getSize(), Sort.by("@timestamp").descending())); |
|||
page.setRecords(((AggregatedPageImpl) all).getContent()); |
|||
page.setTotal(((AggregatedPageImpl) all).getTotalElements()); |
|||
page.setPages(logQuery.getPage()); |
|||
page.setSize(logQuery.getSize()); |
|||
} |
|||
return page; |
|||
} |
|||
|
|||
|
|||
private void extractedParam(LogQuery logQuery, BoolQueryBuilder query) { |
|||
if (StringUtils.isNotEmpty(logQuery.getLogLevel())){ |
|||
query.must().add(QueryBuilders.matchQuery("logLevel", logQuery.getLogLevel())); |
|||
} |
|||
if (StringUtils.isNotEmpty(logQuery.getSystem())){ |
|||
query.must().add(QueryBuilders.matchQuery("system", logQuery.getSystem())); |
|||
} |
|||
if (logQuery.getIsRequest()){ |
|||
query.must().add(QueryBuilders.existsQuery("requestMethod")); |
|||
} |
|||
if (logQuery.getFilterSql()){ |
|||
query.mustNot().add(QueryBuilders.wildcardQuery("logger","org.nl.modules.wql.core.engine.*")); |
|||
} |
|||
query.mustNot().add(QueryBuilders.matchPhraseQuery("logger","org.elasticsearch.client.RestClient")); |
|||
if (StringUtils.isNotEmpty(logQuery.getTraceId())){ |
|||
query.must().add(QueryBuilders.matchQuery("traceId", logQuery.getTraceId())); |
|||
} |
|||
if (StringUtils.isNotEmpty(logQuery.getMessage())){ |
|||
query.must().add(QueryBuilders.matchQuery("message", logQuery.getMessage()).minimumShouldMatch("80%")); |
|||
} |
|||
if (logQuery.getEndTime()!=null ){ |
|||
String script = "doc['@timestamp'].value.millis < " + logQuery.getEndTime().getTime() + "L"; |
|||
query.must().add(QueryBuilders.scriptQuery(new Script(script))); |
|||
} |
|||
if (logQuery.getStartTime()!=null){ |
|||
String script = "doc['@timestamp'].value.millis > " + logQuery.getStartTime().getTime() + "L"; |
|||
query.must().add(QueryBuilders.scriptQuery(new Script(script))); |
|||
} |
|||
} |
|||
|
|||
|
|||
|
|||
@Override |
|||
public JSONArray getLabelsValues(String type) { |
|||
JSONArray result = new JSONArray(); |
|||
FetchSourceFilter fetchSourceFilter = new FetchSourceFilter(new String[]{type}, null); |
|||
NativeSearchQueryBuilder queryBuilder = new NativeSearchQueryBuilder(); |
|||
queryBuilder.withCollapseField(type+".keyword"); |
|||
queryBuilder.withSourceFilter(fetchSourceFilter); |
|||
queryBuilder.addAggregation(AggregationBuilders.terms(type).field(type+".keyword").size(100)); |
|||
Aggregations agg = elasticsearchRestTemplate.query(queryBuilder.build(), SearchResponse::getAggregations); |
|||
Terms terms = agg.get(type); |
|||
List<? extends Terms.Bucket> buckets = terms.getBuckets(); |
|||
if (!CollectionUtils.isEmpty(buckets)){ |
|||
buckets.stream().map(Terms.Bucket::getKeyAsString).forEach(v-> { |
|||
JSONObject item = new JSONObject(); |
|||
item.put("label", v); |
|||
item.put("value", v); |
|||
result.add(item); |
|||
}); |
|||
} |
|||
return result; |
|||
} |
|||
|
|||
} |
|||
@ -0,0 +1,14 @@ |
|||
package org.nl.monitor.server.log.repository; |
|||
|
|||
import org.nl.monitor.server.log.dto.LogRepositoryDTO; |
|||
import org.springframework.data.elasticsearch.repository.ElasticsearchRepository; |
|||
import org.springframework.stereotype.Repository; |
|||
|
|||
/* |
|||
* @author ZZQ |
|||
* @Date 2023/2/8 4:11 下午 |
|||
*/ |
|||
@Repository |
|||
public interface LogRepository extends ElasticsearchRepository<LogRepositoryDTO, String> { |
|||
|
|||
} |
|||
@ -0,0 +1 @@ |
|||
druid.filters.DruidFilter=org.nl.config.DruidFilter |
|||
Loading…
Reference in new issue