zhangzhiqiang
2 years ago
11 changed files with 411 additions and 32 deletions
@ -0,0 +1,95 @@ |
|||||
|
package org.nl.config; |
||||
|
|
||||
|
import com.alibaba.druid.filter.FilterChain; |
||||
|
import com.alibaba.druid.filter.FilterEventAdapter; |
||||
|
import com.alibaba.druid.proxy.jdbc.JdbcParameter; |
||||
|
import com.alibaba.druid.proxy.jdbc.PreparedStatementProxy; |
||||
|
import com.alibaba.druid.proxy.jdbc.ResultSetProxy; |
||||
|
import com.alibaba.druid.proxy.jdbc.StatementProxy; |
||||
|
import com.alibaba.druid.sql.SQLUtils; |
||||
|
import com.alibaba.druid.util.JdbcUtils; |
||||
|
import com.mysql.cj.jdbc.result.ResultSetImpl; |
||||
|
import lombok.extern.slf4j.Slf4j; |
||||
|
import org.apache.commons.lang3.StringUtils; |
||||
|
import org.slf4j.MDC; |
||||
|
|
||||
|
import java.sql.SQLException; |
||||
|
import java.util.ArrayList; |
||||
|
import java.util.Collection; |
||||
|
import java.util.List; |
||||
|
|
||||
|
/* |
||||
|
* @author ZZQ |
||||
|
* @Date 2023/2/10 11:27 上午 |
||||
|
*/ |
||||
|
@Slf4j |
||||
|
public class DruidFilter extends FilterEventAdapter { |
||||
|
|
||||
|
@Override |
||||
|
public int preparedStatement_executeUpdate(FilterChain chain, PreparedStatementProxy statement) throws SQLException { |
||||
|
|
||||
|
return super.preparedStatement_executeUpdate(chain, statement); |
||||
|
} |
||||
|
|
||||
|
@Override |
||||
|
public int statement_executeUpdate(FilterChain chain, StatementProxy statement, String sql) throws SQLException { |
||||
|
|
||||
|
return super.statement_executeUpdate(chain, statement, sql); |
||||
|
} |
||||
|
|
||||
|
@Override |
||||
|
protected void statementExecuteAfter(StatementProxy statement, String sql, boolean result) { |
||||
|
String traceId = MDC.get("traceId"); |
||||
|
int size = statement.getParametersSize(); |
||||
|
String executeSql = sql; |
||||
|
int count = 0; |
||||
|
try { |
||||
|
count=statement.getUpdateCount(); |
||||
|
}catch (Exception ex){ } |
||||
|
if (StringUtils.isNotEmpty(traceId) && count>0) { |
||||
|
if (size > 0) { |
||||
|
Collection<JdbcParameter> values = statement.getParameters().values(); |
||||
|
List<Object> params = new ArrayList<>(); |
||||
|
for (JdbcParameter value : values) { |
||||
|
params.add(value.getValue()); |
||||
|
} |
||||
|
executeSql = SQLUtils.format(executeSql, JdbcUtils.MYSQL, params); |
||||
|
} |
||||
|
log.info("[----SQL----][update][ SQL: {} ]", executeSql); |
||||
|
} |
||||
|
super.statementExecuteAfter(statement, sql, result); |
||||
|
} |
||||
|
@Override |
||||
|
public ResultSetProxy statement_getResultSet(FilterChain chain, StatementProxy statement) throws SQLException { |
||||
|
ResultSetProxy rs = super.statement_getResultSet(chain, statement); |
||||
|
String executeSql = statement.getLastExecuteSql(); |
||||
|
String traceId = MDC.get("traceId"); |
||||
|
if (StringUtils.isNotEmpty(traceId)){ |
||||
|
int result = 0; |
||||
|
if (rs != null) { |
||||
|
ResultSetImpl rss = rs.getResultSetRaw().unwrap(ResultSetImpl.class); |
||||
|
result = rss.getRows().size(); |
||||
|
} |
||||
|
try { |
||||
|
int size = statement.getParametersSize(); |
||||
|
if (size>0){ |
||||
|
Collection<JdbcParameter> values = statement.getParameters().values(); |
||||
|
List<Object> params = new ArrayList<>(); |
||||
|
for (JdbcParameter value : values) { |
||||
|
params.add(value.getValue()); |
||||
|
} |
||||
|
executeSql = SQLUtils.format(executeSql, JdbcUtils.MYSQL, params); |
||||
|
} |
||||
|
}catch (Exception ex){ |
||||
|
log.warn("[-SQL解析异常-][{}]",ex.getMessage()); |
||||
|
} |
||||
|
log.info("[----SQL----][select][执行结果:{}][ SQL: {} ]",result, executeSql); |
||||
|
} |
||||
|
return rs; |
||||
|
} |
||||
|
|
||||
|
|
||||
|
|
||||
|
|
||||
|
|
||||
|
} |
||||
@ -0,0 +1,37 @@ |
|||||
|
package org.nl.monitor.controller.log; |
||||
|
|
||||
|
import io.swagger.annotations.ApiOperation; |
||||
|
import lombok.RequiredArgsConstructor; |
||||
|
import org.nl.monitor.server.LogService; |
||||
|
import org.nl.monitor.server.log.dto.LogQuery; |
||||
|
import org.springframework.http.HttpStatus; |
||||
|
import org.springframework.http.ResponseEntity; |
||||
|
import org.springframework.web.bind.annotation.*; |
||||
|
|
||||
|
/** |
||||
|
* @author ldjun |
||||
|
* @version 1.0 |
||||
|
* @date 2023年01月29日 18:55 |
||||
|
* @desc desc |
||||
|
*/ |
||||
|
|
||||
|
@RestController |
||||
|
@RequiredArgsConstructor |
||||
|
@RequestMapping("/api/esLog") |
||||
|
public class LogController { |
||||
|
private final LogService esLogService; |
||||
|
|
||||
|
|
||||
|
@GetMapping("/labels/{type}") |
||||
|
@ApiOperation("获取标签") |
||||
|
public ResponseEntity<Object> labelsValues(@PathVariable String type) { |
||||
|
return new ResponseEntity<>(esLogService.getLabelsValues(type), HttpStatus.OK); |
||||
|
} |
||||
|
|
||||
|
@PostMapping("/query") |
||||
|
@ApiOperation("日志查询") |
||||
|
public ResponseEntity<Object> queryAll(@RequestBody LogQuery query) { |
||||
|
return new ResponseEntity<>(esLogService.query(query), HttpStatus.OK); |
||||
|
} |
||||
|
|
||||
|
} |
||||
@ -0,0 +1,26 @@ |
|||||
|
package org.nl.monitor.server; |
||||
|
|
||||
|
import com.alibaba.fastjson.JSONArray; |
||||
|
import com.baomidou.mybatisplus.extension.plugins.pagination.Page; |
||||
|
import org.nl.monitor.server.log.dto.LogQuery; |
||||
|
|
||||
|
/** |
||||
|
* @author ldjun |
||||
|
* @version 1.0 |
||||
|
* @date 2023年02月07日 14:34 |
||||
|
* @desc desc |
||||
|
*/ |
||||
|
public interface LogService { |
||||
|
/** |
||||
|
* 获取labels和values树 |
||||
|
* @return |
||||
|
*/ |
||||
|
JSONArray getLabelsValues(String type); |
||||
|
|
||||
|
/** |
||||
|
* 日志查询 |
||||
|
* @param logQuery |
||||
|
* @return |
||||
|
*/ |
||||
|
Page query(LogQuery logQuery); |
||||
|
} |
||||
@ -0,0 +1,46 @@ |
|||||
|
package org.nl.monitor.server.log.dto; |
||||
|
|
||||
|
import lombok.Data; |
||||
|
|
||||
|
import java.util.Date; |
||||
|
|
||||
|
/* |
||||
|
* @author ZZQ |
||||
|
* @Date 2023/2/8 5:18 下午 |
||||
|
*/ |
||||
|
@Data |
||||
|
public class LogQuery { |
||||
|
/** |
||||
|
* 创建时间范围查询 |
||||
|
*/ |
||||
|
private Date startTime; |
||||
|
private Date endTime; |
||||
|
/** |
||||
|
* 追踪id |
||||
|
*/ |
||||
|
private String traceId; |
||||
|
/** |
||||
|
* 日志内容模糊匹配 |
||||
|
*/ |
||||
|
private String message; |
||||
|
/** |
||||
|
* 日志级别 |
||||
|
*/ |
||||
|
private String logLevel; |
||||
|
/** |
||||
|
* 系统标签 |
||||
|
*/ |
||||
|
private String system; |
||||
|
/** |
||||
|
* 是否只查询Http相关请求 |
||||
|
*/ |
||||
|
private Boolean isRequest = Boolean.TRUE; |
||||
|
/** |
||||
|
* 是否过滤wql日志 |
||||
|
*/ |
||||
|
private Boolean filterSql = Boolean.TRUE; |
||||
|
|
||||
|
private Integer size = 20; |
||||
|
|
||||
|
private Integer page = 1; |
||||
|
} |
||||
@ -0,0 +1,28 @@ |
|||||
|
package org.nl.monitor.server.log.dto; |
||||
|
|
||||
|
import lombok.Data; |
||||
|
import org.springframework.data.annotation.Id; |
||||
|
import org.springframework.data.elasticsearch.annotations.Document; |
||||
|
|
||||
|
/* |
||||
|
* @author ZZQ |
||||
|
* @Date 2023/2/8 4:06 下午 |
||||
|
*/ |
||||
|
@Document(indexName = "lms_log", type = "lms_log") |
||||
|
@Data |
||||
|
public class LogRepositoryDTO { |
||||
|
|
||||
|
private String message; |
||||
|
private String host; |
||||
|
private String logLevel; |
||||
|
private String logger; |
||||
|
private String requestTime; |
||||
|
private String requestIp; |
||||
|
@Id |
||||
|
private String id; |
||||
|
private String traceId; |
||||
|
private String requestMethod; |
||||
|
private String thread; |
||||
|
private String system; |
||||
|
|
||||
|
} |
||||
@ -0,0 +1,115 @@ |
|||||
|
package org.nl.monitor.server.log.impl; |
||||
|
|
||||
|
import com.alibaba.fastjson.JSONArray; |
||||
|
import com.alibaba.fastjson.JSONObject; |
||||
|
import com.baomidou.mybatisplus.extension.plugins.pagination.Page; |
||||
|
import lombok.RequiredArgsConstructor; |
||||
|
import org.apache.commons.lang3.StringUtils; |
||||
|
import org.apache.poi.ss.formula.functions.T; |
||||
|
import org.elasticsearch.action.search.SearchResponse; |
||||
|
import org.elasticsearch.index.query.BoolQueryBuilder; |
||||
|
import org.elasticsearch.index.query.QueryBuilders; |
||||
|
import org.elasticsearch.script.Script; |
||||
|
import org.elasticsearch.search.aggregations.AggregationBuilders; |
||||
|
import org.elasticsearch.search.aggregations.Aggregations; |
||||
|
import org.elasticsearch.search.aggregations.bucket.terms.Terms; |
||||
|
import org.nl.monitor.server.LogService; |
||||
|
import org.nl.monitor.server.log.dto.LogQuery; |
||||
|
import org.nl.monitor.server.log.dto.LogRepositoryDTO; |
||||
|
import org.nl.monitor.server.log.repository.LogRepository; |
||||
|
import org.springframework.data.domain.PageRequest; |
||||
|
import org.springframework.data.domain.Sort; |
||||
|
import org.springframework.data.elasticsearch.core.ElasticsearchRestTemplate; |
||||
|
import org.springframework.data.elasticsearch.core.aggregation.impl.AggregatedPageImpl; |
||||
|
import org.springframework.data.elasticsearch.core.query.FetchSourceFilter; |
||||
|
import org.springframework.data.elasticsearch.core.query.NativeSearchQueryBuilder; |
||||
|
import org.springframework.stereotype.Service; |
||||
|
import org.springframework.util.CollectionUtils; |
||||
|
|
||||
|
import java.util.List; |
||||
|
|
||||
|
/** |
||||
|
* @author ldjun |
||||
|
* @version 1.0 |
||||
|
* @date 2023年02月07日 14:35 |
||||
|
* @desc desc |
||||
|
*/ |
||||
|
@Service |
||||
|
@RequiredArgsConstructor |
||||
|
public class LogServiceImpl implements LogService { |
||||
|
|
||||
|
private final LogRepository esLogRepository; |
||||
|
|
||||
|
private final ElasticsearchRestTemplate elasticsearchRestTemplate; |
||||
|
|
||||
|
@Override |
||||
|
public Page query(LogQuery logQuery){ |
||||
|
Page<T> page = new Page<>(); |
||||
|
if (logQuery != null){ |
||||
|
BoolQueryBuilder query = QueryBuilders.boolQuery(); //requestMethod
|
||||
|
extractedParam(logQuery, query); |
||||
|
Iterable<LogRepositoryDTO> all = esLogRepository.search(query, PageRequest.of(logQuery.getPage()-1,logQuery.getSize(), Sort.by("@timestamp").descending())); |
||||
|
page.setRecords(((AggregatedPageImpl) all).getContent()); |
||||
|
page.setTotal(((AggregatedPageImpl) all).getTotalElements()); |
||||
|
page.setPages(logQuery.getPage()); |
||||
|
page.setSize(logQuery.getSize()); |
||||
|
} |
||||
|
return page; |
||||
|
} |
||||
|
|
||||
|
|
||||
|
private void extractedParam(LogQuery logQuery, BoolQueryBuilder query) { |
||||
|
if (StringUtils.isNotEmpty(logQuery.getLogLevel())){ |
||||
|
query.must().add(QueryBuilders.matchQuery("logLevel", logQuery.getLogLevel())); |
||||
|
} |
||||
|
if (StringUtils.isNotEmpty(logQuery.getSystem())){ |
||||
|
query.must().add(QueryBuilders.matchQuery("system", logQuery.getSystem())); |
||||
|
} |
||||
|
if (logQuery.getIsRequest()){ |
||||
|
query.must().add(QueryBuilders.existsQuery("requestMethod")); |
||||
|
} |
||||
|
if (logQuery.getFilterSql()){ |
||||
|
query.mustNot().add(QueryBuilders.wildcardQuery("logger","org.nl.modules.wql.core.engine.*")); |
||||
|
} |
||||
|
query.mustNot().add(QueryBuilders.matchPhraseQuery("logger","org.elasticsearch.client.RestClient")); |
||||
|
if (StringUtils.isNotEmpty(logQuery.getTraceId())){ |
||||
|
query.must().add(QueryBuilders.matchQuery("traceId", logQuery.getTraceId())); |
||||
|
} |
||||
|
if (StringUtils.isNotEmpty(logQuery.getMessage())){ |
||||
|
query.must().add(QueryBuilders.matchQuery("message", logQuery.getMessage()).minimumShouldMatch("80%")); |
||||
|
} |
||||
|
if (logQuery.getEndTime()!=null ){ |
||||
|
String script = "doc['@timestamp'].value.millis < " + logQuery.getEndTime().getTime() + "L"; |
||||
|
query.must().add(QueryBuilders.scriptQuery(new Script(script))); |
||||
|
} |
||||
|
if (logQuery.getStartTime()!=null){ |
||||
|
String script = "doc['@timestamp'].value.millis > " + logQuery.getStartTime().getTime() + "L"; |
||||
|
query.must().add(QueryBuilders.scriptQuery(new Script(script))); |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
|
||||
|
|
||||
|
@Override |
||||
|
public JSONArray getLabelsValues(String type) { |
||||
|
JSONArray result = new JSONArray(); |
||||
|
FetchSourceFilter fetchSourceFilter = new FetchSourceFilter(new String[]{type}, null); |
||||
|
NativeSearchQueryBuilder queryBuilder = new NativeSearchQueryBuilder(); |
||||
|
queryBuilder.withCollapseField(type+".keyword"); |
||||
|
queryBuilder.withSourceFilter(fetchSourceFilter); |
||||
|
queryBuilder.addAggregation(AggregationBuilders.terms(type).field(type+".keyword").size(100)); |
||||
|
Aggregations agg = elasticsearchRestTemplate.query(queryBuilder.build(), SearchResponse::getAggregations); |
||||
|
Terms terms = agg.get(type); |
||||
|
List<? extends Terms.Bucket> buckets = terms.getBuckets(); |
||||
|
if (!CollectionUtils.isEmpty(buckets)){ |
||||
|
buckets.stream().map(Terms.Bucket::getKeyAsString).forEach(v-> { |
||||
|
JSONObject item = new JSONObject(); |
||||
|
item.put("label", v); |
||||
|
item.put("value", v); |
||||
|
result.add(item); |
||||
|
}); |
||||
|
} |
||||
|
return result; |
||||
|
} |
||||
|
|
||||
|
} |
||||
@ -0,0 +1,14 @@ |
|||||
|
package org.nl.monitor.server.log.repository; |
||||
|
|
||||
|
import org.nl.monitor.server.log.dto.LogRepositoryDTO; |
||||
|
import org.springframework.data.elasticsearch.repository.ElasticsearchRepository; |
||||
|
import org.springframework.stereotype.Repository; |
||||
|
|
||||
|
/* |
||||
|
* @author ZZQ |
||||
|
* @Date 2023/2/8 4:11 下午 |
||||
|
*/ |
||||
|
@Repository |
||||
|
public interface LogRepository extends ElasticsearchRepository<LogRepositoryDTO, String> { |
||||
|
|
||||
|
} |
||||
@ -0,0 +1 @@ |
|||||
|
druid.filters.DruidFilter=org.nl.config.DruidFilter |
||||
Loading…
Reference in new issue