|
@ -2,17 +2,20 @@ package org.nl.config.lucene; |
|
|
|
|
|
|
|
|
import cn.hutool.core.date.DateTime; |
|
|
import cn.hutool.core.date.DateTime; |
|
|
import cn.hutool.core.date.DateUtil; |
|
|
import cn.hutool.core.date.DateUtil; |
|
|
import com.alibaba.fastjson.JSONArray; |
|
|
import cn.hutool.core.util.ObjectUtil; |
|
|
import com.alibaba.fastjson.JSONObject; |
|
|
import com.alibaba.fastjson.JSONObject; |
|
|
import lombok.extern.slf4j.Slf4j; |
|
|
import lombok.extern.slf4j.Slf4j; |
|
|
|
|
|
import org.apache.lucene.analysis.Analyzer; |
|
|
import org.apache.lucene.document.Document; |
|
|
import org.apache.lucene.document.Document; |
|
|
import org.apache.lucene.index.DirectoryReader; |
|
|
import org.apache.lucene.index.DirectoryReader; |
|
|
import org.apache.lucene.index.IndexReader; |
|
|
import org.apache.lucene.index.IndexReader; |
|
|
import org.apache.lucene.index.Term; |
|
|
import org.apache.lucene.index.Term; |
|
|
|
|
|
import org.apache.lucene.queryparser.classic.QueryParser; |
|
|
import org.apache.lucene.search.*; |
|
|
import org.apache.lucene.search.*; |
|
|
import org.apache.lucene.store.Directory; |
|
|
import org.apache.lucene.store.Directory; |
|
|
import org.apache.lucene.store.FSDirectory; |
|
|
import org.apache.lucene.store.FSDirectory; |
|
|
import org.apache.lucene.util.BytesRef; |
|
|
import org.apache.lucene.util.BytesRef; |
|
|
|
|
|
import org.wltea.analyzer.lucene.IKAnalyzer; |
|
|
|
|
|
|
|
|
import java.nio.file.Paths; |
|
|
import java.nio.file.Paths; |
|
|
import java.util.ArrayList; |
|
|
import java.util.ArrayList; |
|
@ -26,33 +29,30 @@ import java.util.Map; |
|
|
@Slf4j |
|
|
@Slf4j |
|
|
public class Searcher { |
|
|
public class Searcher { |
|
|
|
|
|
|
|
|
public static Map<String, Object> search(String indexDir, String ext,Map whereJson) throws Exception { |
|
|
public static Map<String, Object> search(String indexDir, JSONObject whereJson) throws Exception { |
|
|
//获取要查询的路径,也就是索引所在的位置
|
|
|
//获取要查询的路径,也就是索引所在的位置
|
|
|
Directory dir = FSDirectory.open(Paths.get(indexDir)); |
|
|
Directory dir = FSDirectory.open(Paths.get(indexDir)); |
|
|
IndexReader reader = DirectoryReader.open(dir); |
|
|
IndexReader reader = DirectoryReader.open(dir); |
|
|
//构建IndexSearcher
|
|
|
//构建IndexSearcher
|
|
|
IndexSearcher searcher = new IndexSearcher(reader); |
|
|
IndexSearcher searcher = new IndexSearcher(reader); |
|
|
//标准分词器,会自动去掉空格啊,is a the等单词
|
|
|
//标准分词器,会自动去掉空格啊,is a the等单词
|
|
|
// Analyzer analyzer = new StandardAnalyzer();
|
|
|
Analyzer analyzer = new IKAnalyzer(true); |
|
|
// Analyzer analyzer = new IKAnalyzer(false);
|
|
|
|
|
|
//查询解析器
|
|
|
|
|
|
// QueryParser queryParser = new QueryParser("fieldContent", analyzer);
|
|
|
|
|
|
|
|
|
|
|
|
//记录索引开始时间
|
|
|
//记录索引开始时间
|
|
|
long startTime = System.currentTimeMillis(); |
|
|
// long startTime = System.currentTimeMillis();
|
|
|
// 实际上Lucene本身不支持分页。因此我们需要自己进行逻辑分页。我们要准备分页参数:
|
|
|
// 实际上Lucene本身不支持分页。因此我们需要自己进行逻辑分页。我们要准备分页参数:
|
|
|
int pageSize = Integer.parseInt(whereJson.get("size").toString());// 每页条数
|
|
|
int pageSize = Integer.parseInt(whereJson.get("size").toString());// 每页条数
|
|
|
int pageNum = Integer.parseInt(whereJson.get("page").toString());// 当前页码
|
|
|
int pageNum = Integer.parseInt(whereJson.get("page").toString()) - 1;// 当前页码
|
|
|
int start = pageNum * pageSize;// 当前页的起始条数
|
|
|
int start = pageNum * pageSize;// 当前页的起始条数
|
|
|
int end = start + pageSize;// 当前页的结束条数(不能包含)
|
|
|
int end = start + pageSize;// 当前页的结束条数(不能包含)
|
|
|
// 创建排序对象,需要排序字段SortField,参数:字段的名称、字段的类型、是否反转如果是false,升序。true降序
|
|
|
// 创建排序对象,需要排序字段SortField,参数:字段的名称、字段的类型、是否反转如果是false,升序。true降序
|
|
|
Sort sort = new Sort(new SortField("logTime", SortField.Type.DOC,true)); |
|
|
Sort sort = new Sort(new SortField("timestamp", SortField.Type.DOC,true)); |
|
|
|
|
|
|
|
|
TopDocs docs = null; |
|
|
TopDocs docs = null; |
|
|
BooleanQuery.Builder booleanQueryBuilder = new BooleanQuery.Builder(); |
|
|
BooleanQuery.Builder booleanQueryBuilder = new BooleanQuery.Builder(); |
|
|
//时间范围查询
|
|
|
//时间范围查询
|
|
|
String startDate = (String) whereJson.get("begin_time"); |
|
|
String startDate = whereJson.getString("begin_time"); |
|
|
String endDate = (String) whereJson.get("end_time"); |
|
|
String endDate = whereJson.getString("end_time"); |
|
|
Calendar calendar=Calendar.getInstance(); |
|
|
Calendar calendar=Calendar.getInstance(); |
|
|
calendar.set(1970, 0, 1); |
|
|
calendar.set(1970, 0, 1); |
|
|
if (startDate == null){ |
|
|
if (startDate == null){ |
|
@ -65,74 +65,90 @@ public class Searcher { |
|
|
} else { |
|
|
} else { |
|
|
endDate = LuceneIndexWriter.getDate(endDate); |
|
|
endDate = LuceneIndexWriter.getDate(endDate); |
|
|
} |
|
|
} |
|
|
TermRangeQuery termRangeQuery = new TermRangeQuery("logTime", new BytesRef(startDate), new BytesRef(endDate), true, true); |
|
|
// 字段之间的与或非关系,MUST表示and,MUST_NOT表示not,SHOULD表示or,有几个fields就必须有几个clauses
|
|
|
|
|
|
TermRangeQuery termRangeQuery = new TermRangeQuery("timestamp", new BytesRef(startDate), |
|
|
|
|
|
new BytesRef(endDate), true, true); |
|
|
booleanQueryBuilder.add(termRangeQuery,BooleanClause.Occur.MUST); |
|
|
booleanQueryBuilder.add(termRangeQuery,BooleanClause.Occur.MUST); |
|
|
if (whereJson.get("device_code") != null){ |
|
|
if (ObjectUtil.isNotEmpty(whereJson.get(LogMessageConstant.FIELD_MESSAGE))){ |
|
|
Query termQuery = new TermQuery(new Term("device_code", (String) whereJson.get("device_code"))); |
|
|
//查询解析器
|
|
|
booleanQueryBuilder.add(termQuery,BooleanClause.Occur.MUST); |
|
|
QueryParser queryParser = new QueryParser("message", analyzer); |
|
|
} |
|
|
Query query = queryParser.parse(whereJson.getString("message")); |
|
|
if (whereJson.get("method") != null){ |
|
|
booleanQueryBuilder.add(query, BooleanClause.Occur.MUST); |
|
|
Query termQuery = new TermQuery(new Term("method", (String) whereJson.get("method"))); |
|
|
|
|
|
booleanQueryBuilder.add(termQuery,BooleanClause.Occur.MUST); |
|
|
|
|
|
} |
|
|
|
|
|
if (whereJson.get("status_code") != null){ |
|
|
|
|
|
Query termQuery = new TermQuery(new Term("status_code", (String) whereJson.get("status_code"))); |
|
|
|
|
|
booleanQueryBuilder.add(termQuery,BooleanClause.Occur.MUST); |
|
|
|
|
|
} |
|
|
} |
|
|
if (whereJson.get("requestparam") != null){ |
|
|
if (ObjectUtil.isNotEmpty(whereJson.get(LogMessageConstant.FIELD_TRACEID))){ |
|
|
WildcardQuery query = new WildcardQuery(new Term("requestparam", "*"+(String) whereJson.get("requestparam")+"*")); |
|
|
//查询解析器
|
|
|
booleanQueryBuilder.add(query,BooleanClause.Occur.MUST); |
|
|
TermQuery termQuery = new TermQuery(new Term(LogMessageConstant.FIELD_TRACEID, |
|
|
|
|
|
whereJson.getString(LogMessageConstant.FIELD_TRACEID).trim())); |
|
|
|
|
|
booleanQueryBuilder.add(termQuery, BooleanClause.Occur.MUST); |
|
|
} |
|
|
} |
|
|
if (whereJson.get("responseparam") != null){ |
|
|
if (ObjectUtil.isNotEmpty(whereJson.get(LogMessageConstant.FIELD_LEVEL))){ |
|
|
WildcardQuery query = new WildcardQuery(new Term("responseparam", "*"+(String) whereJson.get("responseparam")+"*")); |
|
|
//查询解析器
|
|
|
booleanQueryBuilder.add(query,BooleanClause.Occur.MUST); |
|
|
TermQuery termQuery = new TermQuery(new Term(LogMessageConstant.FIELD_LEVEL, |
|
|
} |
|
|
whereJson.get(LogMessageConstant.FIELD_LEVEL).toString())); |
|
|
if (whereJson.get("blurry") != null) { |
|
|
booleanQueryBuilder.add(termQuery, BooleanClause.Occur.MUST); |
|
|
WildcardQuery query = new WildcardQuery(new Term("fieldContent", "*"+(String) whereJson.get("blurry")+"*")); |
|
|
|
|
|
booleanQueryBuilder.add(query, BooleanClause.Occur.MUST); |
|
|
|
|
|
} |
|
|
} |
|
|
docs = searcher.search(booleanQueryBuilder.build(), end,sort); |
|
|
docs = searcher.search(booleanQueryBuilder.build(), end,sort); |
|
|
//记录索引时间
|
|
|
//记录索引时间
|
|
|
long endTime = System.currentTimeMillis(); |
|
|
// long endTime = System.currentTimeMillis();
|
|
|
log.info("匹配{}共耗时{}毫秒",booleanQueryBuilder.build(),(endTime-startTime)); |
|
|
// log.info("匹配{}共耗时{}毫秒",booleanQueryBuilder.build(),(endTime-startTime));
|
|
|
log.info("查询到{}条日志文件", docs.totalHits.value); |
|
|
// log.info("查询到{}条日志文件", docs.totalHits.value);
|
|
|
List<String> list = new ArrayList<>(); |
|
|
List<String> list = new ArrayList<>(); |
|
|
ScoreDoc[] scoreDocs = docs.scoreDocs; |
|
|
ScoreDoc[] scoreDocs = docs.scoreDocs; |
|
|
if (end > docs.totalHits.value) end = (int) docs.totalHits.value; |
|
|
if (end > docs.totalHits.value) end = (int) docs.totalHits.value; |
|
|
JSONArray array = new JSONArray(); |
|
|
|
|
|
|
|
|
|
|
|
for (int i = start; i < end; i++) { |
|
|
for (int i = start; i < end; i++) { |
|
|
ScoreDoc scoreDoc = scoreDocs[i]; |
|
|
ScoreDoc scoreDoc = scoreDocs[i]; |
|
|
Document doc = reader.document(scoreDoc.doc); |
|
|
Document doc = reader.document(scoreDoc.doc); |
|
|
JSONObject object = new JSONObject(); |
|
|
String logInfo = LogMessageConstant.COLOR_YELLOW + doc.get(LogMessageConstant.FIELD_TRACEID) + |
|
|
object.put("content",doc.get("fieldContent")); |
|
|
LogMessageConstant.COLOR_RESET + " - " + |
|
|
object.put("device_code",doc.get("device_code")); |
|
|
LogMessageConstant.COLOR_RED + doc.get(LogMessageConstant.FIELD_TIMESTAMP) + |
|
|
object.put("logTime",doc.get("logTime")); |
|
|
LogMessageConstant.COLOR_RESET + " - " + |
|
|
object.put("method",doc.get("method")); |
|
|
LogMessageConstant.COLOR_GREEN + "[" + doc.get(LogMessageConstant.FIELD_THREAD) + "]" + |
|
|
object.put("status_code",doc.get("status_code")); |
|
|
LogMessageConstant.COLOR_RESET + " - " + |
|
|
object.put("requestparam",doc.get("requestparam")); |
|
|
LogMessageConstant.COLOR_BLACK + doc.get(LogMessageConstant.FIELD_LEVEL) + |
|
|
object.put("responseparam",doc.get("responseparam")); |
|
|
LogMessageConstant.COLOR_RESET + " - " + |
|
|
if(doc.get("fieldContent") != null) { |
|
|
LogMessageConstant.COLOR_MAGENTA + doc.get(LogMessageConstant.FIELD_CLASS_NAME) + |
|
|
array.add(object); |
|
|
LogMessageConstant.COLOR_RESET + " - " + |
|
|
} |
|
|
LogMessageConstant.COLOR_BLACK + highlightKeyword(doc.get(LogMessageConstant.FIELD_MESSAGE), whereJson.getString("message")); |
|
|
} |
|
|
System.out.println(logInfo); |
|
|
for(Object logDto:array){ |
|
|
list.add(logInfo); |
|
|
log.info(logDto.toString()); |
|
|
|
|
|
} |
|
|
} |
|
|
reader.close(); |
|
|
reader.close(); |
|
|
JSONObject jo = new JSONObject(); |
|
|
JSONObject jo = new JSONObject(); |
|
|
jo.put("content", array); |
|
|
jo.put("content", list); |
|
|
jo.put("totalElements", docs.totalHits.value); |
|
|
jo.put("totalElements", docs.totalHits.value); |
|
|
return jo; |
|
|
return jo; |
|
|
} |
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
public static String highlightKeyword(String text, String keyword) { |
|
|
|
|
|
if (ObjectUtil.isEmpty(keyword)) { |
|
|
|
|
|
return text; |
|
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
int startIndex = text.indexOf(keyword); |
|
|
|
|
|
if (startIndex != -1) { |
|
|
|
|
|
int endIndex = startIndex + keyword.length(); |
|
|
|
|
|
String beforeKeyword = text.substring(0, startIndex); |
|
|
|
|
|
String afterKeyword = text.substring(endIndex); |
|
|
|
|
|
String highlightedKeyword = LogMessageConstant.BACKGROUND_YELLOW + keyword + LogMessageConstant.COLOR_RESET |
|
|
|
|
|
+ LogMessageConstant.COLOR_BLACK; |
|
|
|
|
|
return beforeKeyword + highlightedKeyword + afterKeyword; |
|
|
|
|
|
} else { |
|
|
|
|
|
return text; |
|
|
|
|
|
} |
|
|
|
|
|
} |
|
|
|
|
|
|
|
|
public static void main(String[] args) { |
|
|
public static void main(String[] args) { |
|
|
String indexDir = "D:\\lucene\\index"; |
|
|
String indexDir = "D:\\lucene\\index"; |
|
|
//查询这个字符串
|
|
|
//查询这个字符串
|
|
|
String q = "07.832"; |
|
|
JSONObject whereJson = new JSONObject(); |
|
|
Map whereJson = null; |
|
|
whereJson.put("size", "500"); |
|
|
|
|
|
whereJson.put("page", "1"); |
|
|
|
|
|
whereJson.put("message", "请求方法参数"); |
|
|
|
|
|
// whereJson.put(LogMessageConstant.FIELD_TRACEID, "13244183367577216");
|
|
|
|
|
|
|
|
|
try { |
|
|
try { |
|
|
search(indexDir, q,whereJson); |
|
|
search(indexDir, whereJson); |
|
|
} catch (Exception e) { |
|
|
} catch (Exception e) { |
|
|
e.printStackTrace(); |
|
|
e.printStackTrace(); |
|
|
} |
|
|
} |
|
|