From 068394bfe43f8a40cfbc3c842d5800a8bee3390a Mon Sep 17 00:00:00 2001 From: gengby <858962040@qq.com> Date: Tue, 23 Jan 2024 17:47:15 +0800 Subject: [PATCH] rev:lucene --- acs/nladmin-system/.gitignore | 2 + acs/nladmin-system/pom.xml | 12 +- .../main/java/org/nl/acs/ext/wms/LmsUtil.java | 3 - .../org/nl/modules/lucene/common/Indexer.java | 178 ------------------ .../modules/lucene/common/LuceneAppender.java | 123 ------------ .../lucene/common/LuceneIndexWriter.java | 65 ------- .../nl/modules/lucene/common/Searcher.java | 144 -------------- .../lucene/config/AsyncLuceneAppender.java | 43 +++++ .../LogMessageConstant.java | 72 +++++-- .../modules/lucene/config/LuceneAppender.java | 133 +++++++++++++ .../lucene/config/LuceneProperties.java | 23 +++ .../config/LucenePropertyAndEncoder.java | 38 ++++ .../nl/modules/lucene/config/Property.java | 44 +++++ .../modules/lucene/config/StaticConfig.java | 21 --- .../nl/modules/lucene/config/UrlConfig.java | 13 -- .../modules/lucene/rest/LuceneController.java | 5 - .../service/LuceneExecuteLogService.java | 2 - .../modules/lucene/service/LuceneService.java | 6 - .../impl/LuceneExecuteLogServiceImpl.java | 52 ----- .../service/impl/LuceneServiceImpl.java | 152 +++++++++------ .../main/resources/config/application-dev.yml | 10 +- .../resources/config/application-dev2.yml | 151 +++++++++++++++ .../resources/config/application-prod.yml | 3 - .../src/main/resources/config/application.yml | 6 +- .../src/main/resources/log/AcsToWms.xml | 18 +- .../resources/log/AgvNdcOneDeviceDriver.xml | 33 ---- .../src/main/resources/log/AutoCreateInst.xml | 14 +- .../src/main/resources/log/Lucene.xml | 41 ---- .../main/resources/log/NDCAgvServiceImpl.xml | 33 ---- .../log/NDCSocketConnectionAutoRun.xml | 33 ---- .../src/main/resources/log/WmsToAcs.xml | 16 +- .../src/main/resources/logback-spring.xml | 115 ++++------- 32 files changed, 656 insertions(+), 948 deletions(-) delete mode 100644 acs/nladmin-system/src/main/java/org/nl/modules/lucene/common/Indexer.java delete mode 100644 acs/nladmin-system/src/main/java/org/nl/modules/lucene/common/LuceneAppender.java delete mode 100644 acs/nladmin-system/src/main/java/org/nl/modules/lucene/common/LuceneIndexWriter.java delete mode 100644 acs/nladmin-system/src/main/java/org/nl/modules/lucene/common/Searcher.java create mode 100644 acs/nladmin-system/src/main/java/org/nl/modules/lucene/config/AsyncLuceneAppender.java rename acs/nladmin-system/src/main/java/org/nl/modules/lucene/{common => config}/LogMessageConstant.java (56%) create mode 100644 acs/nladmin-system/src/main/java/org/nl/modules/lucene/config/LuceneAppender.java create mode 100644 acs/nladmin-system/src/main/java/org/nl/modules/lucene/config/LuceneProperties.java create mode 100644 acs/nladmin-system/src/main/java/org/nl/modules/lucene/config/LucenePropertyAndEncoder.java create mode 100644 acs/nladmin-system/src/main/java/org/nl/modules/lucene/config/Property.java delete mode 100644 acs/nladmin-system/src/main/java/org/nl/modules/lucene/config/StaticConfig.java delete mode 100644 acs/nladmin-system/src/main/java/org/nl/modules/lucene/config/UrlConfig.java create mode 100644 acs/nladmin-system/src/main/resources/config/application-dev2.yml delete mode 100644 acs/nladmin-system/src/main/resources/log/AgvNdcOneDeviceDriver.xml delete mode 100644 acs/nladmin-system/src/main/resources/log/Lucene.xml delete mode 100644 acs/nladmin-system/src/main/resources/log/NDCAgvServiceImpl.xml delete mode 100644 acs/nladmin-system/src/main/resources/log/NDCSocketConnectionAutoRun.xml diff --git a/acs/nladmin-system/.gitignore b/acs/nladmin-system/.gitignore index caa1a2a..82d97ca 100644 --- a/acs/nladmin-system/.gitignore +++ b/acs/nladmin-system/.gitignore @@ -6,3 +6,5 @@ */*.iml /.gradle/ /target/* +/C:* +/D:* diff --git a/acs/nladmin-system/pom.xml b/acs/nladmin-system/pom.xml index 2095112..0e88bab 100644 --- a/acs/nladmin-system/pom.xml +++ b/acs/nladmin-system/pom.xml @@ -178,13 +178,6 @@ oshi-core 5.0.1 - - - - com.github.loki4j - loki-logback-appender-jdk8 - 1.3.2 - org.apache.httpcomponents httpclient @@ -395,6 +388,11 @@ + + com.yomahub + tlog-all-spring-boot-starter + 1.5.0 + org.apache.lucene lucene-core diff --git a/acs/nladmin-system/src/main/java/org/nl/acs/ext/wms/LmsUtil.java b/acs/nladmin-system/src/main/java/org/nl/acs/ext/wms/LmsUtil.java index 57891f3..9b16a0b 100644 --- a/acs/nladmin-system/src/main/java/org/nl/acs/ext/wms/LmsUtil.java +++ b/acs/nladmin-system/src/main/java/org/nl/acs/ext/wms/LmsUtil.java @@ -6,10 +6,7 @@ import cn.hutool.http.HttpRequest; import com.alibaba.fastjson.JSON; import com.alibaba.fastjson.JSONObject; import lombok.extern.slf4j.Slf4j; -import org.nl.acs.log.LokiLog; -import org.nl.acs.log.LokiLogType; import org.nl.acs.log.service.DeviceExecuteLogService; -import org.nl.modules.common.exception.BadRequestException; import org.nl.modules.system.service.ParamService; import org.nl.modules.system.service.impl.ParamServiceImpl; import org.nl.modules.wql.util.SpringContextHolder; diff --git a/acs/nladmin-system/src/main/java/org/nl/modules/lucene/common/Indexer.java b/acs/nladmin-system/src/main/java/org/nl/modules/lucene/common/Indexer.java deleted file mode 100644 index a9c677c..0000000 --- a/acs/nladmin-system/src/main/java/org/nl/modules/lucene/common/Indexer.java +++ /dev/null @@ -1,178 +0,0 @@ -package org.nl.modules.lucene.common; - -import com.alibaba.fastjson.JSONObject; -import org.apache.commons.io.FileUtils; -import org.apache.lucene.analysis.Analyzer; -import org.apache.lucene.document.Document; -import org.apache.lucene.document.Field; -import org.apache.lucene.document.TextField; -import org.apache.lucene.index.IndexWriter; -import org.apache.lucene.index.IndexWriterConfig; -import org.apache.lucene.store.Directory; -import org.apache.lucene.store.FSDirectory; -import org.springframework.beans.factory.annotation.Value; -import org.wltea.analyzer.lucene.IKAnalyzer; - -import java.io.BufferedReader; -import java.io.File; -import java.io.FileReader; -import java.io.IOException; -import java.nio.file.Paths; -import java.util.Set; - -/** - * lucene索引器 - */ -public class Indexer { - /** - * 写索引实例 - */ - private IndexWriter writer; - - public IndexWriter getWriter() { - return writer; - } - - /** - * 构造方法，实例化IndexWriter - * @param indexDir - * @throws Exception - */ - public Indexer(String indexDir) throws Exception { - Directory dir = FSDirectory.open(Paths.get(indexDir)); - //标准分词器，会自动去掉空格啊，is a the等单词 -// Analyzer analyzer = new StandardAnalyzer(); - Analyzer analyzer = new IKAnalyzer(); - //将标准分词器配到写索引的配置中 - IndexWriterConfig config = new IndexWriterConfig(analyzer); - //实例化写索引对象 - writer = new IndexWriter(dir, config); - } - - /** - * 索引指定目录下的所有文件 - * @param dataDir - * @return - * @throws Exception - */ - public int indexAll(String dataDir) throws Exception { - // 获取该路径下的所有文件 - File[] files = new File(dataDir).listFiles(); - if (null != files) { - for (File file : files) { - //调用下面的indexFile方法，对每个文件进行索引 - indexFile(file); - } - } - //返回索引的文件数 -// return writer.numDocs(); - return writer.numRamDocs(); - } - - /** - * 索引指定的文件 - * @param file - * @throws Exception - */ - private void indexFile(File file) throws Exception { - System.out.println("索引文件的路径：" + file.getCanonicalPath()); - //调用下面的getDocument方法，获取该文件的document - Document doc = getDocument(file); - //添加索引文档 - //Document doc = json2Doc(jsonDoc); -// Document doc = new Document(); -// doc.add(new TextField("content", jsonDoc, Field.Store.YES)); - Field fieldContent=new TextField("fieldContent", FileUtils.readFileToString(null,"UTF-8"), Field.Store.YES); - - //将doc添加到索引中 - writer.addDocument(doc); - } - - /** - * 获取文档，文档里再设置每个字段，就类似于数据库中的一行记录 - * @param file - * @return - * @throws Exception - */ - private Document getDocument(File file) throws Exception { - Document doc = new Document(); - //开始添加字段 - //添加内容 - doc.add(new TextField("contents", new FileReader(file))); - //添加文件名，并把这个字段存到索引文件里 - doc.add(new TextField("fileName", file.getName(), Field.Store.YES)); - //添加文件路径 - doc.add(new TextField("fullPath", file.getCanonicalPath(), Field.Store.YES)); - return doc; - } - public Document json2Doc(String strDoc) { - Document doc = new Document(); - JSONObject jsonDoc = JSONObject.parseObject(strDoc); - Set keys = jsonDoc.keySet(); - for (String key : keys) { - doc.add(new TextField(key, jsonDoc.getString(key), Field.Store.YES)); - } - return doc; - } - - public void addLogIndex(String msg) throws IOException { - //步骤一：创建Directory对象，用于指定索引库的位置 RAMDirectory内存 - Directory directory = FSDirectory.open(new File("D:\\lucene\\index").toPath()); - //步骤二：创建一个IndexWriter对象，用于写索引 -// Analyzer analyzer = new StandardAnalyzer(); - IndexWriter indexWriter=new IndexWriter(directory,new IndexWriterConfig(new IKAnalyzer(false))); -// indexWriter.deleteAll();//清理所有索引库 -// IndexWriter indexWriter=new IndexWriter(directory,new IndexWriterConfig(new StandardAnalyzer())); - //记录索引开始时间 - long startTime = System.currentTimeMillis(); - //步骤三：读取磁盘中文件，对应每一个文件创建一个文档对象 - Document document = new Document(); -// document.add(new TextField("fieldContent", device_id, Field.Store.YES)); - document.add(new TextField("fieldContent", msg, Field.Store.YES)); - indexWriter.addDocument(document); - //记录索引结束时间 - long endTime = System.currentTimeMillis(); - System.out.println("建立索引"+ "共耗时" + (endTime-startTime) + "毫秒"); - indexWriter.commit(); - //步骤八：关闭资源 - indexWriter.close(); - System.out.println("建立索引成功-----关闭资源"); - } - //系统的日志文件路径 - @Value("${logging.file.path}") - private String logUrl; - - public static void main(String[] args)throws IOException { - //步骤一：创建Directory对象，用于指定索引库的位置 RAMDirectory内存 - Directory directory = FSDirectory.open(new File("D:\\lucene\\index").toPath()); - //步骤二：创建一个IndexWriter对象，用于写索引 -// Analyzer analyzer = new StandardAnalyzer(); - IndexWriter indexWriter=new IndexWriter(directory,new IndexWriterConfig(new IKAnalyzer(false))); - - indexWriter.deleteAll();//清理所有索引库 -// IndexWriter indexWriter=new IndexWriter(directory,new IndexWriterConfig(new StandardAnalyzer())); - //记录索引开始时间 - long startTime = System.currentTimeMillis(); - //步骤三：读取磁盘中文件，对应每一个文件创建一个文档对象 - File file=new File("D:\\testlog"); - //步骤四：获取文件列表 - File[] files = file.listFiles(); - for (File item:files) { - BufferedReader bufferedReader = new BufferedReader(new FileReader(item)); - String strLine = null; - while(null != (strLine = bufferedReader.readLine())){ - Document document = new Document(); -// document.add(new Field()); - document.add(new TextField("fieldContent", strLine, Field.Store.YES)); - indexWriter.addDocument(document); - } - } - //记录索引结束时间 - long endTime = System.currentTimeMillis(); - System.out.println("建立索引"+ "共耗时" + (endTime-startTime) + "毫秒"); - indexWriter.commit(); - //步骤八：关闭资源 - indexWriter.close(); - System.out.println("建立索引成功-----关闭资源"); - } -} diff --git a/acs/nladmin-system/src/main/java/org/nl/modules/lucene/common/LuceneAppender.java b/acs/nladmin-system/src/main/java/org/nl/modules/lucene/common/LuceneAppender.java deleted file mode 100644 index 56dd6ca..0000000 --- a/acs/nladmin-system/src/main/java/org/nl/modules/lucene/common/LuceneAppender.java +++ /dev/null @@ -1,123 +0,0 @@ -package org.nl.modules.lucene.common; -/** - * @author ldjun - * @version 1.0 - * @date 2023年08月24日 13:00 - * @desc desc - */ - -import ch.qos.logback.classic.spi.ILoggingEvent; -import ch.qos.logback.core.AppenderBase; -import cn.hutool.core.date.DateTime; -import cn.hutool.core.date.DateUtil; -import cn.hutool.core.util.ObjectUtil; -import com.alibaba.fastjson.JSONObject; -import org.apache.lucene.analysis.Analyzer; -import org.apache.lucene.document.*; -import org.apache.lucene.index.IndexWriter; -import org.apache.lucene.index.IndexWriterConfig; -import org.apache.lucene.store.Directory; -import org.apache.lucene.store.FSDirectory; -import org.nl.acs.opc.OpcItemDto; -import org.nl.modules.lucene.enums.LogTypeEnum; -import org.nl.modules.lucene.service.LuceneExecuteLogService; -import org.nl.modules.lucene.service.dto.LuceneLogDto; -import org.nl.modules.wql.util.SpringContextHolder; -import org.wltea.analyzer.lucene.IKAnalyzer; - -import java.io.IOException; -import java.nio.file.Paths; -import java.util.Map; - -public class LuceneAppender extends AppenderBase { - - private Directory index; - private IndexWriter indexWriter; - - - @Override - public void start() { - super.start(); - try { - index = FSDirectory.open(Paths.get(LogMessageConstant.INDEX_DIR)); - } catch (IOException e) { - e.printStackTrace(); - } - - // 初始化 Lucene 索引 - Analyzer analyzer = new IKAnalyzer(); - IndexWriterConfig config = new IndexWriterConfig(analyzer); - try { - indexWriter = new IndexWriter(index, config); - } catch (IOException e) { - e.printStackTrace(); - } - } - - - @Override - protected void append(ILoggingEvent event) { - String message = event.getFormattedMessage(); - try { - LuceneLogDto luceneLogDto = JSONObject.parseObject(message, LuceneLogDto.class); - -// LuceneLogDto luceneLogDto = new LuceneLogDto(itemDto.getOpc_server_code(), itemDto.getOpc_plc_code(), itemDto.getDevice_code(), itemDto.getItem_code().substring(itemDto.getItem_code().lastIndexOf(".") + 1), -// String.valueOf(itemDto.getHis_item_value()), String.valueOf(itemDto.getItem_value())); -// luceneLogDto.setLogType(LogTypeEnum.DEVICE_LOG.getDesc()); - //IndexWriter indexWriter = LuceneIndexWriter.getIndexWriter(); - //创建一个Document对象 - Document document = new Document(); - try { - //记录索引开始时间 - long startTime = System.currentTimeMillis(); - //向document对象中添加域。 - if (ObjectUtil.isNotEmpty(luceneLogDto.getDevice_code())) { - document.add(new StringField("device_code", luceneLogDto.getDevice_code(), Field.Store.YES)); -// document.add(new TextField("device_code", luceneLogDto.getDevice_code(), Field.Store.YES)); - } - if (ObjectUtil.isNotEmpty(luceneLogDto.getContent())) { - document.add(new StringField("fieldContent", luceneLogDto.getContent(), Field.Store.YES)); - } - if (ObjectUtil.isNotEmpty(luceneLogDto.getMethod())) { - document.add(new StringField("method", luceneLogDto.getMethod(), Field.Store.YES)); - } - if (ObjectUtil.isNotEmpty(luceneLogDto.getStatus_code())) { - document.add(new StringField("status_code", luceneLogDto.getStatus_code(), Field.Store.YES)); - } - if (ObjectUtil.isNotEmpty(luceneLogDto.getRequestparam())) { - document.add(new StringField("requestparam", luceneLogDto.getRequestparam(), Field.Store.YES)); - } - if (ObjectUtil.isNotEmpty(luceneLogDto.getResponseparam())) { - document.add(new StringField("responseparam", luceneLogDto.getResponseparam(), Field.Store.YES)); - } - document.add(new StringField("logType", luceneLogDto.getLogType(), Field.Store.YES)); - document.add(new StringField("logTime", DateUtil.format(new DateTime(), "yyyy-MM-dd HH:mm:ss.SSS"), Field.Store.YES)); - document.add(new NumericDocValuesField("logTime",System.currentTimeMillis()));//排序 - //记录索引结束时间 - long endTime = System.currentTimeMillis(); - // log.info("建立索引共耗时{}毫秒", endTime - startTime); - - try { - indexWriter.addDocument(document); - indexWriter.commit(); - } catch (IOException e) { - e.printStackTrace(); - } - } catch (Exception e) { - return; - } - } catch (Exception e){ - return; - } - } - - @Override - public void stop() { - super.stop(); - try { - indexWriter.close(); - } catch (IOException e) { - e.printStackTrace(); - } - } -} diff --git a/acs/nladmin-system/src/main/java/org/nl/modules/lucene/common/LuceneIndexWriter.java b/acs/nladmin-system/src/main/java/org/nl/modules/lucene/common/LuceneIndexWriter.java deleted file mode 100644 index c37e2f5..0000000 --- a/acs/nladmin-system/src/main/java/org/nl/modules/lucene/common/LuceneIndexWriter.java +++ /dev/null @@ -1,65 +0,0 @@ -package org.nl.modules.lucene.common; - -import cn.hutool.core.date.DateUtil; -import org.apache.lucene.index.CorruptIndexException; -import org.apache.lucene.index.IndexWriter; -import org.apache.lucene.index.IndexWriterConfig; -import org.apache.lucene.store.Directory; -import org.apache.lucene.store.FSDirectory; -import org.nl.modules.lucene.config.UrlConfig; -import org.wltea.analyzer.lucene.IKAnalyzer; - -import java.io.File; -import java.io.IOException; -import java.text.ParseException; -import java.text.SimpleDateFormat; -import java.util.Date; - -public class LuceneIndexWriter { -// private static IndexWriter indexWriter; -// -// static { -// try { -// Directory directory = FSDirectory.open(new File(UrlConfig.luceneUrl).toPath()); -// IndexWriterConfig config = new IndexWriterConfig(new IKAnalyzer()); -// indexWriter = new IndexWriter(directory, config); -// } catch (Exception e) { -// e.printStackTrace(); -// } -// /**当当前线程结束时，自动关闭IndexWriter，使用Runtime对象*/ -// Runtime.getRuntime().addShutdownHook(new Thread(){ -// @Override -// public void run() { -// try { -// closeIndexWriter(); -// } catch (Exception e) { -// e.printStackTrace(); -// } -// } -// }); -// } -// /**在线程结束时，自动关闭IndexWriter*/ -// public static IndexWriter getIndexWriter() { -// return indexWriter; -// } -// /**关闭IndexWriter -// * @throws IOException -// * @throws CorruptIndexException */ -// public static void closeIndexWriter() throws Exception { -// if(indexWriter != null) { -// indexWriter.close(); -// } -// } -// -// public static void main(String[] args) throws IOException { -// indexWriter.deleteAll(); -// } -// - public static String getDate(String timeString) throws ParseException { - SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSSX");//时间格式 - Date date = sdf.parse(timeString); - timeString = DateUtil.format(date, "yyyy-MM-dd HH:mm:ss.SSS");//格式化后的时间 - return timeString; - } - -} diff --git a/acs/nladmin-system/src/main/java/org/nl/modules/lucene/common/Searcher.java b/acs/nladmin-system/src/main/java/org/nl/modules/lucene/common/Searcher.java deleted file mode 100644 index 9cd0af2..0000000 --- a/acs/nladmin-system/src/main/java/org/nl/modules/lucene/common/Searcher.java +++ /dev/null @@ -1,144 +0,0 @@ -package org.nl.modules.lucene.common; - -import cn.hutool.core.date.DateTime; -import cn.hutool.core.date.DateUtil; -import com.alibaba.fastjson.JSONArray; -import com.alibaba.fastjson.JSONObject; -import lombok.extern.slf4j.Slf4j; -import org.apache.lucene.analysis.Analyzer; -import org.apache.lucene.document.Document; -import org.apache.lucene.index.DirectoryReader; -import org.apache.lucene.index.IndexReader; -import org.apache.lucene.index.Term; -import org.apache.lucene.queryparser.classic.QueryParser; -import org.apache.lucene.search.*; -import org.apache.lucene.store.Directory; -import org.apache.lucene.store.FSDirectory; -import org.apache.lucene.util.BytesRef; -import org.wltea.analyzer.lucene.IKAnalyzer; - -import java.nio.file.Paths; -import java.util.ArrayList; -import java.util.Calendar; -import java.util.List; -import java.util.Map; - -/** - * lucene查询器 - */ -@Slf4j -public class Searcher { - - public static Map search(String indexDir, String ext,Map whereJson) throws Exception { - //获取要查询的路径，也就是索引所在的位置 - Directory dir = FSDirectory.open(Paths.get(indexDir)); - IndexReader reader = DirectoryReader.open(dir); - //构建IndexSearcher - IndexSearcher searcher = new IndexSearcher(reader); - //标准分词器，会自动去掉空格啊，is a the等单词 - Analyzer analyzer = new IKAnalyzer(true); -// Analyzer analyzer = new StandardAnalyzer(); -// Analyzer analyzer = new IKAnalyzer(false); - //查询解析器 -// QueryParser queryParser = new QueryParser("fieldContent", analyzer); - - //记录索引开始时间 - long startTime = System.currentTimeMillis(); - // 实际上Lucene本身不支持分页。因此我们需要自己进行逻辑分页。我们要准备分页参数： - int pageSize = Integer.parseInt(whereJson.get("size").toString());// 每页条数 - int pageNum = Integer.parseInt(whereJson.get("page").toString());// 当前页码 - int start = pageNum * pageSize;// 当前页的起始条数 - int end = start + pageSize;// 当前页的结束条数（不能包含） - // 创建排序对象,需要排序字段SortField，参数：字段的名称、字段的类型、是否反转如果是false，升序。true降序 - Sort sort = new Sort(new SortField("logTime", SortField.Type.LONG,true)); - - TopDocs docs = null; - BooleanQuery.Builder booleanQueryBuilder = new BooleanQuery.Builder(); - //时间范围查询 - String startDate = (String) whereJson.get("begin_time"); - String endDate = (String) whereJson.get("end_time"); - Calendar calendar=Calendar.getInstance(); - calendar.set(1970, 0, 1); - if (startDate == null){ - startDate = DateUtil.format(calendar.getTime(),"yyyy-MM-dd HH:mm:ss.SSS"); - }else{ - startDate = LuceneIndexWriter.getDate(startDate); - } - if (endDate == null){ - endDate = DateUtil.format(new DateTime(),"yyyy-MM-dd HH:mm:ss.SSS"); - } else { - endDate = LuceneIndexWriter.getDate(endDate); - } - TermRangeQuery termRangeQuery = new TermRangeQuery("logTime", new BytesRef(startDate), new BytesRef(endDate), true, true); - booleanQueryBuilder.add(termRangeQuery,BooleanClause.Occur.MUST); - if (whereJson.get("device_code") != null){ - Query termQuery = new TermQuery(new Term("device_code", (String) whereJson.get("device_code"))); - booleanQueryBuilder.add(termQuery,BooleanClause.Occur.MUST); - } - if (whereJson.get("method") != null){ - Query termQuery = new TermQuery(new Term("method", (String) whereJson.get("method"))); - booleanQueryBuilder.add(termQuery,BooleanClause.Occur.MUST); - } - if (whereJson.get("status_code") != null){ - Query termQuery = new TermQuery(new Term("status_code", (String) whereJson.get("status_code"))); - booleanQueryBuilder.add(termQuery,BooleanClause.Occur.MUST); - } - if (whereJson.get("requestparam") != null){ - WildcardQuery query = new WildcardQuery(new Term("requestparam", "*"+(String) whereJson.get("requestparam")+"*")); - booleanQueryBuilder.add(query,BooleanClause.Occur.MUST); - } - if (whereJson.get("responseparam") != null){ - WildcardQuery query = new WildcardQuery(new Term("responseparam", "*"+(String) whereJson.get("responseparam")+"*")); - booleanQueryBuilder.add(query,BooleanClause.Occur.MUST); - } - if (whereJson.get("blurry") != null) { - WildcardQuery query = new WildcardQuery(new Term("fieldContent", "*"+(String) whereJson.get("blurry")+"*")); - booleanQueryBuilder.add(query, BooleanClause.Occur.MUST); - } - docs = searcher.search(booleanQueryBuilder.build(), end,sort); - //记录索引时间 - long endTime = System.currentTimeMillis(); - log.info("匹配{}共耗时{}毫秒",booleanQueryBuilder.build(),(endTime-startTime)); - log.info("查询到{}条日志文件", docs.totalHits.value); - List list = new ArrayList<>(); - ScoreDoc[] scoreDocs = docs.scoreDocs; - if (end > docs.totalHits.value) end = (int) docs.totalHits.value; - JSONArray array = new JSONArray(); - - for (int i = start; i < end; i++) { - ScoreDoc scoreDoc = scoreDocs[i]; - Document doc = reader.document(scoreDoc.doc); - JSONObject object = new JSONObject(); - object.put("content",doc.get("fieldContent")); - object.put("device_code",doc.get("device_code")); - object.put("logTime",doc.get("logTime")); - object.put("method",doc.get("method")); - object.put("status_code",doc.get("status_code")); - object.put("requestparam",doc.get("requestparam")); - object.put("responseparam",doc.get("responseparam")); - if(doc.get("fieldContent") != null) { - array.add(object); - } - } - for(Object logDto:array){ - log.info(logDto.toString()); - } - reader.close(); - JSONObject jo = new JSONObject(); - jo.put("content", array); - jo.put("totalElements", docs.totalHits.value); - return jo; - } - - public static void main(String[] args) { - String indexDir = "D:\\lucene\\index"; - //查询这个字符串 - String q = "07.832"; - Map whereJson = null; - try { - search(indexDir, q,whereJson); - } catch (Exception e) { - e.printStackTrace(); - } - } -} diff --git a/acs/nladmin-system/src/main/java/org/nl/modules/lucene/config/AsyncLuceneAppender.java b/acs/nladmin-system/src/main/java/org/nl/modules/lucene/config/AsyncLuceneAppender.java new file mode 100644 index 0000000..5f3565a --- /dev/null +++ b/acs/nladmin-system/src/main/java/org/nl/modules/lucene/config/AsyncLuceneAppender.java @@ -0,0 +1,43 @@ +package org.nl.modules.lucene.config; +/** + * @author ldjun + * @version 1.0 + * @date 2023年08月24日 13:00 + * @desc desc + */ + +import ch.qos.logback.classic.spi.ILoggingEvent; +import cn.hutool.core.util.IdUtil; +import com.yomahub.tlog.core.context.AspectLogContext; +import com.yomahub.tlog.core.enhance.logback.async.AspectLogbackAsyncAppender; +import org.apache.commons.lang3.StringUtils; +import org.slf4j.MDC; + +import java.util.Map; + +public class AsyncLuceneAppender extends AspectLogbackAsyncAppender { + + + @Override + protected void append(ILoggingEvent event) { + String traceId = AspectLogContext.getLogValue(); + if (StringUtils.isEmpty(traceId)){ + traceId = IdUtil.nanoId()+"@"; + AspectLogContext.putLogValue(traceId); + }else { + if (!traceId.contains("@")){ + AspectLogContext.putLogValue(traceId+"@"); + } + } + if (StringUtils.isNotEmpty(traceId)){ + MDC.put("traceId",traceId); + Map mdcPropertyMap = event.getMDCPropertyMap(); + if (mdcPropertyMap.getClass().getName().contains("SynchronizedMap")){ + mdcPropertyMap.put("traceId",traceId); + } + MDC.clear(); + } + super.append(event); + } + +} diff --git a/acs/nladmin-system/src/main/java/org/nl/modules/lucene/common/LogMessageConstant.java b/acs/nladmin-system/src/main/java/org/nl/modules/lucene/config/LogMessageConstant.java similarity index 56% rename from acs/nladmin-system/src/main/java/org/nl/modules/lucene/common/LogMessageConstant.java rename to acs/nladmin-system/src/main/java/org/nl/modules/lucene/config/LogMessageConstant.java index ca7f941..57ae26f 100644 --- a/acs/nladmin-system/src/main/java/org/nl/modules/lucene/common/LogMessageConstant.java +++ b/acs/nladmin-system/src/main/java/org/nl/modules/lucene/config/LogMessageConstant.java @@ -1,4 +1,4 @@ -package org.nl.modules.lucene.common; +package org.nl.modules.lucene.config; /** * @Author: lyd @@ -6,39 +6,75 @@ package org.nl.modules.lucene.common; * @Date: 2023/8/25 */ public class LogMessageConstant { - /** 级别 */ + /** + * + */ + public final static String SORT_NAME = "time"; + /** + * 级别 + */ public final static String FIELD_LEVEL = "level"; - /** 时间 */ + /** + * 时间 + */ public final static String FIELD_TIMESTAMP = "timestamp"; - /** 类的限定名 */ + /** + * 类的限定名 + */ public final static String FIELD_CLASS_NAME = "logger"; - /** 线程名 */ + /** + * 线程名 + */ public final static String FIELD_THREAD = "thread"; - /** 日志内容 */ + /** + * 日志内容 + */ public final static String FIELD_MESSAGE = "message"; public final static String FIELD_TRACEID = "tlogTraceId"; // 定义颜色值 - /** 文本颜色：黑色 */ + /** + * 文本颜色：黑色 + */ public final static String COLOR_BLACK = "\u001B[30m"; - /** 文本颜色：红色 */ + /** + * 文本颜色：红色 + */ public final static String COLOR_RED = "\u001B[31m"; - /** 文本颜色：绿色 */ + /** + * 文本颜色：绿色 + */ public final static String COLOR_GREEN = "\u001B[32m"; - /** 文本颜色：黄色 */ + /** + * 文本颜色：黄色 + */ public final static String COLOR_YELLOW = "\u001B[33m"; - /** 文本颜色：蓝色 */ + /** + * 文本颜色：蓝色 + */ public final static String COLOR_BLUE = "\u001B[34m"; - /** 文本颜色：品红色 */ + /** + * 文本颜色：品红色 + */ public final static String COLOR_MAGENTA = "\u001B[35m"; - /** 文本颜色：青色 */ + /** + * 文本颜色：青色 + */ public final static String COLOR_CYAN = "\u001B[36m"; - /** 文本颜色：白色 */ + /** + * 文本颜色：白色 + */ public final static String COLOR_WHITE = "\u001B[37m"; - /** 文本颜色重置 */ + /** + * 文本颜色重置 + */ public final static String COLOR_RESET = "\u001B[0m"; - /** 背景颜色：黄色 */ + /** + * 背景颜色：黄色 + */ public final static String BACKGROUND_YELLOW = "\u001B[43m"; + /** + * 索引路径 + */ + public final static String INDEX_DIR = "E:\\lucene\\index"; - /** 索引路径 */ - public final static String INDEX_DIR = "D:\\lucene\\index"; } diff --git a/acs/nladmin-system/src/main/java/org/nl/modules/lucene/config/LuceneAppender.java b/acs/nladmin-system/src/main/java/org/nl/modules/lucene/config/LuceneAppender.java new file mode 100644 index 0000000..943d13d --- /dev/null +++ b/acs/nladmin-system/src/main/java/org/nl/modules/lucene/config/LuceneAppender.java @@ -0,0 +1,133 @@ +package org.nl.modules.lucene.config; +/** + * @author ldjun + * @version 1.0 + * @date 2023年08月24日 13:00 + * @desc desc + */ + +import ch.qos.logback.classic.spi.ILoggingEvent; +import ch.qos.logback.core.AppenderBase; +import cn.hutool.core.date.DateTime; +import cn.hutool.core.date.DateUtil; +import cn.hutool.core.util.ObjectUtil; +import com.alibaba.fastjson.JSONObject; +import com.alibaba.ttl.TransmittableThreadLocal; +import org.apache.lucene.analysis.Analyzer; +import org.apache.lucene.document.Document; +import org.apache.lucene.document.Field; +import org.apache.lucene.document.NumericDocValuesField; +import org.apache.lucene.document.StringField; +import org.apache.lucene.index.IndexWriter; +import org.apache.lucene.index.IndexWriterConfig; +import org.apache.lucene.store.Directory; +import org.apache.lucene.store.FSDirectory; +import org.nl.modules.lucene.service.dto.LuceneLogDto; +import org.springframework.beans.factory.config.YamlPropertiesFactoryBean; +import org.springframework.core.io.ClassPathResource; +import org.springframework.core.io.Resource; +import org.wltea.analyzer.lucene.IKAnalyzer; + +import java.io.IOException; +import java.nio.file.Paths; +import java.util.List; +import java.util.Map; +import java.util.Properties; + +public class LuceneAppender extends AppenderBase { + + public static final TransmittableThreadLocal traceIdTL = new TransmittableThreadLocal(); + public LuceneProperties properties; + public static Directory index; + private List encoders; + public static IndexWriter indexWriter; + + + @Override + public void start() { + super.start(); + try { + init(); + } catch (Exception e) { + e.printStackTrace(); + } + } + + public static void init() throws IOException { + Resource resource = new ClassPathResource("config/application.yml"); + YamlPropertiesFactoryBean yamlPropertiesFactoryBean = new YamlPropertiesFactoryBean(); + yamlPropertiesFactoryBean.setResources(resource); + Properties properties = yamlPropertiesFactoryBean.getObject(); + // 获取配置值 + String luceneDir = properties.getProperty("lucene.index.path"); + System.out.println("---index地址----" + luceneDir); + index = FSDirectory.open(Paths.get(luceneDir)); + // 初始化 Lucene 索引 + Analyzer analyzer = new IKAnalyzer(); + IndexWriterConfig config = new IndexWriterConfig(analyzer); + indexWriter = new IndexWriter(index, config); + } + + + @Override + protected void append(ILoggingEvent event) { + String message = event.getFormattedMessage(); + String[] split = message.split("@"); + LuceneLogDto luceneLogDto = JSONObject.parseObject(split[1], LuceneLogDto.class); + Document document = new Document(); + try { + //向document对象中添加域。 + Map mdcPropertyMap = event.getMDCPropertyMap(); + String traceId = mdcPropertyMap.get("traceId"); + System.out.println("---追踪号---"+traceId); + if (ObjectUtil.isNotEmpty(traceId)) { + document.add(new StringField("trace_id", traceId, Field.Store.YES)); + } + if (ObjectUtil.isNotEmpty(luceneLogDto.getDevice_code())) { + document.add(new StringField("device_code", luceneLogDto.getDevice_code(), Field.Store.YES)); + } + if (ObjectUtil.isNotEmpty(luceneLogDto.getContent())) { + document.add(new StringField("fieldContent", luceneLogDto.getContent(), Field.Store.YES)); + } + if (ObjectUtil.isNotEmpty(luceneLogDto.getMethod())) { + document.add(new StringField("method", luceneLogDto.getMethod(), Field.Store.YES)); + } + if (ObjectUtil.isNotEmpty(luceneLogDto.getStatus_code())) { + document.add(new StringField("status_code", luceneLogDto.getStatus_code(), Field.Store.YES)); + } + if (ObjectUtil.isNotEmpty(luceneLogDto.getRequestparam())) { + document.add(new StringField("requestparam", luceneLogDto.getRequestparam(), Field.Store.YES)); + } + if (ObjectUtil.isNotEmpty(luceneLogDto.getResponseparam())) { + document.add(new StringField("responseparam", luceneLogDto.getResponseparam(), Field.Store.YES)); + } + document.add(new StringField("logType", luceneLogDto.getLogType(), Field.Store.YES)); + document.add(new StringField("logTime", DateUtil.format(new DateTime(), "yyyy-MM-dd HH:mm:ss.SSS"), Field.Store.YES)); + document.add(new NumericDocValuesField("logTime",System.currentTimeMillis()));//排序 + + try { + indexWriter.addDocument(document); + indexWriter.commit(); + } catch (IOException e) { + e.printStackTrace(); + } + } catch (Exception e) { + return; + } + } + + @Override + public void stop() { + super.stop(); + try { + indexWriter.close(); + } catch (IOException e) { + e.printStackTrace(); + } + } + + public void setProperties(LuceneProperties properties) { + this.properties = properties; + + } +} diff --git a/acs/nladmin-system/src/main/java/org/nl/modules/lucene/config/LuceneProperties.java b/acs/nladmin-system/src/main/java/org/nl/modules/lucene/config/LuceneProperties.java new file mode 100644 index 0000000..5df7ade --- /dev/null +++ b/acs/nladmin-system/src/main/java/org/nl/modules/lucene/config/LuceneProperties.java @@ -0,0 +1,23 @@ +package org.nl.modules.lucene.config; + + +import java.util.ArrayList; +import java.util.List; + +public class LuceneProperties { + + private List properties; + + public LuceneProperties() { + this.properties = new ArrayList(); + } + + public List getProperties() { + return properties; + } + + public void addProperty(Property property) { + properties.add(property); + } + +} diff --git a/acs/nladmin-system/src/main/java/org/nl/modules/lucene/config/LucenePropertyAndEncoder.java b/acs/nladmin-system/src/main/java/org/nl/modules/lucene/config/LucenePropertyAndEncoder.java new file mode 100644 index 0000000..3b07504 --- /dev/null +++ b/acs/nladmin-system/src/main/java/org/nl/modules/lucene/config/LucenePropertyAndEncoder.java @@ -0,0 +1,38 @@ +package org.nl.modules.lucene.config; + +import ch.qos.logback.classic.PatternLayout; +import ch.qos.logback.classic.spi.ILoggingEvent; +import ch.qos.logback.core.Context; +import ch.qos.logback.core.pattern.PatternLayoutBase; + +/* + * @author ZZQ + * @Date 2023/12/22 18:11 + */ +public class LucenePropertyAndEncoder { + + private Property property; + + private PatternLayoutBase layout = new PatternLayout(); + + public LucenePropertyAndEncoder(Property property, Context context) { + this.property = property; + this.layout.setContext(context); + this.layout.setPattern(String.valueOf(property.getValue())); + this.layout.setPostCompileProcessor(null); + this.layout.start(); + } + + public String encode(ILoggingEvent event) { + return layout.doLayout(event); + } + + public String getName() { + return property.getName(); + } + + public boolean allowEmpty() { + return property.isAllowEmpty(); + } +} + diff --git a/acs/nladmin-system/src/main/java/org/nl/modules/lucene/config/Property.java b/acs/nladmin-system/src/main/java/org/nl/modules/lucene/config/Property.java new file mode 100644 index 0000000..643075c --- /dev/null +++ b/acs/nladmin-system/src/main/java/org/nl/modules/lucene/config/Property.java @@ -0,0 +1,44 @@ +package org.nl.modules.lucene.config; + +/* + * @author ZZQ + * @Date 2023/12/26 15:30 + */ +public class Property { + private String name; + private String value; + private boolean allowEmpty; + + public Property() { + } + + public Property(String name, String value, boolean allowEmpty) { + this.name = name; + this.value = value; + this.allowEmpty = allowEmpty; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public String getValue() { + return value; + } + + public void setValue(String value) { + this.value = value; + } + + public boolean isAllowEmpty() { + return allowEmpty; + } + + public void setAllowEmpty(boolean allowEmpty) { + this.allowEmpty = allowEmpty; + } +} diff --git a/acs/nladmin-system/src/main/java/org/nl/modules/lucene/config/StaticConfig.java b/acs/nladmin-system/src/main/java/org/nl/modules/lucene/config/StaticConfig.java deleted file mode 100644 index d053859..0000000 --- a/acs/nladmin-system/src/main/java/org/nl/modules/lucene/config/StaticConfig.java +++ /dev/null @@ -1,21 +0,0 @@ -package org.nl.modules.lucene.config; - -import org.springframework.beans.factory.annotation.Value; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; - -/** - * @deprecated 设置静态参数初始化 - */ -@Configuration -public class StaticConfig { - //日志索引目录 - @Value("${lucene.index.path}") - private String luceneDir; - - @Bean - public int initStatic() { - UrlConfig.setLuceneUrl(luceneDir); - return 0; - } -} diff --git a/acs/nladmin-system/src/main/java/org/nl/modules/lucene/config/UrlConfig.java b/acs/nladmin-system/src/main/java/org/nl/modules/lucene/config/UrlConfig.java deleted file mode 100644 index d48ca3c..0000000 --- a/acs/nladmin-system/src/main/java/org/nl/modules/lucene/config/UrlConfig.java +++ /dev/null @@ -1,13 +0,0 @@ -package org.nl.modules.lucene.config; - -public class UrlConfig { - public static String luceneUrl; - - public static String getLuceneUrl() { - return luceneUrl; - } - - public static void setLuceneUrl(String luceneUrl) { - UrlConfig.luceneUrl = luceneUrl; - } -} diff --git a/acs/nladmin-system/src/main/java/org/nl/modules/lucene/rest/LuceneController.java b/acs/nladmin-system/src/main/java/org/nl/modules/lucene/rest/LuceneController.java index 6d351c3..c32761d 100644 --- a/acs/nladmin-system/src/main/java/org/nl/modules/lucene/rest/LuceneController.java +++ b/acs/nladmin-system/src/main/java/org/nl/modules/lucene/rest/LuceneController.java @@ -26,11 +26,6 @@ public class LuceneController { private final LuceneService luceneService; - @GetMapping("/labels/values") - @ApiOperation("获取标签") - public ResponseEntity labelsValues() { - return new ResponseEntity<>(luceneService.getLabelsValues(), HttpStatus.OK); - } @GetMapping("/getAll") diff --git a/acs/nladmin-system/src/main/java/org/nl/modules/lucene/service/LuceneExecuteLogService.java b/acs/nladmin-system/src/main/java/org/nl/modules/lucene/service/LuceneExecuteLogService.java index b740d3c..e4b0db1 100644 --- a/acs/nladmin-system/src/main/java/org/nl/modules/lucene/service/LuceneExecuteLogService.java +++ b/acs/nladmin-system/src/main/java/org/nl/modules/lucene/service/LuceneExecuteLogService.java @@ -3,8 +3,6 @@ package org.nl.modules.lucene.service; import org.nl.modules.lucene.service.dto.LuceneLogDto; -import java.io.IOException; - public interface LuceneExecuteLogService { /** * 设备光电变化实时光电信号 diff --git a/acs/nladmin-system/src/main/java/org/nl/modules/lucene/service/LuceneService.java b/acs/nladmin-system/src/main/java/org/nl/modules/lucene/service/LuceneService.java index 18156d3..480ae5e 100644 --- a/acs/nladmin-system/src/main/java/org/nl/modules/lucene/service/LuceneService.java +++ b/acs/nladmin-system/src/main/java/org/nl/modules/lucene/service/LuceneService.java @@ -1,17 +1,11 @@ package org.nl.modules.lucene.service; -import com.alibaba.fastjson.JSONArray; import org.springframework.data.domain.Pageable; import java.util.Map; public interface LuceneService { - /** - * 获取labels和values树 - * @return - */ - JSONArray getLabelsValues(); /** * 获取数据分页 diff --git a/acs/nladmin-system/src/main/java/org/nl/modules/lucene/service/impl/LuceneExecuteLogServiceImpl.java b/acs/nladmin-system/src/main/java/org/nl/modules/lucene/service/impl/LuceneExecuteLogServiceImpl.java index 70619e6..17ceffd 100644 --- a/acs/nladmin-system/src/main/java/org/nl/modules/lucene/service/impl/LuceneExecuteLogServiceImpl.java +++ b/acs/nladmin-system/src/main/java/org/nl/modules/lucene/service/impl/LuceneExecuteLogServiceImpl.java @@ -1,26 +1,15 @@ package org.nl.modules.lucene.service.impl; -import cn.hutool.core.date.DateTime; import cn.hutool.core.date.DateUtil; -import cn.hutool.core.util.ObjectUtil; import com.alibaba.fastjson.JSON; import lombok.RequiredArgsConstructor; -import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; -import org.apache.lucene.document.Document; -import org.apache.lucene.document.Field; -import org.apache.lucene.document.StringField; -import org.apache.lucene.document.TextField; -import org.apache.lucene.index.IndexWriter; -import org.nl.modules.lucene.common.LuceneIndexWriter; import org.nl.modules.lucene.enums.LogTypeEnum; import org.nl.modules.lucene.service.LuceneExecuteLogService; import org.nl.modules.lucene.service.dto.LuceneLogDto; import org.slf4j.MDC; import org.springframework.stereotype.Service; -import java.io.IOException; - /** * @author jlm * @description 服务实现 @@ -45,50 +34,9 @@ public class LuceneExecuteLogServiceImpl implements LuceneExecuteLogService { @Override public void interfaceExecuteLog(LuceneLogDto luceneLogDto) { luceneLogDto.setLogType(LogTypeEnum.INTERFACE_LOG.getDesc()); -// addIndex(luceneLogDto); log.info("{}", JSON.toJSONString(luceneLogDto)); } - private void addIndex(LuceneLogDto luceneLogDto) { -// IndexWriter indexWriter = LuceneIndexWriter.getIndexWriter(); -// //创建一个Document对象 -// Document document = new Document(); -// try { -// //记录索引开始时间 -// long startTime = System.currentTimeMillis(); -// //向document对象中添加域。 -// if (ObjectUtil.isNotEmpty(luceneLogDto.getDevice_code())) { -// document.add(new StringField("device_code", luceneLogDto.getDevice_code(), Field.Store.YES)); -//// document.add(new TextField("device_code", luceneLogDto.getDevice_code(), Field.Store.YES)); -// } -// if (ObjectUtil.isNotEmpty(luceneLogDto.getContent())) { -// document.add(new StringField("fieldContent", luceneLogDto.getContent(), Field.Store.YES)); -// } -// if (ObjectUtil.isNotEmpty(luceneLogDto.getMethod())) { -// document.add(new StringField("method", luceneLogDto.getMethod(), Field.Store.YES)); -// } -// if (ObjectUtil.isNotEmpty(luceneLogDto.getStatus_code())) { -// document.add(new StringField("status_code", luceneLogDto.getStatus_code(), Field.Store.YES)); -// } -// if (ObjectUtil.isNotEmpty(luceneLogDto.getRequestparam())) { -// document.add(new StringField("requestparam", luceneLogDto.getRequestparam(), Field.Store.YES)); -// } -// if (ObjectUtil.isNotEmpty(luceneLogDto.getResponseparam())) { -// document.add(new StringField("responseparam", luceneLogDto.getResponseparam(), Field.Store.YES)); -// } -// document.add(new StringField("logType", luceneLogDto.getLogType(), Field.Store.YES)); -// document.add(new StringField("logTime", DateUtil.format(new DateTime(), "yyyy-MM-dd HH:mm:ss.SSS"), Field.Store.YES)); -// indexWriter.addDocument(document); -// //记录索引结束时间 -// long endTime = System.currentTimeMillis(); -// log.info("建立索引共耗时{}毫秒", endTime - startTime); -// indexWriter.commit(); -// MDC.put("DEVICECODE", luceneLogDto.getDevice_code()); -// } catch (Exception e) { -// log.error(e.getMessage(), e); -// } - } - @Override public void extLog(String name, String message) { try { diff --git a/acs/nladmin-system/src/main/java/org/nl/modules/lucene/service/impl/LuceneServiceImpl.java b/acs/nladmin-system/src/main/java/org/nl/modules/lucene/service/impl/LuceneServiceImpl.java index cfce669..6c10ee4 100644 --- a/acs/nladmin-system/src/main/java/org/nl/modules/lucene/service/impl/LuceneServiceImpl.java +++ b/acs/nladmin-system/src/main/java/org/nl/modules/lucene/service/impl/LuceneServiceImpl.java @@ -1,18 +1,24 @@ package org.nl.modules.lucene.service.impl; -import cn.hutool.core.util.CharsetUtil; -import cn.hutool.http.HttpUtil; -import com.alibaba.fastjson.JSONArray; +import cn.hutool.core.date.DateTime; +import cn.hutool.core.date.DateUtil; import com.alibaba.fastjson.JSONObject; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; -import org.nl.modules.lucene.common.Searcher; +import org.apache.lucene.document.Document; +import org.apache.lucene.index.DirectoryReader; +import org.apache.lucene.index.Term; +import org.apache.lucene.search.*; +import org.apache.lucene.store.FSDirectory; +import org.apache.lucene.util.BytesRef; import org.nl.modules.lucene.service.LuceneService; -import org.springframework.beans.factory.annotation.Value; import org.springframework.data.domain.Pageable; import org.springframework.stereotype.Service; -import java.util.Map; +import java.nio.file.Paths; +import java.text.ParseException; +import java.text.SimpleDateFormat; +import java.util.*; @Service @@ -20,66 +26,102 @@ import java.util.Map; @Slf4j public class LuceneServiceImpl implements LuceneService { - @Value("${loki.url}") - private String lokiUrl; - - @Value("${loki.systemName}") - private String systemName; - //日志索引目录 - @Value("${lucene.index.path}") - private String luceneUrl; - /** - * 获取labels和values树 - * - * @return - */ - @Override - public JSONArray getLabelsValues() { - JSONArray result = new JSONArray(); - // 获取所有标签 - String labelString = HttpUtil.get(lokiUrl + "/labels", CharsetUtil.CHARSET_UTF_8); - JSONObject parse = (JSONObject) JSONObject.parse(labelString); - JSONArray labels = parse.getJSONArray("data"); - for (int i=0; i getAll(Map whereJson, Pageable page) { - JSONObject jo = new JSONObject(); + //获取要查询的路径，也就是索引所在的位置 try { - JSONObject jsonObject = (JSONObject) Searcher.search(luceneUrl, "", whereJson); - JSONArray array = jsonObject.getJSONArray("content"); - int totalElements = Integer.parseInt(jsonObject.get("totalElements").toString()); - jo.put("content", array); - jo.put("totalElements", totalElements); + FSDirectory directory = FSDirectory.open(Paths.get("C:\\acs\\lucene\\index")); + DirectoryReader open = DirectoryReader.open(directory); + IndexSearcher searcher = new IndexSearcher(open); + // 实际上Lucene本身不支持分页。因此我们需要自己进行逻辑分页。我们要准备分页参数： + int pageSize = Integer.parseInt(whereJson.get("size").toString());// 每页条数 + int pageNum = Integer.parseInt(whereJson.get("page").toString());// 当前页码 + + BooleanQuery.Builder booleanQueryBuilder = new BooleanQuery.Builder(); + //时间范围查询 + String startDate = (String) whereJson.get("begin_time"); + String endDate = (String) whereJson.get("end_time"); + + if (startDate == null){ + Calendar calendar=Calendar.getInstance(); + calendar.set(1970, 0, 1); + startDate = DateUtil.format(calendar.getTime(),"yyyy-MM-dd HH:mm:ss.SSS"); + }else{ + startDate = getDate(startDate); + } + if (endDate == null){ + endDate = DateUtil.format(new DateTime(),"yyyy-MM-dd HH:mm:ss.SSS"); + } else { + endDate = getDate(endDate); + } + TermRangeQuery termRangeQuery = new TermRangeQuery("logTime", new BytesRef(startDate), new BytesRef(endDate), true, true); + booleanQueryBuilder.add(termRangeQuery,BooleanClause.Occur.MUST); + if (whereJson.get("device_code") != null){ + Query termQuery = new TermQuery(new Term("device_code", (String) whereJson.get("device_code"))); + booleanQueryBuilder.add(termQuery,BooleanClause.Occur.MUST); + } + if (whereJson.get("method") != null){ + Query termQuery = new TermQuery(new Term("method", (String) whereJson.get("method"))); + booleanQueryBuilder.add(termQuery,BooleanClause.Occur.MUST); + } + if (whereJson.get("status_code") != null){ + Query termQuery = new TermQuery(new Term("status_code", (String) whereJson.get("status_code"))); + booleanQueryBuilder.add(termQuery,BooleanClause.Occur.MUST); + } + if (whereJson.get("requestparam") != null){ + WildcardQuery query = new WildcardQuery(new Term("requestparam", "*"+(String) whereJson.get("requestparam")+"*")); + booleanQueryBuilder.add(query,BooleanClause.Occur.MUST); + } + if (whereJson.get("responseparam") != null){ + WildcardQuery query = new WildcardQuery(new Term("responseparam", "*"+(String) whereJson.get("responseparam")+"*")); + booleanQueryBuilder.add(query,BooleanClause.Occur.MUST); + } + if (whereJson.get("blurry") != null) { + WildcardQuery query = new WildcardQuery(new Term("fieldContent", "*"+(String) whereJson.get("blurry")+"*")); + booleanQueryBuilder.add(query, BooleanClause.Occur.MUST); + } + + TopFieldCollector collector = TopFieldCollector.create(new Sort(new SortField("logTime", SortField.Type.LONG,true)), 20000, 0); + searcher.search(booleanQueryBuilder.build(), collector); + TopDocs topDocs = collector.topDocs(pageNum*pageSize, pageSize); + int totalSize = collector.getTotalHits(); + ScoreDoc[] scoreDocs = topDocs.scoreDocs; + + List list = new ArrayList<>(); + for (ScoreDoc scoreDoc : scoreDocs) { + Document doc = open.document(scoreDoc.doc); + JSONObject object = new JSONObject(); + object.put("content",doc.get("fieldContent")); + object.put("device_code",doc.get("device_code")); + object.put("logTime",doc.get("logTime")); + object.put("method",doc.get("method")); + object.put("status_code",doc.get("status_code")); + object.put("requestparam",doc.get("requestparam")); + object.put("responseparam",doc.get("responseparam")); + if(doc.get("fieldContent") != null) { + list.add(object); + } + } + open.close(); + directory.close(); + JSONObject jo = new JSONObject(); + jo.put("content", list); + jo.put("totalElements", totalSize); + return jo; } catch (Exception e) { log.error("索引查询为空", e); throw new NullPointerException("索引查询为空"); } + } - return jo; + public static String getDate(String timeString) throws ParseException { + SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSSX");//时间格式 + Date date = sdf.parse(timeString); + timeString = DateUtil.format(date, "yyyy-MM-dd HH:mm:ss.SSS");//格式化后的时间 + return timeString; } } diff --git a/acs/nladmin-system/src/main/resources/config/application-dev.yml b/acs/nladmin-system/src/main/resources/config/application-dev.yml index 8bf3b93..7b1aff5 100644 --- a/acs/nladmin-system/src/main/resources/config/application-dev.yml +++ b/acs/nladmin-system/src/main/resources/config/application-dev.yml @@ -6,7 +6,7 @@ spring: druid: db-type: com.alibaba.druid.pool.DruidDataSource driverClassName: net.sf.log4jdbc.sql.jdbcapi.DriverSpy - url: jdbc:log4jdbc:mysql://${DB_HOST:127.0.0.1}:${DB_PORT:3306}/${DB_NAME:nl4_acs}?serverTimezone=Asia/Shanghai&characterEncoding=utf8&useSSL=false&useOldAliasMetadataBehavior=true&allowPublicKeyRetrieval=true + url: jdbc:log4jdbc:mysql://${DB_HOST:127.0.0.1}:${DB_PORT:3306}/${DB_NAME:shangdianke_acs}?serverTimezone=Asia/Shanghai&characterEncoding=utf8&useSSL=false&useOldAliasMetadataBehavior=true&allowPublicKeyRetrieval=true username: ${DB_USER:root} password: ${DB_PWD:password} # 初始连接数 @@ -126,11 +126,8 @@ file: avatarMaxSize: 5 logging: file: - path: /Users/onepiece/myFile/acs_logs + path: C:\acs config: classpath:logback-spring.xml -lucene: - index: - path: /Users/onepiece/myFile/lucene/index # Sa-Token配置 sa-token: @@ -152,6 +149,3 @@ sa-token: # token 前缀 token-prefix: Bearer -loki: - url: http://10.211.55.3:3100/loki/api/v1 - systemName: acs diff --git a/acs/nladmin-system/src/main/resources/config/application-dev2.yml b/acs/nladmin-system/src/main/resources/config/application-dev2.yml new file mode 100644 index 0000000..c0bc6f1 --- /dev/null +++ b/acs/nladmin-system/src/main/resources/config/application-dev2.yml @@ -0,0 +1,151 @@ +server: + port: 8010 +#配置数据源 +spring: + datasource: + druid: + db-type: com.alibaba.druid.pool.DruidDataSource + driverClassName: net.sf.log4jdbc.sql.jdbcapi.DriverSpy + url: jdbc:log4jdbc:mysql://${DB_HOST:127.0.0.1}:${DB_PORT:3306}/${DB_NAME:shangdianke_acs}?serverTimezone=Asia/Shanghai&characterEncoding=utf8&useSSL=false&useOldAliasMetadataBehavior=true&allowPublicKeyRetrieval=true + username: ${DB_USER:root} + password: ${DB_PWD:password} + # 初始连接数 + initial-size: 5 + # 最小连接数 + min-idle: 15 + # 最大连接数 + max-active: 30 + # 超时时间(以秒数为单位) + remove-abandoned-timeout: 180 + # 获取连接超时时间 + max-wait: 3000 + # 连接有效性检测时间 + time-between-eviction-runs-millis: 60000 + # 连接在池中最小生存的时间 + min-evictable-idle-time-millis: 300000 + # 连接在池中最大生存的时间 + max-evictable-idle-time-millis: 900000 + # 指明连接是否被空闲连接回收器(如果有)进行检验.如果检测失败,则连接将被从池中去除 + test-while-idle: true + # 指明是否在从池中取出连接前进行检验,如果检验失败, 则从池中去除连接并尝试取出另一个 + test-on-borrow: true + # 是否在归还到池中前进行检验 + test-on-return: false + # 检测连接是否有效 + validation-query: select 1 + # 配置监控统计 + webStatFilter: + enabled: true + stat-view-servlet: + enabled: true + url-pattern: /druid/* + reset-enable: false + filter: + stat: + enabled: true + # 记录慢SQL + log-slow-sql: true + slow-sql-millis: 1000 + merge-sql: true + wall: + config: + multi-statement-allow: true + redis: + #数据库索引 + database: ${REDIS_DB:15} + host: ${REDIS_HOST:127.0.0.1} + port: ${REDIS_PORT:6379} + password: ${REDIS_PWD:} + +# 登录相关配置 +login: + # 登录缓存 + cache-enable: true + # 是否限制单用户登录 + single-login: false + # 验证码 + login-code: + # 验证码类型配置 查看 LoginProperties 类 + code-type: arithmetic + # 登录图形验证码有效时间/分钟 + expiration: 2 + # 验证码高度 + width: 111 + # 验证码宽度 + heigth: 36 + # 内容长度 + length: 2 + # 字体名称，为空则使用默认字体 + font-name: + # 字体大小 + font-size: 25 + +#jwt +jwt: + header: Authorization + # 令牌前缀 + token-start-with: Bearer + # 必须使用最少88位的Base64对该令牌进行编码 + base64-secret: ZmQ0ZGI5NjQ0MDQwY2I4MjMxY2Y3ZmI3MjdhN2ZmMjNhODViOTg1ZGE0NTBjMGM4NDA5NzYxMjdjOWMwYWRmZTBlZjlhNGY3ZTg4Y2U3YTE1ODVkZDU5Y2Y3OGYwZWE1NzUzNWQ2YjFjZDc0NGMxZWU2MmQ3MjY1NzJmNTE0MzI= + # 令牌过期时间 此处单位/毫秒 ，默认4小时，可在此网站生成 https://www.convertworld.com/zh-hans/time/milliseconds.html + token-validity-in-seconds: 14400000 + # 在线用户key + online-key: online-token- + # 验证码 + code-key: code-key- + # token 续期检查时间范围（默认30分钟，单位毫秒），在token即将过期的一段时间内用户操作了，则给用户的token续期 + detect: 1800000 + # 续期时间范围，默认1小时，单位毫秒 + renew: 3600000 + +#是否允许生成代码，生产环境设置为false +generator: + enabled: true + +#是否开启 swagger-ui +swagger: + enabled: true + +# IP 本地解析 +ip: + local-parsing: true + +# 文件存储路径 +file: + mac: + path: ~/file/ + avatar: ~/avatar/ + linux: + path: /home/eladmin/file/ + avatar: /home/eladmin/avatar/ + windows: + path: C:\eladmin\file\ + avatar: C:\eladmin\avatar\ + # 文件大小 /M + maxSize: 100 + avatarMaxSize: 5 +logging: + file: + path: /Users/onepiece/myFile/acs + config: classpath:logback-spring.xml + +# Sa-Token配置 +sa-token: + # token 名称 (同时也是cookie名称) + token-name: Authorization + # token 有效期，单位s 默认30天, -1代表永不过期 + timeout: 2592000 + # token 临时有效期 (指定时间内无操作就视为token过期) 单位: 秒 + activity-timeout: -1 + # 是否允许同一账号并发登录 (为true时允许一起登录, 为false时新登录挤掉旧登录) + is-concurrent: true + # 在多人登录同一账号时，是否共用一个token (为true时所有登录共用一个token, 为false时每次登录新建一个token) + is-share: false + # token风格 + token-style: random-128 + # 是否输出操作日志 + is-log: false + jwt-secret-key: opsjajisdnnca0sdkksdfaaasdfwwq + # token 前缀 + token-prefix: Bearer + diff --git a/acs/nladmin-system/src/main/resources/config/application-prod.yml b/acs/nladmin-system/src/main/resources/config/application-prod.yml index d6654c8..4fcaa39 100644 --- a/acs/nladmin-system/src/main/resources/config/application-prod.yml +++ b/acs/nladmin-system/src/main/resources/config/application-prod.yml @@ -152,6 +152,3 @@ sa-token: # token 前缀 token-prefix: Bearer -loki: - url: http://localhost:3100/loki/api/v1 - systemName: acs diff --git a/acs/nladmin-system/src/main/resources/config/application.yml b/acs/nladmin-system/src/main/resources/config/application.yml index 43cf9ea..0602e94 100644 --- a/acs/nladmin-system/src/main/resources/config/application.yml +++ b/acs/nladmin-system/src/main/resources/config/application.yml @@ -2,7 +2,7 @@ spring: freemarker: check-template-location: false profiles: - active: prod + active: dev2 jackson: time-zone: GMT+8 data: @@ -78,3 +78,7 @@ security: - /api/localStorage/pictures # 参数 - /api/param/getValueByCode + +lucene: + index: + path: C:\acs\lucene\index \ No newline at end of file diff --git a/acs/nladmin-system/src/main/resources/log/AcsToWms.xml b/acs/nladmin-system/src/main/resources/log/AcsToWms.xml index df3efa3..324ae5c 100644 --- a/acs/nladmin-system/src/main/resources/log/AcsToWms.xml +++ b/acs/nladmin-system/src/main/resources/log/AcsToWms.xml @@ -3,10 +3,10 @@ - + - ${LOG_HOME}/ACS请求WMS/%d{yyyy-MM-dd}.%i.log + ${LOG_HOME}/ACS请求LMS/%d{yyyy-MM-dd}.%i.log 15 @@ -22,12 +22,12 @@ - - - - - + + + + 512 + + + diff --git a/acs/nladmin-system/src/main/resources/log/AgvNdcOneDeviceDriver.xml b/acs/nladmin-system/src/main/resources/log/AgvNdcOneDeviceDriver.xml deleted file mode 100644 index f4629cd..0000000 --- a/acs/nladmin-system/src/main/resources/log/AgvNdcOneDeviceDriver.xml +++ /dev/null @@ -1,33 +0,0 @@ - - - - - - - - - ${LOG_HOME}/AGV驱动与NDC交互/%d{yyyy-MM-dd}.%i.log - - 15 - - 200MB - - 2GB - - - - %d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %-5level %logger{50} - %msg%n - ${log.charset} - - - - - - - - - - - diff --git a/acs/nladmin-system/src/main/resources/log/AutoCreateInst.xml b/acs/nladmin-system/src/main/resources/log/AutoCreateInst.xml index 1ac8de4..68caa7d 100644 --- a/acs/nladmin-system/src/main/resources/log/AutoCreateInst.xml +++ b/acs/nladmin-system/src/main/resources/log/AutoCreateInst.xml @@ -22,12 +22,12 @@ - - - - - + + + + 512 + + + diff --git a/acs/nladmin-system/src/main/resources/log/Lucene.xml b/acs/nladmin-system/src/main/resources/log/Lucene.xml deleted file mode 100644 index f00bdbd..0000000 --- a/acs/nladmin-system/src/main/resources/log/Lucene.xml +++ /dev/null @@ -1,41 +0,0 @@ - - - - - - - - - - ${LOG_HOME}/lucene/${DEVICECODE}/%d{yyyy-MM-dd}.%i.log - - 15 - - 200MB - - 2GB - - - - %d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %-5level %logger{50} - %msg%n - ${log.charset} - - - - - - - - - 512 - - - - - - - - - diff --git a/acs/nladmin-system/src/main/resources/log/NDCAgvServiceImpl.xml b/acs/nladmin-system/src/main/resources/log/NDCAgvServiceImpl.xml deleted file mode 100644 index a10d776..0000000 --- a/acs/nladmin-system/src/main/resources/log/NDCAgvServiceImpl.xml +++ /dev/null @@ -1,33 +0,0 @@ - - - - - - - - - ${LOG_HOME}/NDCAGV服务/%d{yyyy-MM-dd}.%i.log - - 15 - - 200MB - - 2GB - - - - %d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %-5level %logger{50} - %msg%n - ${log.charset} - - - - - - - - - - - diff --git a/acs/nladmin-system/src/main/resources/log/NDCSocketConnectionAutoRun.xml b/acs/nladmin-system/src/main/resources/log/NDCSocketConnectionAutoRun.xml deleted file mode 100644 index 53bcb88..0000000 --- a/acs/nladmin-system/src/main/resources/log/NDCSocketConnectionAutoRun.xml +++ /dev/null @@ -1,33 +0,0 @@ - - - - - - - - - ${LOG_HOME}/NDC交互日志/%d{yyyy-MM-dd}.%i.log - - 15 - - 200MB - - 2GB - - - - %d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %-5level %logger{50} - %msg%n - ${log.charset} - - - - - - - - - - - diff --git a/acs/nladmin-system/src/main/resources/log/WmsToAcs.xml b/acs/nladmin-system/src/main/resources/log/WmsToAcs.xml index e5d9847..b989286 100644 --- a/acs/nladmin-system/src/main/resources/log/WmsToAcs.xml +++ b/acs/nladmin-system/src/main/resources/log/WmsToAcs.xml @@ -3,7 +3,7 @@ - + ${LOG_HOME}/WMS请求ACS/%d{yyyy-MM-dd}.%i.log @@ -22,12 +22,12 @@ - - - - - + + + + 512 + + + diff --git a/acs/nladmin-system/src/main/resources/logback-spring.xml b/acs/nladmin-system/src/main/resources/logback-spring.xml index a5017fc..00383d6 100644 --- a/acs/nladmin-system/src/main/resources/logback-spring.xml +++ b/acs/nladmin-system/src/main/resources/logback-spring.xml @@ -14,9 +14,9 @@ https://juejin.cn/post/6844903775631572999 - + - + @@ -24,11 +24,6 @@ https://juejin.cn/post/6844903775631572999 - - - - - ${log.pattern} @@ -60,75 +55,48 @@ https://juejin.cn/post/6844903775631572999 200 - - - - + + + 512 - - - 1000 - - ${LOKI_URL}/push - - - - - ${log.pattern} - - true - - - - - - - + - - - - + + - - - + + - - - + + - - - - - + + - - + + - - + + - - + + - - + + - - + + + @@ -137,54 +105,41 @@ https://juejin.cn/post/6844903775631572999 - - - + - + - - - + - - + - - - + - - + - - + - - + - - + - - + -