From 523462ddd1d019a85d5217c4e7a657262ccddd6d Mon Sep 17 00:00:00 2001 From: jiaoliming Date: Tue, 25 Apr 2023 19:14:36 +0800 Subject: [PATCH] =?UTF-8?q?add:=20=E6=96=B0=E5=A2=9Elucene=E6=97=A5?= =?UTF-8?q?=E5=BF=97=E6=A3=80=E7=B4=A2?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- acs/nladmin-system/pom.xml | 35 ++++ .../ItemProtocol.java | 2 +- .../ItemProtocol.java | 2 +- .../standard_inspect_site/ItemProtocol.java | 2 +- .../nl/modules/lucence/common/Indexer.java | 181 ++++++++++++++++++ .../lucence/common/LuceneIndexWriter.java | 68 +++++++ .../nl/modules/lucence/common/Searcher.java | 132 +++++++++++++ .../modules/lucence/config/StaticConfig.java | 22 +++ .../nl/modules/lucence/config/UrlConfig.java | 13 ++ .../nl/modules/lucence/enums/LogTypeEnum.java | 15 ++ .../lucence/rest/LuceneController.java | 39 ++++ .../service/LuceneExecuteLogService.java | 41 ++++ .../lucence/service/LuceneService.java | 27 +++ .../lucence/service/dto/LuceneLogDto.java | 86 +++++++++ .../impl/LuceneExecuteLogServiceImpl.java | 104 ++++++++++ .../service/impl/LuceneServiceImpl.java | 85 ++++++++ .../main/resources/config/application-dev.yml | 7 +- .../main/resources/log/XgAgvDeviceDriver.xml | 34 ++++ .../src/main/resources/logback-spring.xml | 4 +- .../src/views/monitor/lucene/api/lucene.js | 18 ++ .../src/views/monitor/lucene/index.vue | 42 ++-- .../src/views/monitor/lucene/search.vue | 45 ++++- 22 files changed, 972 insertions(+), 32 deletions(-) create mode 100644 acs/nladmin-system/src/main/java/org/nl/modules/lucence/common/Indexer.java create mode 100644 acs/nladmin-system/src/main/java/org/nl/modules/lucence/common/LuceneIndexWriter.java create mode 100644 acs/nladmin-system/src/main/java/org/nl/modules/lucence/common/Searcher.java create mode 100644 acs/nladmin-system/src/main/java/org/nl/modules/lucence/config/StaticConfig.java create mode 100644 acs/nladmin-system/src/main/java/org/nl/modules/lucence/config/UrlConfig.java create mode 100644 acs/nladmin-system/src/main/java/org/nl/modules/lucence/enums/LogTypeEnum.java create mode 100644 acs/nladmin-system/src/main/java/org/nl/modules/lucence/rest/LuceneController.java create mode 100644 acs/nladmin-system/src/main/java/org/nl/modules/lucence/service/LuceneExecuteLogService.java create mode 100644 acs/nladmin-system/src/main/java/org/nl/modules/lucence/service/LuceneService.java create mode 100644 acs/nladmin-system/src/main/java/org/nl/modules/lucence/service/dto/LuceneLogDto.java create mode 100644 acs/nladmin-system/src/main/java/org/nl/modules/lucence/service/impl/LuceneExecuteLogServiceImpl.java create mode 100644 acs/nladmin-system/src/main/java/org/nl/modules/lucence/service/impl/LuceneServiceImpl.java create mode 100644 acs/nladmin-system/src/main/resources/log/XgAgvDeviceDriver.xml create mode 100644 acs/nladmin-ui/src/views/monitor/lucene/api/lucene.js diff --git a/acs/nladmin-system/pom.xml b/acs/nladmin-system/pom.xml index 207ff62..97f4309 100644 --- a/acs/nladmin-system/pom.xml +++ b/acs/nladmin-system/pom.xml @@ -30,6 +30,7 @@ 5.9.0 1.9 + 8.2.0 @@ -387,6 +388,40 @@ UserAgentUtils 1.21 + + + org.apache.lucene + lucene-core + ${lucene.version} + + + org.apache.lucene + lucene-highlighter + ${lucene.version} + + + org.apache.lucene + lucene-analyzers-common + ${lucene.version} + + + com.github.magese + ik-analyzer + ${lucene.version} + + + + + org.apache.lucene + lucene-analyzers-smartcn + ${lucene.version} + + + + org.apache.lucene + lucene-queryparser + ${lucene.version} + diff --git a/acs/nladmin-system/src/main/java/org/nl/acs/device_driver/basedriver/standard_conveyor_control/ItemProtocol.java b/acs/nladmin-system/src/main/java/org/nl/acs/device_driver/basedriver/standard_conveyor_control/ItemProtocol.java index e33c7e1..3e69b1e 100644 --- a/acs/nladmin-system/src/main/java/org/nl/acs/device_driver/basedriver/standard_conveyor_control/ItemProtocol.java +++ b/acs/nladmin-system/src/main/java/org/nl/acs/device_driver/basedriver/standard_conveyor_control/ItemProtocol.java @@ -84,7 +84,7 @@ public class ItemProtocol { public int getOpcIntegerValue(String protocol) { Integer value = this.driver.getIntegeregerValue(protocol); if (value == null) { - log.error(this.getDriver().getDeviceCode() + ":protocol " + protocol + " 信号同步异常!"); +// log.error(this.getDriver().getDeviceCode() + ":protocol " + protocol + " 信号同步异常!"); setIsonline(false); } else { setIsonline(true); diff --git a/acs/nladmin-system/src/main/java/org/nl/acs/device_driver/basedriver/standard_conveyor_control_with_plcscanner/ItemProtocol.java b/acs/nladmin-system/src/main/java/org/nl/acs/device_driver/basedriver/standard_conveyor_control_with_plcscanner/ItemProtocol.java index 8854280..6a4d8fc 100644 --- a/acs/nladmin-system/src/main/java/org/nl/acs/device_driver/basedriver/standard_conveyor_control_with_plcscanner/ItemProtocol.java +++ b/acs/nladmin-system/src/main/java/org/nl/acs/device_driver/basedriver/standard_conveyor_control_with_plcscanner/ItemProtocol.java @@ -101,7 +101,7 @@ public class ItemProtocol { public int getOpcIntegerValue(String protocol) { Integer value = this.driver.getIntegeregerValue(protocol); if (value == null) { - log.error(this.getDriver().getDeviceCode() + ":protocol " + protocol + " 信号同步异常!"); +// log.error(this.getDriver().getDeviceCode() + ":protocol " + protocol + " 信号同步异常!"); setIsonline(false); } else { setIsonline(true); diff --git a/acs/nladmin-system/src/main/java/org/nl/acs/device_driver/basedriver/standard_inspect_site/ItemProtocol.java b/acs/nladmin-system/src/main/java/org/nl/acs/device_driver/basedriver/standard_inspect_site/ItemProtocol.java index 749628c..e9f32ef 100644 --- a/acs/nladmin-system/src/main/java/org/nl/acs/device_driver/basedriver/standard_inspect_site/ItemProtocol.java +++ b/acs/nladmin-system/src/main/java/org/nl/acs/device_driver/basedriver/standard_inspect_site/ItemProtocol.java @@ -74,7 +74,7 @@ public class ItemProtocol { public int getOpcIntegerValue(String protocol) { Integer value = this.driver.getIntegeregerValue(protocol); if (value == null) { - log.error(this.getDriver().getDeviceCode() + ":protocol " + protocol + " 信号同步异常!"); +// log.error(this.getDriver().getDeviceCode() + ":protocol " + protocol + " 信号同步异常!"); setIsonline(false); } else { setIsonline(true); diff --git a/acs/nladmin-system/src/main/java/org/nl/modules/lucence/common/Indexer.java b/acs/nladmin-system/src/main/java/org/nl/modules/lucence/common/Indexer.java new file mode 100644 index 0000000..6ec30a1 --- /dev/null +++ b/acs/nladmin-system/src/main/java/org/nl/modules/lucence/common/Indexer.java @@ -0,0 +1,181 @@ +package org.nl.modules.lucence.common; + +import com.alibaba.fastjson.JSONObject; +import org.apache.commons.io.FileUtils; +import org.apache.lucene.analysis.Analyzer; +import org.apache.lucene.analysis.TokenStream; +import org.apache.lucene.analysis.standard.StandardAnalyzer; +import org.apache.lucene.analysis.tokenattributes.CharTermAttribute; +import org.apache.lucene.document.Document; +import org.apache.lucene.document.Field; +import org.apache.lucene.document.StoredField; +import org.apache.lucene.document.TextField; +import org.apache.lucene.index.CodecReader; +import org.apache.lucene.index.IndexWriter; +import org.apache.lucene.index.IndexWriterConfig; +import org.apache.lucene.store.Directory; +import org.apache.lucene.store.FSDirectory; +import org.springframework.beans.factory.annotation.Value; +import org.wltea.analyzer.core.IKSegmenter; +import org.wltea.analyzer.lucene.IKAnalyzer; + +import java.io.*; +import java.nio.file.Paths; +import java.util.Set; + +/** + * lucene索引器 + */ +public class Indexer { + /** + * 写索引实例 + */ + private IndexWriter writer; + + public IndexWriter getWriter() { + return writer; + } + + /** + * 构造方法，实例化IndexWriter + * @param indexDir + * @throws Exception + */ + public Indexer(String indexDir) throws Exception { + Directory dir = FSDirectory.open(Paths.get(indexDir)); + //标准分词器，会自动去掉空格啊，is a the等单词 +// Analyzer analyzer = new StandardAnalyzer(); + Analyzer analyzer = new IKAnalyzer(); + //将标准分词器配到写索引的配置中 + IndexWriterConfig config = new IndexWriterConfig(analyzer); + //实例化写索引对象 + writer = new IndexWriter(dir, config); + } + + /** + * 索引指定目录下的所有文件 + * @param dataDir + * @return + * @throws Exception + */ + public int indexAll(String dataDir) throws Exception { + // 获取该路径下的所有文件 + File[] files = new File(dataDir).listFiles(); + if (null != files) { + for (File file : files) { + //调用下面的indexFile方法，对每个文件进行索引 + indexFile(file); + } + } + //返回索引的文件数 +// return writer.numDocs(); + return writer.numRamDocs(); + } + + /** + * 索引指定的文件 + * @param file + * @throws Exception + */ + private void indexFile(File file) throws Exception { + System.out.println("索引文件的路径：" + file.getCanonicalPath()); + //调用下面的getDocument方法，获取该文件的document + Document doc = getDocument(file); + //添加索引文档 + //Document doc = json2Doc(jsonDoc); +// Document doc = new Document(); +// doc.add(new TextField("content", jsonDoc, Field.Store.YES)); + Field fieldContent=new TextField("fieldContent", FileUtils.readFileToString(null,"UTF-8"), Field.Store.YES); + + //将doc添加到索引中 + writer.addDocument(doc); + } + + /** + * 获取文档，文档里再设置每个字段，就类似于数据库中的一行记录 + * @param file + * @return + * @throws Exception + */ + private Document getDocument(File file) throws Exception { + Document doc = new Document(); + //开始添加字段 + //添加内容 + doc.add(new TextField("contents", new FileReader(file))); + //添加文件名，并把这个字段存到索引文件里 + doc.add(new TextField("fileName", file.getName(), Field.Store.YES)); + //添加文件路径 + doc.add(new TextField("fullPath", file.getCanonicalPath(), Field.Store.YES)); + return doc; + } + public Document json2Doc(String strDoc) { + Document doc = new Document(); + JSONObject jsonDoc = JSONObject.parseObject(strDoc); + Set keys = jsonDoc.keySet(); + for (String key : keys) { + doc.add(new TextField(key, jsonDoc.getString(key), Field.Store.YES)); + } + return doc; + } + + public void addLogIndex(String msg) throws IOException { + //步骤一：创建Directory对象，用于指定索引库的位置 RAMDirectory内存 + Directory directory = FSDirectory.open(new File("D:\\lucene\\index").toPath()); + //步骤二：创建一个IndexWriter对象，用于写索引 +// Analyzer analyzer = new StandardAnalyzer(); + IndexWriter indexWriter=new IndexWriter(directory,new IndexWriterConfig(new IKAnalyzer(false))); +// indexWriter.deleteAll();//清理所有索引库 +// IndexWriter indexWriter=new IndexWriter(directory,new IndexWriterConfig(new StandardAnalyzer())); + //记录索引开始时间 + long startTime = System.currentTimeMillis(); + //步骤三：读取磁盘中文件，对应每一个文件创建一个文档对象 + Document document = new Document(); +// document.add(new TextField("fieldContent", device_id, Field.Store.YES)); + document.add(new TextField("fieldContent", msg, Field.Store.YES)); + indexWriter.addDocument(document); + //记录索引结束时间 + long endTime = System.currentTimeMillis(); + System.out.println("建立索引"+ "共耗时" + (endTime-startTime) + "毫秒"); + indexWriter.commit(); + //步骤八：关闭资源 + indexWriter.close(); + System.out.println("建立索引成功-----关闭资源"); + } + //系统的日志文件路径 + @Value("${logging.file.path}") + private String logUrl; + + public static void main(String[] args)throws IOException { + //步骤一：创建Directory对象，用于指定索引库的位置 RAMDirectory内存 + Directory directory = FSDirectory.open(new File("D:\\lucene\\index").toPath()); + //步骤二：创建一个IndexWriter对象，用于写索引 +// Analyzer analyzer = new StandardAnalyzer(); + IndexWriter indexWriter=new IndexWriter(directory,new IndexWriterConfig(new IKAnalyzer(false))); + + indexWriter.deleteAll();//清理所有索引库 +// IndexWriter indexWriter=new IndexWriter(directory,new IndexWriterConfig(new StandardAnalyzer())); + //记录索引开始时间 + long startTime = System.currentTimeMillis(); + //步骤三：读取磁盘中文件，对应每一个文件创建一个文档对象 + File file=new File("D:\\testlog"); + //步骤四：获取文件列表 + File[] files = file.listFiles(); + for (File item:files) { + BufferedReader bufferedReader = new BufferedReader(new FileReader(item)); + String strLine = null; + while(null != (strLine = bufferedReader.readLine())){ + Document document = new Document(); +// document.add(new Field()); + document.add(new TextField("fieldContent", strLine, Field.Store.YES)); + indexWriter.addDocument(document); + } + } + //记录索引结束时间 + long endTime = System.currentTimeMillis(); + System.out.println("建立索引"+ "共耗时" + (endTime-startTime) + "毫秒"); + indexWriter.commit(); + //步骤八：关闭资源 + indexWriter.close(); + System.out.println("建立索引成功-----关闭资源"); + } +} diff --git a/acs/nladmin-system/src/main/java/org/nl/modules/lucence/common/LuceneIndexWriter.java b/acs/nladmin-system/src/main/java/org/nl/modules/lucence/common/LuceneIndexWriter.java new file mode 100644 index 0000000..89f5e64 --- /dev/null +++ b/acs/nladmin-system/src/main/java/org/nl/modules/lucence/common/LuceneIndexWriter.java @@ -0,0 +1,68 @@ +package org.nl.modules.lucence.common; + +import cn.hutool.core.date.DateUtil; +import org.apache.lucene.index.CorruptIndexException; +import org.apache.lucene.index.IndexWriter; +import org.apache.lucene.index.IndexWriterConfig; +import org.apache.lucene.store.Directory; +import org.apache.lucene.store.FSDirectory; +import org.nl.modules.lucence.config.UrlConfig; +import org.springframework.beans.factory.annotation.Value; +import org.wltea.analyzer.lucene.IKAnalyzer; + +import javax.annotation.PostConstruct; +import java.io.File; +import java.io.IOException; +import java.text.ParseException; +import java.text.SimpleDateFormat; +import java.util.Date; + +public class LuceneIndexWriter { + private static IndexWriter indexWriter; + + static { + try { + Directory directory = FSDirectory.open(new File(UrlConfig.luceneUrl).toPath()); + IndexWriterConfig config = new IndexWriterConfig(new IKAnalyzer()); + indexWriter = new IndexWriter(directory, config); + } catch (Exception e) { + e.printStackTrace(); + } + /**当当前线程结束时，自动关闭IndexWriter，使用Runtime对象*/ + Runtime.getRuntime().addShutdownHook(new Thread(){ + @Override + public void run() { + try { + closeIndexWriter(); + } catch (Exception e) { + e.printStackTrace(); + } + } + }); + } + /**在线程结束时，自动关闭IndexWriter*/ + public static IndexWriter getIndexWriter() { + return indexWriter; + } + /**关闭IndexWriter + * @throws IOException + * @throws CorruptIndexException */ + public static void closeIndexWriter() throws Exception { + if(indexWriter != null) { + indexWriter.close(); + } + } + + public static void main(String[] args) throws IOException { + indexWriter.deleteAll(); + } + + public static String getDate(String timeString) throws ParseException { + SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSSX");//时间格式 + Date date = sdf.parse(timeString); + timeString = DateUtil.format(date, "yyyy-MM-dd HH:mm:ss.SSS");//格式化后的时间 + return timeString; + } + + +} diff --git a/acs/nladmin-system/src/main/java/org/nl/modules/lucence/common/Searcher.java b/acs/nladmin-system/src/main/java/org/nl/modules/lucence/common/Searcher.java new file mode 100644 index 0000000..da39ef7 --- /dev/null +++ b/acs/nladmin-system/src/main/java/org/nl/modules/lucence/common/Searcher.java @@ -0,0 +1,132 @@ +package org.nl.modules.lucence.common; + +import cn.hutool.core.date.DateTime; +import cn.hutool.core.date.DateUtil; +import com.alibaba.fastjson.JSONArray; +import com.alibaba.fastjson.JSONObject; +import lombok.extern.slf4j.Slf4j; +import org.apache.lucene.analysis.Analyzer; +import org.apache.lucene.document.Document; +import org.apache.lucene.index.DirectoryReader; +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.Term; +import org.apache.lucene.util.BytesRef; +import org.apache.lucene.queryparser.classic.QueryParser; +import org.apache.lucene.search.*; +import org.apache.lucene.store.Directory; +import org.apache.lucene.store.FSDirectory; +import org.wltea.analyzer.lucene.IKAnalyzer; + +import java.nio.file.Paths; +import java.util.*; + +/** + * lucene查询器 + */ +@Slf4j +public class Searcher { + + public static Map search(String indexDir, String ext,Map whereJson) throws Exception { + //获取要查询的路径，也就是索引所在的位置 + Directory dir = FSDirectory.open(Paths.get(indexDir)); + IndexReader reader = DirectoryReader.open(dir); + //构建IndexSearcher + IndexSearcher searcher = new IndexSearcher(reader); + //标准分词器，会自动去掉空格啊，is a the等单词 +// Analyzer analyzer = new StandardAnalyzer(); + Analyzer analyzer = new IKAnalyzer(false); + //查询解析器 + QueryParser queryParser = new QueryParser("fieldContent", analyzer); + + //记录索引开始时间 + long startTime = System.currentTimeMillis(); + // 实际上Lucene本身不支持分页。因此我们需要自己进行逻辑分页。我们要准备分页参数： + int pageSize = Integer.parseInt(whereJson.get("size").toString());// 每页条数 + int pageNum = Integer.parseInt(whereJson.get("page").toString());// 当前页码 + int start = pageNum * pageSize;// 当前页的起始条数 + int end = start + pageSize;// 当前页的结束条数（不能包含） + // 创建排序对象,需要排序字段SortField，参数：字段的名称、字段的类型、是否反转如果是false，升序。true降序 + Sort sort = new Sort(new SortField("logTime", SortField.Type.DOC,true)); + + TopDocs docs = null; + BooleanQuery.Builder booleanQueryBuilder = new BooleanQuery.Builder(); + //时间范围查询 + String startDate = (String) whereJson.get("begin_time"); + String endDate = (String) whereJson.get("end_time"); + Calendar calendar=Calendar.getInstance(); + calendar.set(1970, 0, 1); + if (startDate == null){ + startDate = DateUtil.format(calendar.getTime(),"yyyy-MM-dd HH:mm:ss.SSS"); + }else{ + startDate = LuceneIndexWriter.getDate(startDate); + } + if (endDate == null){ + endDate = DateUtil.format(new DateTime(),"yyyy-MM-dd HH:mm:ss.SSS"); + } else { + endDate = LuceneIndexWriter.getDate(endDate); + } + TermRangeQuery termRangeQuery = new TermRangeQuery("logTime", new BytesRef(startDate), new BytesRef(endDate), true, true); + booleanQueryBuilder.add(termRangeQuery,BooleanClause.Occur.MUST); + if (whereJson.get("device_code") != null){ + Query termQuery = new TermQuery(new Term("device_code", (String) whereJson.get("device_code"))); + booleanQueryBuilder.add(termQuery,BooleanClause.Occur.MUST); + } + if (whereJson.get("method") != null){ + Query termQuery = new TermQuery(new Term("method", (String) whereJson.get("method"))); + booleanQueryBuilder.add(termQuery,BooleanClause.Occur.MUST); + } + if (whereJson.get("status_code") != null){ + Query termQuery = new TermQuery(new Term("status_code", (String) whereJson.get("status_code"))); + booleanQueryBuilder.add(termQuery,BooleanClause.Occur.MUST); + } + if (whereJson.get("blurry") != null) { + Query query = queryParser.parse((String) whereJson.get("blurry")); + booleanQueryBuilder.add(query, BooleanClause.Occur.MUST); + } + docs = searcher.search(booleanQueryBuilder.build(), end,sort); + //记录索引时间 + long endTime = System.currentTimeMillis(); + log.info("匹配{}共耗时{}毫秒",booleanQueryBuilder.build(),(endTime-startTime)); + log.info("查询到{}条日志文件", docs.totalHits.value); + List list = new ArrayList<>(); + ScoreDoc[] scoreDocs = docs.scoreDocs; + if (end > docs.totalHits.value) end = (int) docs.totalHits.value; + JSONArray array = new JSONArray(); + + for (int i = start; i < end; i++) { + ScoreDoc scoreDoc = scoreDocs[i]; + Document doc = reader.document(scoreDoc.doc); + JSONObject object = new JSONObject(); + object.put("content",doc.get("fieldContent")); + object.put("device_code",doc.get("device_code")); + object.put("logTime",doc.get("logTime")); + object.put("method",doc.get("method")); + object.put("status_code",doc.get("status_code")); + object.put("requestparam",doc.get("requestparam")); + object.put("responseparam",doc.get("responseparam")); + if(doc.get("fieldContent") != null) { + array.add(object); + } + } + for(Object logDto:array){ + log.info(logDto.toString()); + } + reader.close(); + JSONObject jo = new JSONObject(); + jo.put("content", array); + jo.put("totalElements", docs.totalHits.value); + return jo; + } + + public static void main(String[] args) { + String indexDir = "D:\\lucene\\index"; + //查询这个字符串 + String q = "07.832"; + Map whereJson = null; + try { + search(indexDir, q,whereJson); + } catch (Exception e) { + e.printStackTrace(); + } + } +} diff --git a/acs/nladmin-system/src/main/java/org/nl/modules/lucence/config/StaticConfig.java b/acs/nladmin-system/src/main/java/org/nl/modules/lucence/config/StaticConfig.java new file mode 100644 index 0000000..19b9245 --- /dev/null +++ b/acs/nladmin-system/src/main/java/org/nl/modules/lucence/config/StaticConfig.java @@ -0,0 +1,22 @@ +package org.nl.modules.lucence.config; + +import org.nl.modules.lucence.common.LuceneIndexWriter; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +/** + * @deprecated 设置静态参数初始化 + */ +@Configuration +public class StaticConfig { + //日志索引目录 + @Value("${lucene.index.path}") + private String luceneDir; + + @Bean + public int initStatic() { + UrlConfig.setLuceneUrl(luceneDir); + return 0; + } +} diff --git a/acs/nladmin-system/src/main/java/org/nl/modules/lucence/config/UrlConfig.java b/acs/nladmin-system/src/main/java/org/nl/modules/lucence/config/UrlConfig.java new file mode 100644 index 0000000..e15b3e6 --- /dev/null +++ b/acs/nladmin-system/src/main/java/org/nl/modules/lucence/config/UrlConfig.java @@ -0,0 +1,13 @@ +package org.nl.modules.lucence.config; + +public class UrlConfig { + public static String luceneUrl; + + public static String getLuceneUrl() { + return luceneUrl; + } + + public static void setLuceneUrl(String luceneUrl) { + UrlConfig.luceneUrl = luceneUrl; + } +} diff --git a/acs/nladmin-system/src/main/java/org/nl/modules/lucence/enums/LogTypeEnum.java b/acs/nladmin-system/src/main/java/org/nl/modules/lucence/enums/LogTypeEnum.java new file mode 100644 index 0000000..8993072 --- /dev/null +++ b/acs/nladmin-system/src/main/java/org/nl/modules/lucence/enums/LogTypeEnum.java @@ -0,0 +1,15 @@ +package org.nl.modules.lucence.enums; + +public enum LogTypeEnum { + DEVICE_LOG("设备日志"), + INTERFACE_LOG("接口日志"); + + private String desc; + + LogTypeEnum(String desc) { + this.desc = desc; + } + public String getDesc() { + return desc; + } +} diff --git a/acs/nladmin-system/src/main/java/org/nl/modules/lucence/rest/LuceneController.java b/acs/nladmin-system/src/main/java/org/nl/modules/lucence/rest/LuceneController.java new file mode 100644 index 0000000..5fc6983 --- /dev/null +++ b/acs/nladmin-system/src/main/java/org/nl/modules/lucence/rest/LuceneController.java @@ -0,0 +1,39 @@ +package org.nl.modules.lucence.rest; + +import io.swagger.annotations.Api; +import io.swagger.annotations.ApiOperation; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.nl.modules.logging.annotation.Log; +import org.nl.modules.lucence.service.LuceneService; +import org.springframework.data.domain.Pageable; +import org.springframework.http.HttpStatus; +import org.springframework.http.ResponseEntity; +import org.springframework.web.bind.annotation.*; +import java.util.Map; + + +@RestController +@RequiredArgsConstructor +@Api(tags = "日志检索") +@RequestMapping("/api/lucene") +@Slf4j +public class LuceneController { + + private final LuceneService luceneService; + + @GetMapping("/labels/values") + @ApiOperation("获取标签") + public ResponseEntity labelsValues() { + return new ResponseEntity<>(luceneService.getLabelsValues(), HttpStatus.OK); + } + + + @GetMapping("/getAll") + @Log("日志检索") + @ApiOperation("日志检索") + //@PreAuthorize("@el.check('task:list')") + public ResponseEntity get(@RequestParam Map whereJson, Pageable page) { + return new ResponseEntity<>(luceneService.getAll(whereJson, page), HttpStatus.OK); + } +} diff --git a/acs/nladmin-system/src/main/java/org/nl/modules/lucence/service/LuceneExecuteLogService.java b/acs/nladmin-system/src/main/java/org/nl/modules/lucence/service/LuceneExecuteLogService.java new file mode 100644 index 0000000..fed8824 --- /dev/null +++ b/acs/nladmin-system/src/main/java/org/nl/modules/lucence/service/LuceneExecuteLogService.java @@ -0,0 +1,41 @@ +package org.nl.modules.lucence.service; + + +import org.nl.modules.lucence.service.dto.LuceneLogDto; + +import java.io.IOException; + +public interface LuceneExecuteLogService { + /** + * 设备光电变化实时光电信号 + * + * @param device_code 设备编号 + * @param key plc信号 + * @param value plc信号值 + */ + void deviceItemValue(String device_code, String key, String value); + + /** + * 设备执行日志,会保留历史记录 + * + * @param luceneLogDto 日志结果对象 + */ + void deviceExecuteLog(LuceneLogDto luceneLogDto); + + /** + * 接口日志,会保留历史记录 + * + * @param luceneLogDto 日志结果对象 + */ + void interfaceExecuteLog(LuceneLogDto luceneLogDto) throws IOException; + + /** + * 设备执行日志,会保留历史记录 + * + * @param name 日志名称 + * @param message 日志信息 + */ + void extLog(String name, String message); + + +} diff --git a/acs/nladmin-system/src/main/java/org/nl/modules/lucence/service/LuceneService.java b/acs/nladmin-system/src/main/java/org/nl/modules/lucence/service/LuceneService.java new file mode 100644 index 0000000..7bf0da9 --- /dev/null +++ b/acs/nladmin-system/src/main/java/org/nl/modules/lucence/service/LuceneService.java @@ -0,0 +1,27 @@ +package org.nl.modules.lucence.service; + +import com.alibaba.fastjson.JSONArray; +import com.alibaba.fastjson.JSONObject; +import org.nl.modules.logging.service.dto.LogQueryCriteria; +import org.springframework.data.domain.Pageable; + +import java.util.List; +import java.util.Map; + + +public interface LuceneService { + /** + * 获取labels和values树 + * @return + */ + JSONArray getLabelsValues(); + + /** + * 获取数据分页 + * + * @param whereJson 条件 + * @param page 分页参数 + * @return Map + */ + Map getAll(Map whereJson, Pageable page); +} diff --git a/acs/nladmin-system/src/main/java/org/nl/modules/lucence/service/dto/LuceneLogDto.java b/acs/nladmin-system/src/main/java/org/nl/modules/lucence/service/dto/LuceneLogDto.java new file mode 100644 index 0000000..3b67124 --- /dev/null +++ b/acs/nladmin-system/src/main/java/org/nl/modules/lucence/service/dto/LuceneLogDto.java @@ -0,0 +1,86 @@ +package org.nl.modules.lucence.service.dto; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@Builder +@NoArgsConstructor +@AllArgsConstructor +public class LuceneLogDto { + + /* 日志标识 */ + private String log_uuid; + /*日志类型*/ + private String logType; + /*设备编号*/ + private String device_code; + /*内容详情*/ + private String content; + + /* 任务编码 */ + private String task_code; + + /* 指令编码 */ + private String instruct_code; + + /* 任务标识 */ + private String task_id; + + /* 载具号 */ + private String vehicle_code; + + /* 备注 */ + private String remark; + + /* 日志类型 */ + private String log_type; + + /* 方法 */ + private String method; + + /* 请求参数 */ + private String requestparam; + + /* 响应参数 */ + private String responseparam; + + /* 请求地址 */ + private String requesturl; + + /* 状态码 */ + private String status_code; + + /* 是否删除 1:是；0：否 */ + private String is_delete; + + /* 创建者 */ + private String create_by; + + /* 创建时间 YYYY-MM-DD hh:mm:ss */ + private String create_time; + + /* 修改者 */ + private String update_by; + + /* 修改时间 */ + private String update_time; + + + + public LuceneLogDto (final String opc_server_code,final String opc_plc_code, + final String device_code,final String to_home,final int last_home, + final int home) { + super (); + this.device_code = device_code; + this.content = "信号" + + opc_server_code + "." + + opc_plc_code + "." + + device_code + "." + + to_home + "变更从" + + last_home + "->" + + home; + } +} diff --git a/acs/nladmin-system/src/main/java/org/nl/modules/lucence/service/impl/LuceneExecuteLogServiceImpl.java b/acs/nladmin-system/src/main/java/org/nl/modules/lucence/service/impl/LuceneExecuteLogServiceImpl.java new file mode 100644 index 0000000..7902f11 --- /dev/null +++ b/acs/nladmin-system/src/main/java/org/nl/modules/lucence/service/impl/LuceneExecuteLogServiceImpl.java @@ -0,0 +1,104 @@ +package org.nl.modules.lucence.service.impl; + +import cn.hutool.core.date.DateTime; +import cn.hutool.core.date.DateUtil; +import cn.hutool.core.util.ObjectUtil; +import lombok.RequiredArgsConstructor; +import lombok.SneakyThrows; +import lombok.extern.slf4j.Slf4j; +import org.apache.lucene.document.Document; +import org.apache.lucene.document.Field; +import org.apache.lucene.document.StringField; +import org.apache.lucene.document.TextField; +import org.apache.lucene.index.IndexWriter; +import org.nl.modules.lucence.enums.LogTypeEnum; +import org.nl.modules.lucence.service.LuceneExecuteLogService; +import org.nl.modules.lucence.service.dto.LuceneLogDto; +import org.nl.modules.lucence.common.LuceneIndexWriter; +import org.slf4j.MDC; +import org.springframework.stereotype.Service; + +import java.io.IOException; + +/** + * @author jlm + * @description 服务实现 + * @date 2023-04-11 + */ +@Service +@RequiredArgsConstructor +@Slf4j +public class LuceneExecuteLogServiceImpl implements LuceneExecuteLogService { + + @Override + public void deviceItemValue(String device_code, String key, String value) { + String now = DateUtil.now(); + } + + @SneakyThrows + @Override + public void deviceExecuteLog(LuceneLogDto luceneLogDto) { + luceneLogDto.setLogType(LogTypeEnum.DEVICE_LOG.getDesc()); + addIndex(luceneLogDto); + } + + @Override + public void interfaceExecuteLog(LuceneLogDto luceneLogDto) throws IOException { + luceneLogDto.setLogType(LogTypeEnum.INTERFACE_LOG.getDesc()); + addIndex(luceneLogDto); + } + + private void addIndex(LuceneLogDto luceneLogDto) throws IOException { + IndexWriter indexWriter = LuceneIndexWriter.getIndexWriter(); + //创建一个Document对象 + Document document = new Document(); + try { + //记录索引开始时间 + long startTime = System.currentTimeMillis(); + //向document对象中添加域。 + if (ObjectUtil.isNotEmpty(luceneLogDto.getDevice_code())){ + document.add(new StringField("device_code", luceneLogDto.getDevice_code(), Field.Store.YES)); +// document.add(new TextField("device_code", luceneLogDto.getDevice_code(), Field.Store.YES)); + } + if (ObjectUtil.isNotEmpty(luceneLogDto.getContent())) { + document.add(new TextField("fieldContent", luceneLogDto.getContent(), Field.Store.YES)); + } + if (ObjectUtil.isNotEmpty(luceneLogDto.getMethod())) { + document.add(new StringField("method", luceneLogDto.getMethod(), Field.Store.YES)); + } + if (ObjectUtil.isNotEmpty(luceneLogDto.getStatus_code())) { + document.add(new StringField("status_code", luceneLogDto.getStatus_code(), Field.Store.YES)); + } + if (ObjectUtil.isNotEmpty(luceneLogDto.getRequestparam())) { + document.add(new TextField("requestparam", luceneLogDto.getRequestparam(), Field.Store.YES)); + } + if (ObjectUtil.isNotEmpty(luceneLogDto.getResponseparam())) { + document.add(new TextField("responseparam", luceneLogDto.getResponseparam(), Field.Store.YES)); + } + document.add(new StringField("logType", luceneLogDto.getLogType(), Field.Store.YES)); + document.add(new StringField("logTime", DateUtil.format(new DateTime(),"yyyy-MM-dd HH:mm:ss.SSS"), Field.Store.YES)); + indexWriter.addDocument(document); + //记录索引结束时间 + long endTime = System.currentTimeMillis(); + log.info("建立索引共耗时{}毫秒",endTime-startTime); + indexWriter.commit(); + MDC.put("DEVICECODE", luceneLogDto.getDevice_code()); + }catch (Exception e) { + log.error(e.getMessage(),e); + }finally { +// MDC.remove("DEVICECODE"); + } + } + + @Override + public void extLog(String name, String message) { + try { + MDC.put(name, name); + log.info("{}", message); + } catch (Exception e) { + e.printStackTrace(); + } finally { + MDC.remove(name); + } + } +} diff --git a/acs/nladmin-system/src/main/java/org/nl/modules/lucence/service/impl/LuceneServiceImpl.java b/acs/nladmin-system/src/main/java/org/nl/modules/lucence/service/impl/LuceneServiceImpl.java new file mode 100644 index 0000000..d232daf --- /dev/null +++ b/acs/nladmin-system/src/main/java/org/nl/modules/lucence/service/impl/LuceneServiceImpl.java @@ -0,0 +1,85 @@ +package org.nl.modules.lucence.service.impl; + +import cn.hutool.core.util.CharsetUtil; +import cn.hutool.http.HttpUtil; +import com.alibaba.fastjson.JSONArray; +import com.alibaba.fastjson.JSONObject; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.nl.modules.lucence.common.Searcher; +import org.nl.modules.lucence.service.LuceneService; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.data.domain.Pageable; +import org.springframework.stereotype.Service; + +import java.util.Map; + + +@Service +@RequiredArgsConstructor +@Slf4j +public class LuceneServiceImpl implements LuceneService { + + @Value("${loki.url}") + private String lokiUrl; + + @Value("${loki.systemName}") + private String systemName; + + //日志索引目录 + @Value("${lucene.index.path}") + private String luceneUrl; + + /** + * 获取labels和values树 + * + * @return + */ + @Override + public JSONArray getLabelsValues() { + JSONArray result = new JSONArray(); + // 获取所有标签 + String labelString = HttpUtil.get(lokiUrl + "/labels", CharsetUtil.CHARSET_UTF_8); + JSONObject parse = (JSONObject) JSONObject.parse(labelString); + JSONArray labels = parse.getJSONArray("data"); + for (int i=0; i getAll(Map whereJson, Pageable page) { + JSONObject jo = new JSONObject(); + try { + JSONObject jsonObject = (JSONObject) Searcher.search(luceneUrl, "", whereJson); + JSONArray array = jsonObject.getJSONArray("content"); + int totalElements = Integer.parseInt(jsonObject.get("totalElements").toString()); + jo.put("content", array); + jo.put("totalElements", totalElements); + } catch (Exception e) { + log.error("索引查询为空", e); + throw new NullPointerException("索引查询为空"); + } + + return jo; + } + +} diff --git a/acs/nladmin-system/src/main/resources/config/application-dev.yml b/acs/nladmin-system/src/main/resources/config/application-dev.yml index 1d3349c..8f584e2 100644 --- a/acs/nladmin-system/src/main/resources/config/application-dev.yml +++ b/acs/nladmin-system/src/main/resources/config/application-dev.yml @@ -9,7 +9,8 @@ spring: url: jdbc:log4jdbc:mysql://${DB_HOST:127.0.0.1}:${DB_PORT:3306}/${DB_NAME:lnsh_acs2}?serverTimezone=Asia/Shanghai&characterEncoding=utf8&useSSL=false&useOldAliasMetadataBehavior=true # url: jdbc:log4jdbc:mysql://${DB_HOST:192.168.81.252}:${DB_PORT:3306}/${DB_NAME:yongyu_acs2}?serverTimezone=Asia/Shanghai&characterEncoding=utf8&useSSL=false&useOldAliasMetadataBehavior=true username: ${DB_USER:root} - password: ${DB_PWD:zjw123} +# password: ${DB_PWD:zjw123} + password: ${DB_PWD:123456} # password: ${DB_PWD:Root.123456} # 初始连接数 initial-size: 5 @@ -130,7 +131,9 @@ logging: file: path: C:\log\wms config: classpath:logback-spring.xml - +lucene: + index: + path: D:\lucene\index # Sa-Token配置 sa-token: # token 名称 (同时也是cookie名称) diff --git a/acs/nladmin-system/src/main/resources/log/XgAgvDeviceDriver.xml b/acs/nladmin-system/src/main/resources/log/XgAgvDeviceDriver.xml new file mode 100644 index 0000000..d2a9062 --- /dev/null +++ b/acs/nladmin-system/src/main/resources/log/XgAgvDeviceDriver.xml @@ -0,0 +1,34 @@ + + + + + + + + + + ${LOG_HOME}/XgAgvDeviceDriver/${DEVICECODE}/%d{yyyy-MM-dd}.%i.log + + 15 + + 200MB + + 2GB + + + + %d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %-5level %logger{50} - %msg%n + ${log.charset} + + + + + + + + + + + diff --git a/acs/nladmin-system/src/main/resources/logback-spring.xml b/acs/nladmin-system/src/main/resources/logback-spring.xml index b83da1b..81bffa2 100644 --- a/acs/nladmin-system/src/main/resources/logback-spring.xml +++ b/acs/nladmin-system/src/main/resources/logback-spring.xml @@ -31,6 +31,7 @@ https://juejin.cn/post/6844903775631572999 --> + @@ -84,9 +85,10 @@ https://juejin.cn/post/6844903775631572999 - + + diff --git a/acs/nladmin-ui/src/views/monitor/lucene/api/lucene.js b/acs/nladmin-ui/src/views/monitor/lucene/api/lucene.js new file mode 100644 index 0000000..674bb2c --- /dev/null +++ b/acs/nladmin-ui/src/views/monitor/lucene/api/lucene.js @@ -0,0 +1,18 @@ +import request from '@/utils/request' + +export function getLogData(param) { + return request({ + url: 'api/lucene/getAll', + method: 'get', + data: param + }) +} + +export function labelsValues() { + return request({ + url: 'api/loki/labels/values', + method: 'get' + }) +} + +export default { getLogData, labelsValues } diff --git a/acs/nladmin-ui/src/views/monitor/lucene/index.vue b/acs/nladmin-ui/src/views/monitor/lucene/index.vue index 49b7e27..a048077 100644 --- a/acs/nladmin-ui/src/views/monitor/lucene/index.vue +++ b/acs/nladmin-ui/src/views/monitor/lucene/index.vue @@ -1,20 +1,8 @@ @@ -43,14 +38,13 @@ import crudOperation from '@crud/CRUD.operation' import pagination from '@crud/Pagination' import { delAll } from '@/api/acs/lucene/log' -const start = new Date() export default { name: 'LuceneLog', components: { Search, pagination, crudOperation }, mixins: [presenter(), header(), crud()], - cruds() { + cruds: function() { return CRUD({ - title: '系统参数', url: 'api/lucene', idField: 'id', sort: 'id,desc', + title: '系统参数', url: 'api/lucene/getAll', idField: 'id', sort: 'id,desc', queryOnPresenterCreated: true, optShow: { add: false, @@ -64,7 +58,7 @@ export default { page: 0 }, query: { - createTime: [start.setTime(start.getTime() - 3600 * 1000 * 0.5), new Date()] + createTime: [new Date(new Date().setTime(new Date().getTime() - 3600 * 1000 * 24)), new Date()] } }) }, diff --git a/acs/nladmin-ui/src/views/monitor/lucene/search.vue b/acs/nladmin-ui/src/views/monitor/lucene/search.vue index bfb9c19..88a7655 100644 --- a/acs/nladmin-ui/src/views/monitor/lucene/search.vue +++ b/acs/nladmin-ui/src/views/monitor/lucene/search.vue @@ -1,10 +1,50 @@