解决ES查询和ES数据清理

doc-admin/src/main/java/com/doc/task/CleanEsTask.java

@@ -0,0 +1,101 @@
+package com.doc.task;
+
+import cn.hutool.http.HttpRequest;
+import com.doc.biz.domain.DocInfo;
+import com.doc.biz.domain.DocSpace;
+import com.doc.biz.service.IDocInfoService;
+import com.doc.biz.service.IDocSpaceService;
+import com.doc.biz.service.IEsDocInfoService;
+import com.doc.common.config.EsConfig;
+import org.elasticsearch.client.HeapBufferedAsyncResponseConsumer;
+import org.elasticsearch.client.HttpAsyncResponseConsumerFactory;
+import org.elasticsearch.client.RequestOptions;
+import org.springframework.core.env.Environment;
+import org.springframework.stereotype.Component;
+
+import javax.annotation.Resource;
+import java.lang.reflect.Field;
+import java.lang.reflect.Modifier;
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * 定时任务调度测试
+ *
+ * @author ruoyi
+ */
+@Component("cleanEsTask")
+public class CleanEsTask {
+    @Resource
+    private IEsDocInfoService esService;
+    @Resource
+    private IDocInfoService infoService;
+    @Resource
+    private IDocSpaceService spaceService;
+    @Resource
+    private EsConfig esConfig;
+    @Resource
+    private Environment environment;
+
+    public void clean() {
+        setParam();
+        List<Long> ids = new ArrayList<>();
+        spaceService.selectDocSpaceList(new DocSpace()).forEach(docSpace -> {
+            String indexName = "docs_" + docSpace.getSpaceId();
+            esConfig.setIndexName(indexName);
+            try {
+                esService.findAll().forEach(esDocInfo -> {
+                    DocInfo info = infoService.selectDocInfoByDocId(esDocInfo.getId());
+                    if (info == null) {
+                        ids.add(esDocInfo.getId());
+                        esService.delete(esDocInfo);
+                    }
+                });
+            } catch (Exception e) {
+                e.printStackTrace();
+            }
+        });
+
+        System.err.println(ids);
+    }
+
+    /**
+     * 设置高量显示偏移
+     * 设置查询BUFFER大小
+     */
+    private void setParam() {
+        try {
+            //设置高亮显示偏移--start
+            String esUri = environment.getProperty("spring.elasticsearch.rest.uris");
+            String offset = environment.getProperty("spring.elasticsearch.highlight.max-analyzed-offset");
+            String body = "{" +
+                    "    \"index\" : {" +
+                    "        \"highlight.max_analyzed_offset\":%s" +
+                    "    }" +
+                    "}";
+            body = String.format(body, offset);
+            HttpRequest.put(esUri + "/_settings").header("Content-Type", "application/json")
+                    .body(body).execute().body();
+            //设置高亮显示偏移--end
+
+
+            //设置es查询buffer大小--start
+            RequestOptions requestOptions = RequestOptions.DEFAULT;
+            Class<? extends RequestOptions> reqClass = requestOptions.getClass();
+            Field reqField = reqClass.getDeclaredField("httpAsyncResponseConsumerFactory");
+            reqField.setAccessible(true);
+            //去除final
+            Field modifiersField = Field.class.getDeclaredField("modifiers");
+            modifiersField.setAccessible(true);
+            modifiersField.setInt(reqField, reqField.getModifiers() & ~Modifier.FINAL);
+            //设置默认的工厂
+            reqField.set(requestOptions, (HttpAsyncResponseConsumerFactory) () -> {
+                //500MB
+                return new HeapBufferedAsyncResponseConsumer(5 * 100 * 1024 * 1024);
+            });
+            //设置es查询buffer大小--end
+        } catch (NoSuchFieldException | IllegalAccessException e) {
+            e.printStackTrace();
+        }
+    }
+}

doc-admin/src/main/resources/application-t2.yml

@@ -16,6 +16,8 @@ spring:
       connection-timeout: 60000
       # 读超时时间，单位：毫秒
       read-timeout: 60000
+    highlight:
+      max-analyzed-offset: 6654321
   #mongodb配置
   data:
     mongodb:

doc-admin/src/test/java/com/test/CleanEsTest.java

@@ -0,0 +1,31 @@
+package com.test;
+
+import com.doc.RuoYiApplication;
+import com.doc.task.CleanEsTask;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.springframework.boot.test.context.SpringBootTest;
+import org.springframework.test.context.ActiveProfiles;
+import org.springframework.test.context.junit4.SpringRunner;
+
+import javax.annotation.Resource;
+import java.io.FileNotFoundException;
+
+/**
+ * Test$
+ *
+ * @author wukai
+ * @date 2023/9/7 16:57
+ */
+@RunWith(SpringRunner.class)
+@SpringBootTest(classes = RuoYiApplication.class, webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT)
+@ActiveProfiles("t2")
+public class CleanEsTest {
+    @Resource
+    CleanEsTask esTask;
+
+    @Test
+    public void test() throws FileNotFoundException {
+        esTask.clean();
+    }
+}

doc-admin/src/test/java/com/test/TestPut.java

@@ -0,0 +1,22 @@
+package com.test;
+
+import cn.hutool.http.HttpRequest;
+
+/**
+ * TestPut$
+ *
+ * @author wukai
+ * @date 2024/3/12 11:53
+ */
+public class TestPut {
+    public static void main(String[] args) {
+        String xx = HttpRequest.put("http://192.168.188.88:19200/_settings").header("Content-Type", "application/json")
+                .body("{\n" +
+                        "    \"index\" : {\n" +
+                        "        \"highlight.max_analyzed_offset\":333211\n" +
+                        "    }\n" +
+                        "}\n").execute().body();
+        System.err.println(xx);
+
+    }
+}

doc-biz/src/main/java/com/doc/biz/controller/DocInfoController.java

@@ -414,7 +414,7 @@ public class DocInfoController extends BaseController {
              }*/
             try {
                 //删除es记录
-                elasticSearchService.delete(info.getDocId());
+                elasticSearchService.delete(info);
             } catch (Exception ignored) {
             }
             //删除收藏记录

doc-biz/src/main/java/com/doc/biz/controller/ElasticSearchController.java

@@ -1,5 +1,6 @@
 package com.doc.biz.controller;
 
+import cn.hutool.http.HttpRequest;
 import com.doc.biz.domain.DocInfo;
 import com.doc.biz.domain.DocSpace;
 import com.doc.biz.domain.EsDocInfo;
@@ -15,11 +16,10 @@ import io.swagger.annotations.ApiImplicitParam;
 import io.swagger.annotations.ApiImplicitParams;
 import io.swagger.annotations.ApiOperation;
 import javafx.util.Pair;
-import org.apache.http.HttpResponse;
-import org.apache.http.nio.protocol.HttpAsyncResponseConsumer;
 import org.elasticsearch.client.HeapBufferedAsyncResponseConsumer;
 import org.elasticsearch.client.HttpAsyncResponseConsumerFactory;
 import org.elasticsearch.client.RequestOptions;
+import org.springframework.core.env.Environment;
 import org.springframework.data.domain.PageRequest;
 import org.springframework.data.domain.Pageable;
 import org.springframework.data.domain.Sort;
@@ -54,6 +54,8 @@ public class ElasticSearchController {
     private IDocSpaceService spaceService;
     @Resource
     private IDocDirService dirService;
+    @Resource
+    private Environment environment;
 
 
     /**
@@ -115,9 +117,8 @@ public class ElasticSearchController {
         page = page - 1;
         String indexName = "docs_" + spaceId;
         esConfig.setIndexName(indexName);
-
         try {
-            setBuffer();
+            setParam();
             Pageable pageable = PageRequest.of(page, size, Sort.by(Sort.Order.desc("id")));
             long total = esService.countByContent(keyword);
             List<SearchHit<EsDocInfo>> all = esService.findByContent(keyword, pageable);
@@ -132,6 +133,9 @@ public class ElasticSearchController {
                         docInfo.setDir(dirService.selectDocDirByDirId(docInfo.getDirId()));
                         esDocInfo.setDocInfo(docInfo);
                         result.add(re);
+                    } else {
+                        esService.delete(esDocInfo);
+                        total--;
                     }
                 }
                 return new Pair<>(total, result);
@@ -144,11 +148,26 @@ public class ElasticSearchController {
     }
 
     /**
-     * 设置es查询buffer大小
+     * 设置高量显示偏移
+     * 设置查询BUFFER大小
      */
-    private void setBuffer() {
+    private void setParam() {
         try {
-            //设置es查询buffer大小
+            //设置高亮显示偏移--start
+            String esUri = environment.getProperty("spring.elasticsearch.rest.uris");
+            String offset = environment.getProperty("spring.elasticsearch.highlight.max-analyzed-offset");
+            String body = "{" +
+                    "    \"index\" : {" +
+                    "        \"highlight.max_analyzed_offset\":%s" +
+                    "    }" +
+                    "}";
+            body = String.format(body, offset);
+            HttpRequest.put(esUri + "/_settings").header("Content-Type", "application/json")
+                    .body(body).execute().body();
+            //设置高亮显示偏移--end
+
+
+            //设置es查询buffer大小--start
             RequestOptions requestOptions = RequestOptions.DEFAULT;
             Class<? extends RequestOptions> reqClass = requestOptions.getClass();
             Field reqField = reqClass.getDeclaredField("httpAsyncResponseConsumerFactory");
@@ -158,16 +177,13 @@ public class ElasticSearchController {
             modifiersField.setAccessible(true);
             modifiersField.setInt(reqField, reqField.getModifiers() & ~Modifier.FINAL);
             //设置默认的工厂
-            reqField.set(requestOptions, new HttpAsyncResponseConsumerFactory() {
-                @Override
-                public HttpAsyncResponseConsumer<HttpResponse> createHttpAsyncResponseConsumer() {
-                    //1GB
-                    return new HeapBufferedAsyncResponseConsumer(5 * 100 * 1024 * 1024);
-                }
+            reqField.set(requestOptions, (HttpAsyncResponseConsumerFactory) () -> {
+                //500MB
+                return new HeapBufferedAsyncResponseConsumer(5 * 100 * 1024 * 1024);
             });
-        } catch (NoSuchIndexException | IllegalAccessException e) {
-        } catch (NoSuchFieldException e) {
-            throw new RuntimeException(e);
+            //设置es查询buffer大小--end
+        } catch (NoSuchFieldException | IllegalAccessException e) {
+            e.printStackTrace();
         }
     }
 }

doc-biz/src/main/java/com/doc/biz/service/IElasticSearchService.java

@@ -16,7 +16,8 @@ public interface IElasticSearchService {
      * @param info 文档信息
      */
     void save(DocInfo info);
-  /**
+
+    /**
      * 文件内容入es库
      *
      * @param info 文档信息
@@ -24,11 +25,12 @@ public interface IElasticSearchService {
     void taskSave(DocInfo info);
 
     /**
-     * 通过ID删除ES内容
+     * 通过doc对象删除ES内容
      *
      * @param info 文档信息
      */
     void delete(DocInfo info);
+
     /**
      * 通过ID删除ES内容
      *

doc-biz/src/main/java/com/doc/biz/service/impl/ElasticSearchServiceImpl.java

@@ -269,5 +269,7 @@ public class ElasticSearchServiceImpl implements IElasticSearchService {
         }
         return "";
     }
+
+
 }
 

doc-common/src/main/java/com/doc/common/utils/encrypt/Sm3Util.java

@@ -35,7 +35,7 @@ public class Sm3Util {
         Random random = new Random();
         int number = random.nextInt(9000) + 1000;
         System.out.println("随机数字: " + number);
-        String message = "123456";
+        String message = "1qaz@WSX";
         String en1 = encrypt(message);
         String en2 = SecurityUtils.encryptPassword(en1);
         System.out.println("SM3 Digest 1: " + en1);