mongo全量备份恢复处理。

sync-common/src/main/java/com/jjt/common/utils/CompressZip.java

@@ -12,6 +12,7 @@ import org.apache.commons.lang3.StringUtils;
 import java.io.File;
 import java.io.FileNotFoundException;
 import java.nio.charset.Charset;
+import java.nio.charset.StandardCharsets;
 import java.util.ArrayList;
 import java.util.List;
 
@@ -126,45 +127,21 @@ public class CompressZip {
         unzip(zipFile, file, Charset.forName("GBK"), null);
     }
 
-//    /**
-//     * 预览压缩包文件
-//     *
-//     * @param zipFile 压缩包文件
-//     * @param charset 编码
-//     * @return 文件列表（不含子层文件夹（夹））
-//     */
-//    public static List<View> view(File zipFile, Charset charset) {
-//        ZipFile outZipFile = new ZipFile(zipFile);
-//        if (charset != null) {
-//            outZipFile.setCharset(charset);
-//        }
-//        List<FileHeader> fileHeaders;
-//        try {
-//            fileHeaders = outZipFile.getFileHeaders();
-//            List<View> resultList = new ArrayList<>();
-//            for (FileHeader header : fileHeaders) {
-//                View item = new View();
-//                item.setFileName(header.getFileName());
-//                item.setCompressedSize(header.getCompressedSize());
-//                item.setUncompressedSize(header.getUncompressedSize());
-//                item.setDirectory(header.isDirectory());
-//                item.setLastModifiedTime(header.getLastModifiedTime());
-//                resultList.add(item);
-//            }
-//            return resultList;
-//        } catch (ZipException e) {
-//            throw new RuntimeException(e);
-//        }
-//    }
-//
-//    /**
-//     * 预览压缩包文件
-//     *
-//     * @param zipFile 压缩包文件
-//     * @return 文件列表（不含子层文件夹（夹））
-//     */
-//    public static List<View> view(File zipFile) {
-//        return view(zipFile, Charset.forName("GBK"));
-//    }
+    /**
+     * 分卷压缩文件
+     *
+     * @param targetDir 待打包文件目录
+     * @param zipFile   打包文件名
+     * @param splitSize 分卷大小，单位byte
+     * @throws ZipException
+     */
+    public static void splitZip(File targetDir, ZipFile zipFile, long splitSize) throws ZipException {
+        ZipParameters parameter = new ZipParameters();
+        //压缩方式，使用JDK内置zip
+        parameter.setCompressionMethod(CompressionMethod.DEFLATE);
+
+        zipFile.setCharset(StandardCharsets.UTF_8);
+        zipFile.createSplitZipFileFromFolder(targetDir, parameter, true, splitSize);
+    }
 
 }

sync-in/src/main/java/com/jjt/in/service/IInMongoService.java

@@ -10,7 +10,8 @@ public interface IInMongoService {
     /**
      * 解析同步文件
      *
-     * @param dir 解析文件目录
+     * @param dir   解析文件目录
+     * @param isInc 是否增量
      */
-    public void parseSyncFile(String dir) throws Exception;
+    public void parseSyncFile(String dir, boolean isInc) throws Exception;
 }

sync-in/src/main/java/com/jjt/in/service/impl/InMongoServiceImpl.java

@@ -39,7 +39,7 @@ public class InMongoServiceImpl extends InBaseService implements IInMongoService
      * 解析同步文件
      */
     @Override
-    public void parseSyncFile(String dir) throws Exception {
+    public void parseSyncFile(String dir, boolean isInc) throws Exception {
         String params = sysConfigService.selectConfigByKey("in.mongo.info");
 
         JSONObject mongoInfo = JSONObject.parseObject(params);
@@ -49,7 +49,12 @@ public class InMongoServiceImpl extends InBaseService implements IInMongoService
             dir += "/";
         }
 
-        String filaName = dir + "local/oplog.rs.bson";
+        String filaName = dir;
+        if (isInc) {
+            //如果是增量，则需要指定 local/oplog.rs.bson
+            //如果是全量，只需指定目录即可
+            filaName += "local/oplog.rs.bson";
+        }
         try {
             String cmd = String.format("/usr/bin/mongorestore --host %s --port %s --drop --oplogReplay %s", host, port, filaName);
             LinuxCommand.exec(cmd);

sync-in/src/main/java/com/jjt/in/service/impl/InProcessServiceImpl.java

@@ -11,6 +11,7 @@ import com.jjt.in.service.IInMongoService;
 import com.jjt.in.service.IInProcessService;
 import com.jjt.in.service.IInSyncInfoService;
 import com.jjt.system.service.ISysConfigService;
+import net.lingala.zip4j.ZipFile;
 import org.apache.commons.codec.digest.DigestUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -112,9 +113,20 @@ public class InProcessServiceImpl extends InBaseService implements IInProcessSer
                                     break;
                                 case mongo:
                                     File mongoTargetDir = new File(tmpDir + "mongo/");
-                                    CompressZip.unzip(zipFile, mongoTargetDir);
+                                    ZipFile zf = new ZipFile(zipFile);
+                                    if (zf.isSplitArchive()) {
+                                        //如果是分卷压缩文件，并且分卷完整
+                                        if (zf.isValidZipFile()) {
+                                            zf.extractAll(mongoTargetDir.getPath());
+                                        } else {
+                                            throw new Exception("分卷不完整");
+                                        }
+                                    } else {
+                                        CompressZip.unzip(zipFile, mongoTargetDir);
+                                    }
                                     String mongoDir = tmpDir + "mongo/" + desc.getName().split("\\.")[0].split("-")[2];
-                                    mongoService.parseSyncFile(mongoDir);
+                                    boolean isInc = file.getName().indexOf("-78-") != -1;
+                                    mongoService.parseSyncFile(mongoDir, isInc);
                                     break;
                                 case mysql:
                                     handleMysqlType(zipFile);

sync-out/src/main/java/com/jjt/out/service/impl/OutMongoServiceImpl.java

@@ -12,6 +12,7 @@ import com.jjt.out.service.IOutMongoService;
 import com.jjt.out.service.IOutMysqlService;
 import com.jjt.out.service.IOutProcessInfoService;
 import com.jjt.system.service.ISysConfigService;
+import net.lingala.zip4j.ZipFile;
 import org.apache.commons.codec.digest.DigestUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -42,31 +43,22 @@ public class OutMongoServiceImpl extends OutBaseService implements IOutMongoServ
     @Resource
     private IOutProcessInfoService processInfoService;
 
-
-    /**
-     * 导出数据库
-     */
-    @Override
-    public void full() {
-
-    }
-
     /**
-     * 增量导出
+     * 执行备份
+     *
+     * @param isInc 是否增量 true:增量 false:全量
      */
-    @Override
-    public void inc() {
+    private void exec(boolean isInc) {
         String params = sysConfigService.selectConfigByKey("out.mongo.info");
 
         JSONObject mongoInfo = JSONObject.parseObject(params);
         String host = mongoInfo.getString("host");
         String port = mongoInfo.getString("port");
+        String dir = mongoInfo.getString("dir");
 
         String tmpDir = tmpDIr();
         OutProcessInfo opi = new OutProcessInfo();
         opi.setProcessType(SyncType.mongo.toString());
-        List<OutProcessInfo> list = processInfoService.selectOutProcessInfoList(opi);
-        String time = list.get(0).getProcessKey();
 
         String nowTime = DateUtils.dateTimeNow();
 
@@ -84,13 +76,18 @@ public class OutMongoServiceImpl extends OutBaseService implements IOutMongoServ
             //组装导出命令
             List<String> commands = new ArrayList<>();
             commands.add("/usr/bin/sh");
-            commands.add("mongo-inc-bak.sh");
+            if (isInc) {
+                //增量
+                commands.add("mongo-inc-bak.sh");
+            } else {
+                //全量
+                commands.add("mongo-full-bak.sh");
+            }
             commands.add(host);
             commands.add(port);
-            commands.add(time);
             commands.add(tmpDir);
 
-            LinuxCommand.exec(commands, "/mnt/");
+            LinuxCommand.exec(commands, dir);
 
             Date et = new Date();
             opi.setCostTime(et.getTime() - st.getTime());
@@ -106,14 +103,24 @@ public class OutMongoServiceImpl extends OutBaseService implements IOutMongoServ
 
             //打包目标目录
             File targetDir = new File(tmpDir);
-            File zipFile = new File(syncDir + zipName);
-            CompressZip.zip(targetDir, zipFile);
+            ZipFile zipFile = new ZipFile(syncDir + zipName);
+            /**
+             * 获取分卷大小
+             */
+            String size = sysConfigService.selectConfigByKey("file.split.size");
+            //GB转换成byte
+            long splitSize = 1024 * 1024 * 1024 * Integer.parseInt(size);
+            CompressZip.splitZip(targetDir, zipFile, splitSize);
             //打包文件--end
 
             //生成描述json文件--start
             try {
-                String descName = syncDir + "sync-70-" + nowTime + ".json";
-                String md5 = DigestUtils.md5Hex(Files.newInputStream(zipFile.toPath()));
+                String descName = syncDir + "sync-78-" + nowTime + ".json";
+                if (!isInc) {
+                    //如果是全量，执行顺序要靠前,内网解析时,顺序号77代表全量，78代表增量
+                    descName = syncDir + "sync-77-" + nowTime + ".json";
+                }
+                String md5 = DigestUtils.md5Hex(Files.newInputStream(zipFile.getFile().toPath()));
                 FileDesc desc = new FileDesc();
                 desc.setName(zipName);
                 desc.setMd5(md5);
@@ -129,4 +136,20 @@ public class OutMongoServiceImpl extends OutBaseService implements IOutMongoServ
             e.printStackTrace();
         }
     }
+
+    /**
+     * 全量备份
+     */
+    @Override
+    public void full() {
+        exec(false);
+    }
+
+    /**
+     * 增量备份
+     */
+    @Override
+    public void inc() {
+        exec(true);
+    }
 }

sync-out/src/main/java/com/jjt/out/service/impl/OutProcessServiceImpl.java

@@ -9,6 +9,7 @@ import com.jjt.out.domain.OutSyncInfo;
 import com.jjt.out.service.IOutProcessService;
 import com.jjt.out.service.IOutSyncInfoService;
 import com.jjt.system.service.ISysConfigService;
+import net.lingala.zip4j.ZipFile;
 import org.apache.commons.codec.digest.DigestUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -22,6 +23,7 @@ import java.nio.file.StandardCopyOption;
 import java.util.Arrays;
 import java.util.Comparator;
 import java.util.Date;
+import java.util.List;
 
 /**
  * 数据同步Service业务层处理
@@ -71,21 +73,31 @@ public class OutProcessServiceImpl extends OutBaseService implements IOutProcess
                         ObjectMapper mapper = new ObjectMapper();
                         FileDesc desc = mapper.readValue(file, FileDesc.class);
 
-                        File zipFile = new File(syncDir + desc.getName());
-                        String md5 = DigestUtils.md5Hex(Files.newInputStream(zipFile.toPath()));
+                        ZipFile zipFile = new ZipFile(syncDir + desc.getName());
+                        String md5 = DigestUtils.md5Hex(Files.newInputStream(zipFile.getFile().toPath()));
                         if (md5.equals(desc.getMd5())) {
                             Date start = new Date();
                             //上传json文件
                             boolean flag = ftpUtil.upload(file);
+                            boolean flag1 = false;
                             //上传zip文件
-                            boolean flag1 = ftpUtil.upload(zipFile);
+                            if (zipFile.isSplitArchive()) {
+                                //如果是分卷压缩文件，则需要读取所有文件名上传
+                                List<File> list = zipFile.getSplitZipFiles();
+                                for (File f : list) {
+                                    flag1 = flag && ftpUtil.upload(f);
+                                }
+                            } else {
+                                flag1 = ftpUtil.upload(zipFile.getFile());
+
+                            }
                             Date end = new Date();
 
                             String status = "N";
                             if (flag && flag1) {
                                 //移动文件到备份目录
                                 Files.move(file.toPath(), Paths.get(bakDir + file.getName()), StandardCopyOption.REPLACE_EXISTING);
-                                Files.move(zipFile.toPath(), Paths.get(bakDir + zipFile.getName()), StandardCopyOption.REPLACE_EXISTING);
+                                Files.move(zipFile.getFile().toPath(), Paths.get(bakDir + zipFile.getFile().getName()), StandardCopyOption.REPLACE_EXISTING);
                                 status = "Y";
                             }
 

sync-out/src/test/java/test/Zip4jTest.java

@@ -19,23 +19,23 @@ import java.util.List;
 @FixMethodOrder(MethodSorters.NAME_ASCENDING)
 public class Zip4jTest {
     public static void main(String[] args) throws ZipException {
-        ZipFile zipFile = new ZipFile("D:\\SYSTEM\\Desktop\\temp\\zip\\test.zip");
-        File file = new File("D:\\SYSTEM\\Desktop\\temp\\XNYQT");
+        ZipFile zipFile = new ZipFile("D:\\SYSTEM\\Desktop\\temp\\zip\\test1.zip");
+        File file = new File("D:\\SYSTEM\\Desktop\\temp\\ser");
         ZipParameters parameter = new ZipParameters();
         //压缩方式，使用JDK内置zip
         parameter.setCompressionMethod(CompressionMethod.DEFLATE);
         zipFile.setCharset(StandardCharsets.UTF_8);
         long splitSize = 1024 * 1024 * 10;
 
-        List<File> filesToAdd = Arrays.asList(
-                new File("D:\\SYSTEM\\Desktop\\temp\\test.mp4")
-        );
+//        List<File> filesToAdd = Arrays.asList(
+//                new File("D:\\SYSTEM\\Desktop\\temp\\test.mp4")
+//        );
         zipFile.createSplitZipFileFromFolder(file, parameter, true, splitSize);
     }
 
     @Test
     public void extract() throws ZipException {
-        String filePath = "D:\\SYSTEM\\Desktop\\temp\\zip\\test.zip";
+        String filePath = "D:\\SYSTEM\\Desktop\\temp\\sharding5.1.1.zip";
         ZipFile zipFile = new ZipFile(filePath);
         System.err.println(zipFile.isValidZipFile());
         System.err.println(zipFile.isSplitArchive());