场景下载代码优化

4dkankan-center-scene-download/src/main/java/com/fdkankan/download/service/impl/SceneDownloadHandlerServiceImpl.java

@@ -4,37 +4,32 @@ import cn.hutool.core.collection.CollUtil;
 import cn.hutool.core.collection.ConcurrentHashSet;
 import cn.hutool.core.exceptions.ExceptionUtil;
 import cn.hutool.core.util.StrUtil;
-import cn.hutool.json.JSON;
 import cn.hutool.json.JSONObject;
 import cn.hutool.json.JSONUtil;
-import com.fdkankan.common.constant.DownloadStatus;
+import com.fdkankan.common.bean.DownLoadProgressBean;
+import com.fdkankan.common.bean.DownLoadTaskBean;
 import com.fdkankan.common.constant.SceneDownloadProgressStatus;
 import com.fdkankan.common.constant.ServerCode;
 import com.fdkankan.common.constant.UploadFilePath;
 import com.fdkankan.common.response.ResultData;
 import com.fdkankan.common.util.FileUtils;
 import com.fdkankan.download.bean.CurrentDownloadNumUtil;
+import com.fdkankan.download.bean.ImageType;
+import com.fdkankan.download.bean.ImageTypeDetail;
 import com.fdkankan.fyun.constant.StorageType;
 import com.fdkankan.fyun.oss.UploadToOssUtil;
-import com.fdkankan.platform.api.feign.PlatformUserClient;
 import com.fdkankan.redis.constant.RedisKey;
 import com.fdkankan.redis.util.RedisUtil;
-import com.fdkankan.common.bean.DownLoadProgressBean;
-import com.fdkankan.common.bean.DownLoadTaskBean;
 import com.fdkankan.scene.api.dto.SceneInfoDTO;
 import com.fdkankan.scene.api.feign.SceneUserSceneClient;
-import com.fdkankan.download.bean.ImageType;
-import com.fdkankan.download.bean.ImageTypeDetail;
 import com.google.common.collect.Lists;
 import java.io.File;
 import java.io.FileInputStream;
-import java.io.UnsupportedEncodingException;
 import java.math.BigDecimal;
 import java.net.URLEncoder;
 import java.util.ArrayList;
 import java.util.Calendar;
 import java.util.HashMap;
-import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
@@ -84,12 +79,9 @@ public class SceneDownloadHandlerServiceImpl {
 
     @Value("${path.zip-root}")
     private String wwwroot;
-//    private static final String[] prefixArr = new String[]{
-//        "data/data%s/",
-//        "voice/voice%s/",
-//        "video/video%s/",
-//        "images/images%s/"
-//    };
+
+    @Value("${zip.nThreads}")
+    private int zipNthreads;
 
     private static final String[] prefixArr = new String[]{
         UploadFilePath.DATA_VIEW_PATH,
@@ -161,15 +153,9 @@ public class SceneDownloadHandlerServiceImpl {
     public void downloadHandler(DownLoadTaskBean downLoadTaskBean) throws Exception{
 
         String num = downLoadTaskBean.getNum();
-        Long userId = downLoadTaskBean.getUserId();
-
         //zip包路径
         String zipPath = null;
 
-        //代码文件路径
-        //String v3localPath = "/downloads/v3local/";
-//        String v3localPath = "F:\\downloads\\v3local\\";
-
         try {
             Set<String> cacheKeys = new ConcurrentHashSet<>();
 
@@ -181,8 +167,6 @@ public class SceneDownloadHandlerServiceImpl {
             int total = ossFilePaths.size() + v3localFilePaths.size();
             AtomicInteger count = new AtomicInteger(0);
             //定义压缩包
-            //            zipPath = "/downloads/scenes/" + num + ".zip";
-//            zipPath = "F:\\downloads\\scenes\\" + num + ".zip";
             zipPath = String.format(this.zipLocalFormat, num);
             File zipFile = new File(zipPath);
             if(!zipFile.getParentFile().exists()){
@@ -210,12 +194,12 @@ public class SceneDownloadHandlerServiceImpl {
             //固定文件写入
             this.zipLocalFiles(out, v3localFilePaths, v3localPath, num, count, total);
             long end1 = Calendar.getInstance().getTimeInMillis();
-            log.info("打包固定文件耗时:{}", end1 - start);
+            log.info("打包固定文件耗时, num:{}, time:{}", num, end1 - start);
 
             //oss文件写入
             this.zipOssFiles(out, ossFilePaths, num, count, total, resolution, imagesVersion, cacheKeys);
             long end2 = Calendar.getInstance().getTimeInMillis();
-            log.info("打包oss文件耗时:{}", end2 - end1);
+            log.info("打包oss文件耗时, num:{}, time:{}", num, end2 - end1);
 
 
             //写入启动命令
@@ -255,62 +239,50 @@ public class SceneDownloadHandlerServiceImpl {
         }
     }
 
-    private void zipOssFiles(ZipOutputStream out, List<String> ossFilePaths, String num, AtomicInteger count, int total, String resolution, int imagesVersion, Set<String> cacheKeys) throws Exception{
+    private void zipOssFiles(ZipOutputStream out, List<String> ossFilePaths, String num, AtomicInteger count,
+        int total, String resolution, int imagesVersion, Set<String> cacheKeys) throws Exception{
         String imageNumPath = String.format(UploadFilePath.IMG_VIEW_PATH, num);
-        ExecutorService executorService = Executors.newFixedThreadPool(10);
-        List<String> qtFilePaths = new ArrayList<>();
+        ExecutorService executorService = Executors.newFixedThreadPool(this.zipNthreads);
         List<Future> futureList = new ArrayList<>();
         for (String filePath : ossFilePaths) {
             Callable<Boolean> call = new Callable() {
                 @Override
                 public Boolean call() throws Exception {
-                    test(out, ossFilePaths, num, count, total, resolution, imagesVersion, cacheKeys,filePath, imageNumPath);
+                    zipOssFilesHandler(out, num, count, total, resolution,
+                        imagesVersion, cacheKeys,filePath, imageNumPath);
                     return true;
                 }
             };
             futureList.add(executorService.submit(call));
         }
-//        List<Future> futureList = new ArrayList<>();
-//        for (String filePath : qtFilePaths) {
-//            Callable<Boolean> call = new Callable() {
-//                @Override
-//                public Boolean call() throws Exception {
-//                    processImage(filePath, out, resolution, imagesVersion, cacheKeys);
-//                    //更新进度
-//                    updateProgress(new BigDecimal(count.incrementAndGet()).divide(new BigDecimal(total), 6, BigDecimal.ROUND_HALF_UP),
-//                        num, SceneDownloadProgressStatus.DOWNLOAD_COMPRESSING.code(), null);
-//                    return true;
-//                }
-//            };
-//            futureList.add(executorService.submit(call));
-//        }
+        //这里一定要加阻塞，不然会导致oss文件还没打包好，主程序已经结束返回了
         for (Future future : futureList) {
             future.get();
         }
     }
 
-    private void test(ZipOutputStream out, List<String> ossFilePaths,
-        String num, AtomicInteger count, int total, String resolution,
+    private void zipOssFilesHandler(ZipOutputStream out, String num,
+        AtomicInteger count, int total, String resolution,
         int imagesVersion, Set<String> cacheKeys,
         String filePath, String imageNumPath) throws Exception{
-        if(filePath.contains(imageNumPath + "panorama/panorama_edit/")){
-            //如果是编辑目录，只需要更新进度，不需要放进压缩包
-            this.updateProgress(new BigDecimal(count.incrementAndGet()).divide(new BigDecimal(total), 6, BigDecimal.ROUND_HALF_UP),
-                num, SceneDownloadProgressStatus.DOWNLOADING.code(), null);
-        }else if((filePath.contains(imageNumPath + "panorama/") && filePath.contains("tiles/" + resolution)) || filePath.contains(imageNumPath + "tiles/" + resolution + "/")) {
-//                qtFilePaths.add(filePath);
-            processImage(filePath, out, resolution, imagesVersion, cacheKeys);
-            //更新进度
-            updateProgress(new BigDecimal(count.incrementAndGet()).divide(new BigDecimal(total), 6, BigDecimal.ROUND_HALF_UP),
-                num, SceneDownloadProgressStatus.DOWNLOAD_COMPRESSING.code(), null);
-        }else{
-            long start = Calendar.getInstance().getTimeInMillis();
-            this.ProcessFiles(filePath, out, this.wwwroot, cacheKeys);
-            log.info("非切图文件耗时,key:{},time:{}", filePath, Calendar.getInstance().getTimeInMillis() - start);
-            //更新进度
-            this.updateProgress(new BigDecimal(count.incrementAndGet()).divide(new BigDecimal(total), 6, BigDecimal.ROUND_HALF_UP),
-                num, SceneDownloadProgressStatus.DOWNLOAD_COMPRESSING.code(), null);
+
+        //更新进度
+        this.updateProgress(new BigDecimal(count.incrementAndGet()).divide(new BigDecimal(total), 6, BigDecimal.ROUND_HALF_UP),
+            num, SceneDownloadProgressStatus.DOWNLOADING.code(), null);
+
+        //某个目录不需要打包
+        if(filePath.contains(imageNumPath + "panorama/panorama_edit/"))
+            return;
+
+        //切图
+        if((filePath.contains(imageNumPath + "panorama/") && filePath.contains("tiles/" + resolution))
+            || filePath.contains(imageNumPath + "tiles/" + resolution + "/")) {
+            this.processImage(filePath, out, resolution, imagesVersion, cacheKeys);
+            return;
         }
+
+        //其他文件打包
+        this.ProcessFiles(filePath, out, this.wwwroot, cacheKeys);
     }
 
     private void zipLocalFiles(ZipOutputStream out, List<String> v3localFilePaths, String v3localPath, String num, AtomicInteger count, int total) throws Exception{
@@ -448,9 +420,7 @@ public class SceneDownloadHandlerServiceImpl {
                     continue;
                 }
                 imgKeys.add(fky);
-                long start = Calendar.getInstance().getTimeInMillis();
                 this.zipBytes(out, wwwroot + fky, FileUtils.getBytesFromUrl(url));
-                log.info("切图耗时，url:{},time:{}", url, Calendar.getInstance().getTimeInMillis() - start);
             }
 
         }