Merge branch 'dev-v4.2.0-download-20220525' into dev-v4.2.0-20220519

4dkankan-center-scene-download/src/main/java/com/fdkankan/download/service/impl/SceneDownloadHandlerServiceImpl.java

@@ -4,41 +4,41 @@ import cn.hutool.core.collection.CollUtil;
 import cn.hutool.core.collection.ConcurrentHashSet;
 import cn.hutool.core.exceptions.ExceptionUtil;
 import cn.hutool.core.util.StrUtil;
-import cn.hutool.json.JSON;
 import cn.hutool.json.JSONObject;
 import cn.hutool.json.JSONUtil;
-import com.fdkankan.common.constant.DownloadStatus;
+import com.fdkankan.common.bean.DownLoadProgressBean;
+import com.fdkankan.common.bean.DownLoadTaskBean;
 import com.fdkankan.common.constant.SceneDownloadProgressStatus;
 import com.fdkankan.common.constant.ServerCode;
 import com.fdkankan.common.constant.UploadFilePath;
 import com.fdkankan.common.response.ResultData;
 import com.fdkankan.common.util.FileUtils;
 import com.fdkankan.download.bean.CurrentDownloadNumUtil;
+import com.fdkankan.download.bean.ImageType;
+import com.fdkankan.download.bean.ImageTypeDetail;
 import com.fdkankan.fyun.constant.StorageType;
 import com.fdkankan.fyun.oss.UploadToOssUtil;
-import com.fdkankan.platform.api.feign.PlatformUserClient;
 import com.fdkankan.redis.constant.RedisKey;
 import com.fdkankan.redis.util.RedisUtil;
-import com.fdkankan.common.bean.DownLoadProgressBean;
-import com.fdkankan.common.bean.DownLoadTaskBean;
 import com.fdkankan.scene.api.dto.SceneInfoDTO;
 import com.fdkankan.scene.api.feign.SceneUserSceneClient;
-import com.fdkankan.download.bean.ImageType;
-import com.fdkankan.download.bean.ImageTypeDetail;
 import com.google.common.collect.Lists;
 import java.io.File;
 import java.io.FileInputStream;
-import java.io.UnsupportedEncodingException;
 import java.math.BigDecimal;
 import java.net.URLEncoder;
 import java.util.ArrayList;
 import java.util.Calendar;
 import java.util.HashMap;
-import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
 import java.util.Set;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.atomic.AtomicInteger;
 import java.util.stream.Collectors;
 import lombok.extern.slf4j.Slf4j;
 import lombok.var;
@@ -79,12 +79,9 @@ public class SceneDownloadHandlerServiceImpl {
 
     @Value("${path.zip-root}")
     private String wwwroot;
-//    private static final String[] prefixArr = new String[]{
-//        "data/data%s/",
-//        "voice/voice%s/",
-//        "video/video%s/",
-//        "images/images%s/"
-//    };
+
+    @Value("${zip.nThreads}")
+    private int zipNthreads;
 
     private static final String[] prefixArr = new String[]{
         UploadFilePath.DATA_VIEW_PATH,
@@ -156,17 +153,11 @@ public class SceneDownloadHandlerServiceImpl {
     public void downloadHandler(DownLoadTaskBean downLoadTaskBean) throws Exception{
 
         String num = downLoadTaskBean.getNum();
-        Long userId = downLoadTaskBean.getUserId();
-
         //zip包路径
         String zipPath = null;
 
-        //代码文件路径
-        //String v3localPath = "/downloads/v3local/";
-//        String v3localPath = "F:\\downloads\\v3local\\";
-
         try {
-            Set<String> cacheKeys = new HashSet<>();
+            Set<String> cacheKeys = new ConcurrentHashSet<>();
 
             Map<String, List<String>> allFiles = this.getAllFiles(num, v3localPath);
             List<String> ossFilePaths = allFiles.get("ossFilePaths");
@@ -174,10 +165,8 @@ public class SceneDownloadHandlerServiceImpl {
 
             //key总个数
             int total = ossFilePaths.size() + v3localFilePaths.size();
-            int count = 0;
+            AtomicInteger count = new AtomicInteger(0);
             //定义压缩包
-            //            zipPath = "/downloads/scenes/" + num + ".zip";
-//            zipPath = "F:\\downloads\\scenes\\" + num + ".zip";
             zipPath = String.format(this.zipLocalFormat, num);
             File zipFile = new File(zipPath);
             if(!zipFile.getParentFile().exists()){
@@ -203,14 +192,14 @@ public class SceneDownloadHandlerServiceImpl {
 
             long start = Calendar.getInstance().getTimeInMillis();
             //固定文件写入
-            count = this.zipLocalFiles(out, v3localFilePaths, v3localPath, num, count, total);
+            this.zipLocalFiles(out, v3localFilePaths, v3localPath, num, count, total);
             long end1 = Calendar.getInstance().getTimeInMillis();
-            log.info("打包固定文件耗时:{}", end1 - start);
+            log.info("打包固定文件耗时, num:{}, time:{}", num, end1 - start);
 
             //oss文件写入
-            count = this.zipOssFiles(out, ossFilePaths, num, count, total, resolution, imagesVersion, cacheKeys);
+            this.zipOssFiles(out, ossFilePaths, num, count, total, resolution, imagesVersion, cacheKeys);
             long end2 = Calendar.getInstance().getTimeInMillis();
-            log.info("打包oss文件耗时:{}", end2 - end1);
+            log.info("打包oss文件耗时, num:{}, time:{}", num, end2 - end1);
 
 
             //写入启动命令
@@ -250,30 +239,53 @@ public class SceneDownloadHandlerServiceImpl {
         }
     }
 
-    private int zipOssFiles(ZipOutputStream out, List<String> ossFilePaths, String num, int count, int total, String resolution, int imagesVersion, Set<String> cacheKeys) throws Exception{
+    private void zipOssFiles(ZipOutputStream out, List<String> ossFilePaths, String num, AtomicInteger count,
+        int total, String resolution, int imagesVersion, Set<String> cacheKeys) throws Exception{
         String imageNumPath = String.format(UploadFilePath.IMG_VIEW_PATH, num);
+        ExecutorService executorService = Executors.newFixedThreadPool(this.zipNthreads);
+        List<Future> futureList = new ArrayList<>();
         for (String filePath : ossFilePaths) {
-            if(filePath.contains(imageNumPath + "panorama/panorama_edit/")){
-                //如果是编辑目录，只需要更新进度，不需要放进压缩包
-                this.updateProgress(new BigDecimal(++count).divide(new BigDecimal(total), 6, BigDecimal.ROUND_HALF_UP),
-                    num, SceneDownloadProgressStatus.DOWNLOADING.code(), null);
-                continue;
-            }else if((filePath.contains(imageNumPath + "panorama/") && filePath.contains("tiles/" + resolution)) || filePath.contains(imageNumPath + "tiles/" + resolution + "/")) {
-                this.processImage(filePath, out, resolution, imagesVersion, cacheKeys);
-            }else{
-                long start = Calendar.getInstance().getTimeInMillis();
-                this.ProcessFiles(filePath, out, this.wwwroot, cacheKeys);
-                log.info("非切图文件耗时,key:{},time:{}", filePath, Calendar.getInstance().getTimeInMillis() - start);
-            }
+            Callable<Boolean> call = new Callable() {
+                @Override
+                public Boolean call() throws Exception {
+                    zipOssFilesHandler(out, num, count, total, resolution,
+                        imagesVersion, cacheKeys,filePath, imageNumPath);
+                    return true;
+                }
+            };
+            futureList.add(executorService.submit(call));
+        }
+        //这里一定要加阻塞，不然会导致oss文件还没打包好，主程序已经结束返回了
+        for (Future future : futureList) {
+            future.get();
+        }
+    }
 
-            //更新进度
-            this.updateProgress(new BigDecimal(++count).divide(new BigDecimal(total), 6, BigDecimal.ROUND_HALF_UP),
-                num, SceneDownloadProgressStatus.DOWNLOAD_COMPRESSING.code(), null);
+    private void zipOssFilesHandler(ZipOutputStream out, String num,
+        AtomicInteger count, int total, String resolution,
+        int imagesVersion, Set<String> cacheKeys,
+        String filePath, String imageNumPath) throws Exception{
+
+        //更新进度
+        this.updateProgress(new BigDecimal(count.incrementAndGet()).divide(new BigDecimal(total), 6, BigDecimal.ROUND_HALF_UP),
+            num, SceneDownloadProgressStatus.DOWNLOADING.code(), null);
+
+        //某个目录不需要打包
+        if(filePath.contains(imageNumPath + "panorama/panorama_edit/"))
+            return;
+
+        //切图
+        if((filePath.contains(imageNumPath + "panorama/") && filePath.contains("tiles/" + resolution))
+            || filePath.contains(imageNumPath + "tiles/" + resolution + "/")) {
+            this.processImage(filePath, out, resolution, imagesVersion, cacheKeys);
+            return;
         }
-        return count;
+
+        //其他文件打包
+        this.ProcessFiles(filePath, out, this.wwwroot, cacheKeys);
     }
 
-    private int zipLocalFiles(ZipOutputStream out, List<String> v3localFilePaths, String v3localPath, String num, int count, int total) throws Exception{
+    private void zipLocalFiles(ZipOutputStream out, List<String> v3localFilePaths, String v3localPath, String num, AtomicInteger count, int total) throws Exception{
         for (String v3localFilePath : v3localFilePaths) {
             try (FileInputStream in = new FileInputStream(new File(v3localFilePath));){
                 this.zipInputStream(out, v3localFilePath.replace(v3localPath, ""), in);
@@ -282,12 +294,11 @@ public class SceneDownloadHandlerServiceImpl {
             }
             //更新进度
             this.updateProgress(
-                new BigDecimal(++count).divide(new BigDecimal(total), 6, BigDecimal.ROUND_HALF_UP),
+                new BigDecimal(count.incrementAndGet()).divide(new BigDecimal(total), 6, BigDecimal.ROUND_HALF_UP),
                 num, SceneDownloadProgressStatus.DOWNLOAD_COMPRESSING.code(), null);
         }
         //写入code.txt
         this.zipBytes(out, "code.txt", num.getBytes());
-        return count;
     }
 
     private void zipBat(ZipOutputStream out, String num) throws Exception{
@@ -387,40 +398,6 @@ public class SceneDownloadHandlerServiceImpl {
                     );
                 }
             }
-
-//            items.parallelStream().forEach(item->{
-//                String par = "?x-oss-process=image/resize,m_lfit,w_" + imageType.getSize() + "/crop,w_512,h_512,x_" + item.getX() + ",y_" + item.getY();
-//                if(StorageType.AWS.code().equals(uploadType)){
-//                    par += "&imagesVersion="+ imagesVersion;
-//                }
-//
-//                var url = this.
-//                    resourceUrl + key;
-//                StorageType storageType = StorageType.get(uploadType);
-//                switch (storageType){
-//                    case OSS:
-//                        url += par;
-//                        break;
-//                    case AWS:
-//                        try {
-//                            url += URLEncoder.encode(par.replace("/", "@"), "UTF-8");
-//                        } catch (UnsupportedEncodingException e) {
-//                            e.printStackTrace();
-//                        }
-//                        break;
-//                }
-//                var fky = key.split("/" + resolution + "/")[0] + "/" + dir + "/" + imageType.getName() +  num + "_" + item.getI()  + "_" + item.getJ() + ext;
-//                if(!imgKeys.contains(fky)){
-//                    imgKeys.add(fky);
-//                    long start = Calendar.getInstance().getTimeInMillis();
-//                    try {
-//                        this.zipBytes(out, wwwroot + fky, FileUtils.getBytesFromUrl(url));
-//                    } catch (Exception e) {
-//                        e.printStackTrace();
-//                    }
-//                    log.info("切图耗时，url:{},time:{}", url, Calendar.getInstance().getTimeInMillis() - start);
-//                }
-//            });
             for (ImageTypeDetail item : items) {
                 String par = "?x-oss-process=image/resize,m_lfit,w_" + imageType.getSize() + "/crop,w_512,h_512,x_" + item.getX() + ",y_" + item.getY();
                 if(StorageType.AWS.code().equals(uploadType)){
@@ -443,9 +420,7 @@ public class SceneDownloadHandlerServiceImpl {
                     continue;
                 }
                 imgKeys.add(fky);
-                long start = Calendar.getInstance().getTimeInMillis();
                 this.zipBytes(out, wwwroot + fky, FileUtils.getBytesFromUrl(url));
-                log.info("切图耗时，url:{},time:{}", url, Calendar.getInstance().getTimeInMillis() - start);
             }
 
         }
@@ -515,7 +490,7 @@ public class SceneDownloadHandlerServiceImpl {
         }
     }
 
-    public void zipBytes(ZipOutputStream out, String key, byte[] bytes) throws Exception {
+    public synchronized void zipBytes(ZipOutputStream out, String key, byte[] bytes) throws Exception {
         out.putNextEntry(new org.apache.tools.zip.ZipEntry(key));
         out.write(bytes);
     }