场景下载速度优化

4dkankan-center-scene-download/src/main/java/com/fdkankan/download/service/impl/SceneDownloadHandlerServiceImpl.java

@@ -39,6 +39,11 @@ import java.util.List;
 import java.util.Map;
 import java.util.Objects;
 import java.util.Set;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.atomic.AtomicInteger;
 import java.util.stream.Collectors;
 import lombok.extern.slf4j.Slf4j;
 import lombok.var;
@@ -166,7 +171,7 @@ public class SceneDownloadHandlerServiceImpl {
 //        String v3localPath = "F:\\downloads\\v3local\\";
 
         try {
-            Set<String> cacheKeys = new HashSet<>();
+            Set<String> cacheKeys = new ConcurrentHashSet<>();
 
             Map<String, List<String>> allFiles = this.getAllFiles(num, v3localPath);
             List<String> ossFilePaths = allFiles.get("ossFilePaths");
@@ -174,7 +179,7 @@ public class SceneDownloadHandlerServiceImpl {
 
             //key总个数
             int total = ossFilePaths.size() + v3localFilePaths.size();
-            int count = 0;
+            AtomicInteger count = new AtomicInteger(0);
             //定义压缩包
             //            zipPath = "/downloads/scenes/" + num + ".zip";
 //            zipPath = "F:\\downloads\\scenes\\" + num + ".zip";
@@ -203,12 +208,12 @@ public class SceneDownloadHandlerServiceImpl {
 
             long start = Calendar.getInstance().getTimeInMillis();
             //固定文件写入
-            count = this.zipLocalFiles(out, v3localFilePaths, v3localPath, num, count, total);
+            this.zipLocalFiles(out, v3localFilePaths, v3localPath, num, count, total);
             long end1 = Calendar.getInstance().getTimeInMillis();
             log.info("打包固定文件耗时:{}", end1 - start);
 
             //oss文件写入
-            count = this.zipOssFiles(out, ossFilePaths, num, count, total, resolution, imagesVersion, cacheKeys);
+            this.zipOssFiles(out, ossFilePaths, num, count, total, resolution, imagesVersion, cacheKeys);
             long end2 = Calendar.getInstance().getTimeInMillis();
             log.info("打包oss文件耗时:{}", end2 - end1);
 
@@ -250,30 +255,64 @@ public class SceneDownloadHandlerServiceImpl {
         }
     }
 
-    private int zipOssFiles(ZipOutputStream out, List<String> ossFilePaths, String num, int count, int total, String resolution, int imagesVersion, Set<String> cacheKeys) throws Exception{
+    private void zipOssFiles(ZipOutputStream out, List<String> ossFilePaths, String num, AtomicInteger count, int total, String resolution, int imagesVersion, Set<String> cacheKeys) throws Exception{
         String imageNumPath = String.format(UploadFilePath.IMG_VIEW_PATH, num);
+        ExecutorService executorService = Executors.newFixedThreadPool(10);
+        List<String> qtFilePaths = new ArrayList<>();
         for (String filePath : ossFilePaths) {
-            if(filePath.contains(imageNumPath + "panorama/panorama_edit/")){
-                //如果是编辑目录，只需要更新进度，不需要放进压缩包
-                this.updateProgress(new BigDecimal(++count).divide(new BigDecimal(total), 6, BigDecimal.ROUND_HALF_UP),
-                    num, SceneDownloadProgressStatus.DOWNLOADING.code(), null);
-                continue;
-            }else if((filePath.contains(imageNumPath + "panorama/") && filePath.contains("tiles/" + resolution)) || filePath.contains(imageNumPath + "tiles/" + resolution + "/")) {
-                this.processImage(filePath, out, resolution, imagesVersion, cacheKeys);
-            }else{
-                long start = Calendar.getInstance().getTimeInMillis();
-                this.ProcessFiles(filePath, out, this.wwwroot, cacheKeys);
-                log.info("非切图文件耗时,key:{},time:{}", filePath, Calendar.getInstance().getTimeInMillis() - start);
-            }
+            Callable<Boolean> call = new Callable() {
+                @Override
+                public Boolean call() throws Exception {
+                    test(out, ossFilePaths, num, count, total, resolution, imagesVersion, cacheKeys,filePath, imageNumPath);
+                    return true;
+                }
+            };
+            executorService.submit(call);
+        }
+//        List<Future> futureList = new ArrayList<>();
+//        for (String filePath : qtFilePaths) {
+//            Callable<Boolean> call = new Callable() {
+//                @Override
+//                public Boolean call() throws Exception {
+//                    processImage(filePath, out, resolution, imagesVersion, cacheKeys);
+//                    //更新进度
+//                    updateProgress(new BigDecimal(count.incrementAndGet()).divide(new BigDecimal(total), 6, BigDecimal.ROUND_HALF_UP),
+//                        num, SceneDownloadProgressStatus.DOWNLOAD_COMPRESSING.code(), null);
+//                    return true;
+//                }
+//            };
+//            futureList.add(executorService.submit(call));
+//        }
+//        for (Future future : futureList) {
+//            future.get();
+//        }
+    }
 
+    private void test(ZipOutputStream out, List<String> ossFilePaths,
+        String num, AtomicInteger count, int total, String resolution,
+        int imagesVersion, Set<String> cacheKeys,
+        String filePath, String imageNumPath) throws Exception{
+        if(filePath.contains(imageNumPath + "panorama/panorama_edit/")){
+            //如果是编辑目录，只需要更新进度，不需要放进压缩包
+            this.updateProgress(new BigDecimal(count.incrementAndGet()).divide(new BigDecimal(total), 6, BigDecimal.ROUND_HALF_UP),
+                num, SceneDownloadProgressStatus.DOWNLOADING.code(), null);
+        }else if((filePath.contains(imageNumPath + "panorama/") && filePath.contains("tiles/" + resolution)) || filePath.contains(imageNumPath + "tiles/" + resolution + "/")) {
+//                qtFilePaths.add(filePath);
+            processImage(filePath, out, resolution, imagesVersion, cacheKeys);
             //更新进度
-            this.updateProgress(new BigDecimal(++count).divide(new BigDecimal(total), 6, BigDecimal.ROUND_HALF_UP),
+            updateProgress(new BigDecimal(count.incrementAndGet()).divide(new BigDecimal(total), 6, BigDecimal.ROUND_HALF_UP),
+                num, SceneDownloadProgressStatus.DOWNLOAD_COMPRESSING.code(), null);
+        }else{
+            long start = Calendar.getInstance().getTimeInMillis();
+            this.ProcessFiles(filePath, out, this.wwwroot, cacheKeys);
+            log.info("非切图文件耗时,key:{},time:{}", filePath, Calendar.getInstance().getTimeInMillis() - start);
+            //更新进度
+            this.updateProgress(new BigDecimal(count.incrementAndGet()).divide(new BigDecimal(total), 6, BigDecimal.ROUND_HALF_UP),
                 num, SceneDownloadProgressStatus.DOWNLOAD_COMPRESSING.code(), null);
         }
-        return count;
     }
 
-    private int zipLocalFiles(ZipOutputStream out, List<String> v3localFilePaths, String v3localPath, String num, int count, int total) throws Exception{
+    private void zipLocalFiles(ZipOutputStream out, List<String> v3localFilePaths, String v3localPath, String num, AtomicInteger count, int total) throws Exception{
         for (String v3localFilePath : v3localFilePaths) {
             try (FileInputStream in = new FileInputStream(new File(v3localFilePath));){
                 this.zipInputStream(out, v3localFilePath.replace(v3localPath, ""), in);
@@ -282,12 +321,11 @@ public class SceneDownloadHandlerServiceImpl {
             }
             //更新进度
             this.updateProgress(
-                new BigDecimal(++count).divide(new BigDecimal(total), 6, BigDecimal.ROUND_HALF_UP),
+                new BigDecimal(count.incrementAndGet()).divide(new BigDecimal(total), 6, BigDecimal.ROUND_HALF_UP),
                 num, SceneDownloadProgressStatus.DOWNLOAD_COMPRESSING.code(), null);
         }
         //写入code.txt
         this.zipBytes(out, "code.txt", num.getBytes());
-        return count;
     }
 
     private void zipBat(ZipOutputStream out, String num) throws Exception{
@@ -387,40 +425,6 @@ public class SceneDownloadHandlerServiceImpl {
                     );
                 }
             }
-
-//            items.parallelStream().forEach(item->{
-//                String par = "?x-oss-process=image/resize,m_lfit,w_" + imageType.getSize() + "/crop,w_512,h_512,x_" + item.getX() + ",y_" + item.getY();
-//                if(StorageType.AWS.code().equals(uploadType)){
-//                    par += "&imagesVersion="+ imagesVersion;
-//                }
-//
-//                var url = this.
-//                    resourceUrl + key;
-//                StorageType storageType = StorageType.get(uploadType);
-//                switch (storageType){
-//                    case OSS:
-//                        url += par;
-//                        break;
-//                    case AWS:
-//                        try {
-//                            url += URLEncoder.encode(par.replace("/", "@"), "UTF-8");
-//                        } catch (UnsupportedEncodingException e) {
-//                            e.printStackTrace();
-//                        }
-//                        break;
-//                }
-//                var fky = key.split("/" + resolution + "/")[0] + "/" + dir + "/" + imageType.getName() +  num + "_" + item.getI()  + "_" + item.getJ() + ext;
-//                if(!imgKeys.contains(fky)){
-//                    imgKeys.add(fky);
-//                    long start = Calendar.getInstance().getTimeInMillis();
-//                    try {
-//                        this.zipBytes(out, wwwroot + fky, FileUtils.getBytesFromUrl(url));
-//                    } catch (Exception e) {
-//                        e.printStackTrace();
-//                    }
-//                    log.info("切图耗时，url:{},time:{}", url, Calendar.getInstance().getTimeInMillis() - start);
-//                }
-//            });
             for (ImageTypeDetail item : items) {
                 String par = "?x-oss-process=image/resize,m_lfit,w_" + imageType.getSize() + "/crop,w_512,h_512,x_" + item.getX() + ",y_" + item.getY();
                 if(StorageType.AWS.code().equals(uploadType)){
@@ -515,7 +519,7 @@ public class SceneDownloadHandlerServiceImpl {
         }
     }
 
-    public void zipBytes(ZipOutputStream out, String key, byte[] bytes) throws Exception {
+    public synchronized void zipBytes(ZipOutputStream out, String key, byte[] bytes) throws Exception {
         out.putNextEntry(new org.apache.tools.zip.ZipEntry(key));
         out.write(bytes);
     }