场景下载速度优化

4dkankan-center-scene-download/src/main/java/com/fdkankan/download/service/impl/SceneDownloadHandlerServiceImpl.java

@@ -39,10 +39,6 @@ import java.util.List;
 import java.util.Map;
 import java.util.Objects;
 import java.util.Set;
-import java.util.concurrent.Callable;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.concurrent.atomic.AtomicInteger;
 import java.util.stream.Collectors;
 import lombok.extern.slf4j.Slf4j;
 import lombok.var;
@@ -67,7 +63,7 @@ import org.springframework.web.client.RestTemplate;
 @Service
 public class SceneDownloadHandlerServiceImpl {
 
-//    @Autowired
+    //    @Autowired
 //    private PlatformUserClient platformUserClient;
     @Autowired
     private SceneUserSceneClient sceneUserSceneClient;
@@ -170,7 +166,7 @@ public class SceneDownloadHandlerServiceImpl {
 //        String v3localPath = "F:\\downloads\\v3local\\";
 
         try {
-            Set<String> cacheKeys = new ConcurrentHashSet<>();
+            Set<String> cacheKeys = new HashSet<>();
 
             Map<String, List<String>> allFiles = this.getAllFiles(num, v3localPath);
             List<String> ossFilePaths = allFiles.get("ossFilePaths");
@@ -178,7 +174,7 @@ public class SceneDownloadHandlerServiceImpl {
 
             //key总个数
             int total = ossFilePaths.size() + v3localFilePaths.size();
-            AtomicInteger count = new AtomicInteger(0);
+            int count = 0;
             //定义压缩包
             //            zipPath = "/downloads/scenes/" + num + ".zip";
 //            zipPath = "F:\\downloads\\scenes\\" + num + ".zip";
@@ -207,12 +203,12 @@ public class SceneDownloadHandlerServiceImpl {
 
             long start = Calendar.getInstance().getTimeInMillis();
             //固定文件写入
-            this.zipLocalFiles(out, v3localFilePaths, v3localPath, num, count, total);
+            count = this.zipLocalFiles(out, v3localFilePaths, v3localPath, num, count, total);
             long end1 = Calendar.getInstance().getTimeInMillis();
             log.info("打包固定文件耗时:{}", end1 - start);
 
             //oss文件写入
-            this.zipOssFiles(out, ossFilePaths, num, count, total, resolution, imagesVersion, cacheKeys);
+            count = this.zipOssFiles(out, ossFilePaths, num, count, total, resolution, imagesVersion, cacheKeys);
             long end2 = Calendar.getInstance().getTimeInMillis();
             log.info("打包oss文件耗时:{}", end2 - end1);
 
@@ -254,40 +250,30 @@ public class SceneDownloadHandlerServiceImpl {
         }
     }
 
-    private void zipOssFiles(ZipOutputStream out, List<String> ossFilePaths, String num, AtomicInteger count, int total, String resolution, int imagesVersion, Set<String> cacheKeys) throws Exception{
+    private int zipOssFiles(ZipOutputStream out, List<String> ossFilePaths, String num, int count, int total, String resolution, int imagesVersion, Set<String> cacheKeys) throws Exception{
         String imageNumPath = String.format(UploadFilePath.IMG_VIEW_PATH, num);
-        ExecutorService executorService = Executors.newFixedThreadPool(10);
         for (String filePath : ossFilePaths) {
             if(filePath.contains(imageNumPath + "panorama/panorama_edit/")){
                 //如果是编辑目录，只需要更新进度，不需要放进压缩包
-                this.updateProgress(new BigDecimal(count.incrementAndGet()).divide(new BigDecimal(total), 6, BigDecimal.ROUND_HALF_UP),
+                this.updateProgress(new BigDecimal(++count).divide(new BigDecimal(total), 6, BigDecimal.ROUND_HALF_UP),
                     num, SceneDownloadProgressStatus.DOWNLOADING.code(), null);
+                continue;
             }else if((filePath.contains(imageNumPath + "panorama/") && filePath.contains("tiles/" + resolution)) || filePath.contains(imageNumPath + "tiles/" + resolution + "/")) {
-                Callable<Boolean> call = new Callable() {
-                    @Override
-                    public Boolean call() throws Exception {
-                        processImage(filePath, out, resolution, imagesVersion, cacheKeys);
-                        //更新进度
-                        updateProgress(new BigDecimal(count.incrementAndGet()).divide(new BigDecimal(total), 6, BigDecimal.ROUND_HALF_UP),
-                            num, SceneDownloadProgressStatus.DOWNLOAD_COMPRESSING.code(), null);
-                        return true;
-                    }
-                };
-                executorService.submit(call);
+                this.processImage(filePath, out, resolution, imagesVersion, cacheKeys);
             }else{
                 long start = Calendar.getInstance().getTimeInMillis();
                 this.ProcessFiles(filePath, out, this.wwwroot, cacheKeys);
                 log.info("非切图文件耗时,key:{},time:{}", filePath, Calendar.getInstance().getTimeInMillis() - start);
-                //更新进度
-                this.updateProgress(new BigDecimal(count.incrementAndGet()).divide(new BigDecimal(total), 6, BigDecimal.ROUND_HALF_UP),
-                    num, SceneDownloadProgressStatus.DOWNLOAD_COMPRESSING.code(), null);
             }
 
-
+            //更新进度
+            this.updateProgress(new BigDecimal(++count).divide(new BigDecimal(total), 6, BigDecimal.ROUND_HALF_UP),
+                num, SceneDownloadProgressStatus.DOWNLOAD_COMPRESSING.code(), null);
         }
+        return count;
     }
 
-    private void zipLocalFiles(ZipOutputStream out, List<String> v3localFilePaths, String v3localPath, String num, AtomicInteger count, int total) throws Exception{
+    private int zipLocalFiles(ZipOutputStream out, List<String> v3localFilePaths, String v3localPath, String num, int count, int total) throws Exception{
         for (String v3localFilePath : v3localFilePaths) {
             try (FileInputStream in = new FileInputStream(new File(v3localFilePath));){
                 this.zipInputStream(out, v3localFilePath.replace(v3localPath, ""), in);
@@ -296,11 +282,12 @@ public class SceneDownloadHandlerServiceImpl {
             }
             //更新进度
             this.updateProgress(
-                new BigDecimal(count.incrementAndGet()).divide(new BigDecimal(total), 6, BigDecimal.ROUND_HALF_UP),
+                new BigDecimal(++count).divide(new BigDecimal(total), 6, BigDecimal.ROUND_HALF_UP),
                 num, SceneDownloadProgressStatus.DOWNLOAD_COMPRESSING.code(), null);
         }
         //写入code.txt
         this.zipBytes(out, "code.txt", num.getBytes());
+        return count;
     }
 
     private void zipBat(ZipOutputStream out, String num) throws Exception{
@@ -400,7 +387,8 @@ public class SceneDownloadHandlerServiceImpl {
                     );
                 }
             }
-            for (ImageTypeDetail item : items) {
+
+            items.parallelStream().forEach(item->{
                 String par = "?x-oss-process=image/resize,m_lfit,w_" + imageType.getSize() + "/crop,w_512,h_512,x_" + item.getX() + ",y_" + item.getY();
                 if(StorageType.AWS.code().equals(uploadType)){
                     par += "&imagesVersion="+ imagesVersion;
@@ -414,18 +402,51 @@ public class SceneDownloadHandlerServiceImpl {
                         url += par;
                         break;
                     case AWS:
-                        url += URLEncoder.encode(par.replace("/", "@"), "UTF-8");
+                        try {
+                            url += URLEncoder.encode(par.replace("/", "@"), "UTF-8");
+                        } catch (UnsupportedEncodingException e) {
+                            e.printStackTrace();
+                        }
                         break;
                 }
                 var fky = key.split("/" + resolution + "/")[0] + "/" + dir + "/" + imageType.getName() +  num + "_" + item.getI()  + "_" + item.getJ() + ext;
-                if(imgKeys.contains(fky)){
-                    continue;
+                if(!imgKeys.contains(fky)){
+                    imgKeys.add(fky);
+                    long start = Calendar.getInstance().getTimeInMillis();
+                    try {
+                        this.zipBytes(out, wwwroot + fky, FileUtils.getBytesFromUrl(url));
+                    } catch (Exception e) {
+                        e.printStackTrace();
+                    }
+                    log.info("切图耗时，url:{},time:{}", url, Calendar.getInstance().getTimeInMillis() - start);
                 }
-                imgKeys.add(fky);
-                long start = Calendar.getInstance().getTimeInMillis();
-                this.zipBytes(out, wwwroot + fky, FileUtils.getBytesFromUrl(url));
-                log.info("切图耗时，url:{},time:{}", url, Calendar.getInstance().getTimeInMillis() - start);
-            }
+            });
+//            for (ImageTypeDetail item : items) {
+//                String par = "?x-oss-process=image/resize,m_lfit,w_" + imageType.getSize() + "/crop,w_512,h_512,x_" + item.getX() + ",y_" + item.getY();
+//                if(StorageType.AWS.code().equals(uploadType)){
+//                    par += "&imagesVersion="+ imagesVersion;
+//                }
+//
+//                var url = this.
+//                    resourceUrl + key;
+//                StorageType storageType = StorageType.get(uploadType);
+//                switch (storageType){
+//                    case OSS:
+//                        url += par;
+//                        break;
+//                    case AWS:
+//                        url += URLEncoder.encode(par.replace("/", "@"), "UTF-8");
+//                        break;
+//                }
+//                var fky = key.split("/" + resolution + "/")[0] + "/" + dir + "/" + imageType.getName() +  num + "_" + item.getI()  + "_" + item.getJ() + ext;
+//                if(imgKeys.contains(fky)){
+//                    continue;
+//                }
+//                imgKeys.add(fky);
+//                long start = Calendar.getInstance().getTimeInMillis();
+//                this.zipBytes(out, wwwroot + fky, FileUtils.getBytesFromUrl(url));
+//                log.info("切图耗时，url:{},time:{}", url, Calendar.getInstance().getTimeInMillis() - start);
+//            }
 
         }