Merge branch 'project-jmga-2.2.0-2.0.1' into project-jmga-dev

# Conflicts:
#	src/main/java/com/fdkankan/contro/mq/service/impl/BuildSceneServiceImpl.java

src/main/java/com/fdkankan/contro/entity/BuildLog.java

@@ -104,8 +104,4 @@ public class BuildLog implements Serializable {
     @TableField("result_queue_name")
     private String resultQuequeName;
 
-    @TableField("rec_status")
-    @TableLogic(value = "A", delval = "I")
-    private String recStatus;
-
 }

src/main/java/com/fdkankan/contro/mq/service/impl/BuildSceneServiceImpl.java

@@ -11,9 +11,11 @@ import cn.hutool.http.HttpUtil;
 import com.alibaba.fastjson.JSON;
 import com.alibaba.fastjson.JSONArray;
 import com.alibaba.fastjson.JSONObject;
+import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
 import com.baomidou.mybatisplus.core.conditions.update.LambdaUpdateWrapper;
 import com.fdkankan.common.constant.*;
 import com.fdkankan.common.exception.BusinessException;
+import com.fdkankan.common.util.DateExtUtil;
 import com.fdkankan.common.util.DateUtil;
 import com.fdkankan.common.util.FileUtils;
 import com.fdkankan.contro.bean.SyncLaserResultBean;
@@ -56,7 +58,6 @@ import java.io.IOException;
 import java.nio.charset.StandardCharsets;
 import java.util.*;
 import java.util.Map.Entry;
-import java.util.stream.Collectors;
 
 
 /**
@@ -169,21 +170,8 @@ public class BuildSceneServiceImpl implements IBuildSceneService {
         boolean success = false;
         String num = message.getSceneNum();
         String batchIds = (String) message.getExt().get("batchId");
-        String threeCamType = (String) message.getExt().get("threeCamType");
-        String filename = (String)message.getExt().get("filename");
         try {
 
-            //如果场景原始资源上传批次id不为空，则需要下载批次文件上传到oss目录
-            if(StrUtil.isNotEmpty(batchIds)){
-                for (String batchId : batchIds.split(",")) {
-                    if(StrUtil.isNotEmpty(threeCamType) && "yzl".equals(threeCamType)){
-                        this.downloadOrigFile4Yzl(batchId, message.getPath());
-                    }else{
-                        this.downloadOrigFile(batchId, message.getPath());
-                    }
-                }
-            }
-
             //重新计算时需要删除文件夹，否知使用缓存
             if(new File(message.getPath() + File.separator + "results").exists()){
                 FileUtils.deleteDirectory(message.getPath() + File.separator + "results");
@@ -317,6 +305,17 @@ public class BuildSceneServiceImpl implements IBuildSceneService {
                 reason = "前置处理异常";
             }
 
+            if(StrUtil.isNotEmpty(batchIds)){
+                List<OrigFileUploadBatch> list = origFileUploadBatchService.list(new LambdaQueryWrapper<OrigFileUploadBatch>().in(OrigFileUploadBatch::getBatchId, batchIds));
+                for (OrigFileUploadBatch origFileUploadBatch : list) {
+                    if(origFileUploadBatch.getCreateTime().before(DateExtUtil.offsetDay(new Date(), -1))){//一天之内重试，超过一天就认为此批次没有可用价值，置为异常状态
+                        origFileUploadBatch.setStatus(-1);
+                        origFileUploadBatch.setUpdateTime(new Date());
+                        origFileUploadBatchService.updateById(origFileUploadBatch);
+                    }
+                }
+            }
+
             jmgaService.updateSceneFailAndZipLog(num, reason);
 
             this.sendFailToLaser(num);
@@ -344,84 +343,6 @@ public class BuildSceneServiceImpl implements IBuildSceneService {
         }
     }
 
-
-
-    private void downloadOrigFile(String batchId, String dataSource){
-        if(StrUtil.isEmpty(batchId)){
-            return;
-        }
-
-        List<OrigFileUpload> fileList = origFileUploadService.getByBatchId(batchId);
-        if(CollUtil.isEmpty(fileList)){
-            return;
-        }
-        String homePath = "/oss/4dkankan/" + SceneUtil.getHomePath(dataSource);
-
-
-        for (OrigFileUpload origFileUpload : fileList) {
-            int times = 0;
-            String filePath = homePath.concat(origFileUpload.getFileName());
-            log.info("开始下载原始文件, url:{}, filePath:{}", origFileUpload.getFileUrl(), filePath);
-            do{
-                try {
-                    ++times;
-                    HttpUtil.downloadFile(origFileUpload.getFileUrl(), new File(filePath), 3 * 60 * 1000);
-                    if(FileUtil.exist(filePath)){
-                        break;
-                    }
-                }catch (Exception e){
-                    log.info("原始文件第{}次下载失败，fileUrl:{}, filePath:{}", times, origFileUpload.getFileUrl(), filePath);
-                }
-            }while (times < 3);
-            log.info("下载原始文件结束, url:{}, filePath:{}", origFileUpload.getFileUrl(), filePath);
-//            if(!FileUtil.exist(filePath)){
-//                throw new RuntimeException("原始文件下载失败，fileUrl:" + origFileUpload.getFileUrl() + ", filePath:" + filePath);
-//            }
-        }
-    }
-
-    private void downloadOrigFile4Yzl(String batchId, String dataSource){
-        if(StrUtil.isEmpty(batchId)){
-            return;
-        }
-
-        OrigFileUpload sceneUpData = origFileUploadService.getByBatchIdAndFileName(batchId, "scene_up_data.txt");
-        String sceneUpDataPath = dataSource + "/" + "scene_up_data.txt";
-        HttpUtilExt.downloadFileAndCheck(sceneUpData.getFileUrl(), sceneUpDataPath, 60000);
-        String sceneUpDataStr = FileUtil.readUtf8String(sceneUpDataPath);
-        JSONArray fileJsonArray = JSON.parseArray(sceneUpDataStr);
-        Map<String, String> fileMap = fileJsonArray.stream().collect(Collectors.toMap(v -> ((JSONObject) v).getString("fileName"), v -> ((JSONObject) v).getString("filePath")));
-
-        List<OrigFileUpload> fileList = origFileUploadService.getByBatchId(batchId);
-        if(CollUtil.isEmpty(fileList)){
-            return;
-        }
-        String homePath = "/oss/4dkankan/" + SceneUtil.getHomePath(dataSource);
-
-
-        for (OrigFileUpload origFileUpload : fileList) {
-            String relativeFilePath = fileMap.get(origFileUpload.getFileName());
-            if(StrUtil.isEmpty(relativeFilePath)){
-                continue;
-            }
-            int times = 0;
-            String filePath = homePath.concat(relativeFilePath);
-            log.info("开始下载原始文件, url:{}, filePath:{}", origFileUpload.getFileUrl(), filePath);
-            do{
-                try {
-                    ++times;
-                    HttpUtil.downloadFile(origFileUpload.getFileUrl(), new File(filePath), 3 * 60 * 1000);
-                    if(FileUtil.exist(filePath)){
-                        break;
-                    }
-                }catch (Exception e){
-                    log.info("原始文件第{}次下载失败，fileUrl:{}, filePath:{}", times, origFileUpload.getFileUrl(), filePath);
-                }
-            }while (times < 4);
-            log.info("下载原始文件结束, url:{}, filePath:{}", origFileUpload.getFileUrl(), filePath);
-        }
-    }
-
     private String getOssPath(String path) {
         String ossPath = ConstantFilePath.OSS_PREFIX
                 + path.replace(ConstantFilePath.BUILD_MODEL_PATH, "")

src/main/java/com/fdkankan/contro/mq/service/impl/CallBuildServiceImpl.java

@@ -1,14 +1,22 @@
 package com.fdkankan.contro.mq.service.impl;
 
 import cn.hutool.core.collection.CollUtil;
+import cn.hutool.core.io.FileUtil;
+import cn.hutool.core.util.StrUtil;
+import cn.hutool.http.HttpUtil;
 import com.alibaba.fastjson.JSON;
+import com.alibaba.fastjson.JSONArray;
+import com.alibaba.fastjson.JSONObject;
 import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
+import com.fdkankan.contro.entity.OrigFileUpload;
 import com.fdkankan.contro.entity.OrigFileUploadBatch;
 import com.fdkankan.contro.entity.ScenePlus;
 import com.fdkankan.contro.mq.service.CallBuildService;
 import com.fdkankan.contro.service.IOrigFileUploadBatchService;
 import com.fdkankan.contro.service.IOrigFileUploadService;
 import com.fdkankan.contro.service.IScenePlusService;
+import com.fdkankan.contro.util.HttpUtilExt;
+import com.fdkankan.model.utils.SceneUtil;
 import com.fdkankan.rabbitmq.bean.BuildSceneCallMessage;
 import com.fdkankan.rabbitmq.util.RabbitMqProducer;
 import com.fdkankan.redis.util.RedisLockUtil;
@@ -19,7 +27,9 @@ import org.springframework.beans.factory.annotation.Value;
 import org.springframework.stereotype.Service;
 
 import javax.annotation.Resource;
+import java.io.File;
 import java.util.*;
+import java.util.stream.Collectors;
 
 @Slf4j
 @Service
@@ -67,7 +77,7 @@ public class CallBuildServiceImpl implements CallBuildService {
         }
         for (String uuid : map.keySet()) {
             ScenePlus scenePlus = scenePlusService.getByUuid(uuid);
-            boolean lock = redisLockUtil.lock("push-modeling-pre:" + scenePlus.getNum(), scenePlus.getNum(), 20*60);
+            boolean lock = redisLockUtil.lock("push-modeling-pre:" + scenePlus.getNum(), scenePlus.getNum(), 30*60);
             if(!lock){
                 continue;
             }
@@ -83,6 +93,19 @@ public class CallBuildServiceImpl implements CallBuildService {
                 }
                 BuildSceneCallMessage mqMessage = JSON.parseObject(map.get(uuid).getLast().getMqContent(), BuildSceneCallMessage.class);
                 mqMessage.getExt().put("batchId", batchIdStr.substring(1));
+
+                String threeCamType = (String) mqMessage.getExt().get("threeCamType");
+                //如果场景原始资源上传批次id不为空，则需要下载批次文件上传到oss目录
+                if(StrUtil.isNotEmpty(batchIdStr)){
+                    for (String batchId : batchIdStr.substring(1).split(",")) {
+                        if(StrUtil.isNotEmpty(threeCamType) && "yzl".equals(threeCamType)){
+                            this.downloadOrigFile4Yzl(batchId, mqMessage.getPath());
+                        }else{
+                            this.downloadOrigFile(batchId, mqMessage.getPath());
+                        }
+                    }
+                }
+
                 rabbitMqProducer.sendByWorkQueue(queueModelingPre, mqMessage);
 
                 long incr = redisUtil.incr("modeling-count:" + scenePlus.getNum(), 1);
@@ -97,6 +120,76 @@ public class CallBuildServiceImpl implements CallBuildService {
         log.info("推送计算定时任务结束");
     }
 
+    private void downloadOrigFile4Yzl(String batchId, String dataSource){
+        if(StrUtil.isEmpty(batchId)){
+            return;
+        }
+
+        OrigFileUpload sceneUpData = origFileUploadService.getByBatchIdAndFileName(batchId, "scene_up_data.txt");
+        String sceneUpDataPath = dataSource + "/" + "scene_up_data.txt";
+        HttpUtilExt.downloadFileAndCheck(sceneUpData.getFileUrl(), sceneUpDataPath, 60000);
+        String sceneUpDataStr = FileUtil.readUtf8String(sceneUpDataPath);
+        JSONArray fileJsonArray = JSON.parseArray(sceneUpDataStr);
+        Map<String, String> fileMap = fileJsonArray.stream().collect(Collectors.toMap(v -> ((JSONObject) v).getString("fileName"), v -> ((JSONObject) v).getString("filePath")));
+
+        List<OrigFileUpload> fileList = origFileUploadService.getByBatchId(batchId);
+        if(CollUtil.isEmpty(fileList)){
+            return;
+        }
+        String homePath = "/oss/4dkankan/" + SceneUtil.getHomePath(dataSource);
+
+        fileList.parallelStream().forEach(origFileUpload->{
+            String relativeFilePath = fileMap.get(origFileUpload.getFileName());
+            if(StrUtil.isNotEmpty(relativeFilePath)){
+                int times = 0;
+                String filePath = homePath.concat(relativeFilePath);
+                log.info("开始下载原始文件, url:{}, filePath:{}", origFileUpload.getFileUrl(), filePath);
+                do{
+                    try {
+                        ++times;
+                        HttpUtil.downloadFile(origFileUpload.getFileUrl(), new File(filePath), 3 * 60 * 1000);
+                        if(FileUtil.exist(filePath)){
+                            break;
+                        }
+                    }catch (Exception e){
+                        log.info("原始文件第{}次下载失败，fileUrl:{}, filePath:{}", times, origFileUpload.getFileUrl(), filePath);
+                    }
+                }while (times < 4);
+                log.info("下载原始文件结束, url:{}, filePath:{}", origFileUpload.getFileUrl(), filePath);
+            }
+        });
+    }
+
+    private void downloadOrigFile(String batchId, String dataSource){
+        if(StrUtil.isEmpty(batchId)){
+            return;
+        }
+
+        List<OrigFileUpload> fileList = origFileUploadService.getByBatchId(batchId);
+        if(CollUtil.isEmpty(fileList)){
+            return;
+        }
+        String homePath = "/oss/4dkankan/" + SceneUtil.getHomePath(dataSource);
+
+        fileList.parallelStream().forEach(origFileUpload->{
+            int times = 0;
+            String filePath = homePath.concat(origFileUpload.getFileName());
+            log.info("开始下载原始文件, url:{}, filePath:{}", origFileUpload.getFileUrl(), filePath);
+            do{
+                try {
+                    ++times;
+                    HttpUtil.downloadFile(origFileUpload.getFileUrl(), new File(filePath), 3 * 60 * 1000);
+                    if(FileUtil.exist(filePath)){
+                        break;
+                    }
+                }catch (Exception e){
+                    log.info("原始文件第{}次下载失败，fileUrl:{}, filePath:{}", times, origFileUpload.getFileUrl(), filePath);
+                }
+            }while (times < 3);
+            log.info("下载原始文件结束, url:{}, filePath:{}", origFileUpload.getFileUrl(), filePath);
+        });
+    }
+
     public static void main(String[] args) {
         List<OrigFileUploadBatch> list = new ArrayList<>();
         OrigFileUploadBatch a = new OrigFileUploadBatch();
@@ -116,7 +209,5 @@ public class CallBuildServiceImpl implements CallBuildService {
                 }
             }
         }
-
-
     }
 }

src/main/java/com/fdkankan/contro/service/impl/JmgaServiceImpl.java

@@ -16,6 +16,7 @@ import com.fdkankan.common.constant.ErrorCode;
 import com.fdkankan.common.constant.SceneStatus;
 import com.fdkankan.common.exception.BusinessException;
 import com.fdkankan.common.util.CmdUtils;
+import com.fdkankan.common.util.DateExtUtil;
 import com.fdkankan.contro.bean.SendCallAlgorithmDetail;
 import com.fdkankan.contro.constant.ZipConstant;
 import com.fdkankan.contro.entity.*;
@@ -325,7 +326,11 @@ public class JmgaServiceImpl implements IJmgaService {
 
         //上传日志，删除本地压缩包
         ZipUtil.zip(logPath, zipPath, false);
-        fYunFileService.uploadFile(zipPath, "build_log/");
+        String logName = "build_log/" + num + "/" + num + "-log.zip";
+        if(fYunFileService.fileExist(logName)){
+            fYunFileService.copyFileInBucket(logName, logName + "-bak-" + DateExtUtil.format(Calendar.getInstance().getTime(), DateExtUtil.dateStyle11));
+        }
+        fYunFileService.uploadFile(zipPath, "build_log/" + num + "/" + num + "-log");
 
     }