|
@@ -4,27 +4,24 @@ import cn.hutool.core.collection.CollUtil;
|
|
|
import cn.hutool.core.collection.ConcurrentHashSet;
|
|
|
import cn.hutool.core.exceptions.ExceptionUtil;
|
|
|
import cn.hutool.core.util.StrUtil;
|
|
|
-import cn.hutool.json.JSON;
|
|
|
import cn.hutool.json.JSONObject;
|
|
|
import cn.hutool.json.JSONUtil;
|
|
|
-import com.fdkankan.common.constant.DownloadStatus;
|
|
|
+import com.fdkankan.common.bean.DownLoadProgressBean;
|
|
|
+import com.fdkankan.common.bean.DownLoadTaskBean;
|
|
|
import com.fdkankan.common.constant.SceneDownloadProgressStatus;
|
|
|
import com.fdkankan.common.constant.ServerCode;
|
|
|
import com.fdkankan.common.constant.UploadFilePath;
|
|
|
import com.fdkankan.common.response.ResultData;
|
|
|
import com.fdkankan.common.util.FileUtils;
|
|
|
import com.fdkankan.download.bean.CurrentDownloadNumUtil;
|
|
|
+import com.fdkankan.download.bean.ImageType;
|
|
|
+import com.fdkankan.download.bean.ImageTypeDetail;
|
|
|
import com.fdkankan.fyun.constant.StorageType;
|
|
|
import com.fdkankan.fyun.oss.UploadToOssUtil;
|
|
|
-import com.fdkankan.platform.api.feign.PlatformUserClient;
|
|
|
import com.fdkankan.redis.constant.RedisKey;
|
|
|
import com.fdkankan.redis.util.RedisUtil;
|
|
|
-import com.fdkankan.common.bean.DownLoadProgressBean;
|
|
|
-import com.fdkankan.common.bean.DownLoadTaskBean;
|
|
|
import com.fdkankan.scene.api.dto.SceneInfoDTO;
|
|
|
import com.fdkankan.scene.api.feign.SceneUserSceneClient;
|
|
|
-import com.fdkankan.download.bean.ImageType;
|
|
|
-import com.fdkankan.download.bean.ImageTypeDetail;
|
|
|
import com.google.common.collect.Lists;
|
|
|
import java.io.File;
|
|
|
import java.io.FileInputStream;
|
|
@@ -33,17 +30,22 @@ import java.net.URLEncoder;
|
|
|
import java.util.ArrayList;
|
|
|
import java.util.Calendar;
|
|
|
import java.util.HashMap;
|
|
|
-import java.util.HashSet;
|
|
|
import java.util.List;
|
|
|
import java.util.Map;
|
|
|
import java.util.Objects;
|
|
|
import java.util.Set;
|
|
|
+import java.util.concurrent.Callable;
|
|
|
+import java.util.concurrent.ExecutorService;
|
|
|
+import java.util.concurrent.Executors;
|
|
|
+import java.util.concurrent.Future;
|
|
|
+import java.util.concurrent.atomic.AtomicInteger;
|
|
|
import java.util.stream.Collectors;
|
|
|
import lombok.extern.slf4j.Slf4j;
|
|
|
import lombok.var;
|
|
|
import org.apache.tools.zip.ZipOutputStream;
|
|
|
import org.springframework.beans.factory.annotation.Autowired;
|
|
|
import org.springframework.beans.factory.annotation.Value;
|
|
|
+import org.springframework.cloud.context.config.annotation.RefreshScope;
|
|
|
import org.springframework.scheduling.annotation.Async;
|
|
|
import org.springframework.stereotype.Service;
|
|
|
import org.springframework.web.client.RestTemplate;
|
|
@@ -56,12 +58,13 @@ import org.springframework.web.client.RestTemplate;
|
|
|
* @author dengsixing
|
|
|
* @since 2022/2/22
|
|
|
**/
|
|
|
+@RefreshScope
|
|
|
@Slf4j
|
|
|
@Service
|
|
|
public class SceneDownloadHandlerServiceImpl {
|
|
|
|
|
|
- @Autowired
|
|
|
- private PlatformUserClient platformUserClient;
|
|
|
+ // @Autowired
|
|
|
+// private PlatformUserClient platformUserClient;
|
|
|
@Autowired
|
|
|
private SceneUserSceneClient sceneUserSceneClient;
|
|
|
|
|
@@ -76,18 +79,16 @@ public class SceneDownloadHandlerServiceImpl {
|
|
|
|
|
|
@Value("${path.zip-root}")
|
|
|
private String wwwroot;
|
|
|
-// private static final String[] prefixArr = new String[]{
|
|
|
-// "data/data%s/",
|
|
|
-// "voice/voice%s/",
|
|
|
-// "video/video%s/",
|
|
|
-// "images/images%s/"
|
|
|
-// };
|
|
|
+
|
|
|
+ @Value("${zip.nThreads}")
|
|
|
+ private int zipNthreads;
|
|
|
|
|
|
private static final String[] prefixArr = new String[]{
|
|
|
UploadFilePath.DATA_VIEW_PATH,
|
|
|
UploadFilePath.VOICE_VIEW_PATH,
|
|
|
UploadFilePath.VIDEOS_VIEW_PATH,
|
|
|
- UploadFilePath.IMG_VIEW_PATH
|
|
|
+ UploadFilePath.IMG_VIEW_PATH,
|
|
|
+ UploadFilePath.USER_VIEW_PATH,
|
|
|
};
|
|
|
|
|
|
private static final List<ImageType> imageTypes = Lists.newArrayList();
|
|
@@ -152,17 +153,11 @@ public class SceneDownloadHandlerServiceImpl {
|
|
|
public void downloadHandler(DownLoadTaskBean downLoadTaskBean) throws Exception{
|
|
|
|
|
|
String num = downLoadTaskBean.getNum();
|
|
|
- Long userId = downLoadTaskBean.getUserId();
|
|
|
-
|
|
|
//zip包路径
|
|
|
String zipPath = null;
|
|
|
|
|
|
- //代码文件路径
|
|
|
- //String v3localPath = "/downloads/v3local/";
|
|
|
-// String v3localPath = "F:\\downloads\\v3local\\";
|
|
|
-
|
|
|
try {
|
|
|
- Set<String> cacheKeys = new HashSet<>();
|
|
|
+ Set<String> cacheKeys = new ConcurrentHashSet<>();
|
|
|
|
|
|
Map<String, List<String>> allFiles = this.getAllFiles(num, v3localPath);
|
|
|
List<String> ossFilePaths = allFiles.get("ossFilePaths");
|
|
@@ -170,12 +165,13 @@ public class SceneDownloadHandlerServiceImpl {
|
|
|
|
|
|
//key总个数
|
|
|
int total = ossFilePaths.size() + v3localFilePaths.size();
|
|
|
- int count = 0;
|
|
|
+ AtomicInteger count = new AtomicInteger(0);
|
|
|
//定义压缩包
|
|
|
- // zipPath = "/downloads/scenes/" + num + ".zip";
|
|
|
-// zipPath = "F:\\downloads\\scenes\\" + num + ".zip";
|
|
|
zipPath = String.format(this.zipLocalFormat, num);
|
|
|
File zipFile = new File(zipPath);
|
|
|
+ if(!zipFile.getParentFile().exists()){
|
|
|
+ zipFile.getParentFile().mkdirs();
|
|
|
+ }
|
|
|
ZipOutputStream out = new ZipOutputStream(zipFile);
|
|
|
|
|
|
JSONObject getInfoJson = this.zipGetInfoJson(out, this.wwwroot, num);
|
|
@@ -194,11 +190,17 @@ public class SceneDownloadHandlerServiceImpl {
|
|
|
}
|
|
|
|
|
|
|
|
|
+ long start = Calendar.getInstance().getTimeInMillis();
|
|
|
//固定文件写入
|
|
|
- count = this.zipLocalFiles(out, v3localFilePaths, v3localPath, num, count, total);
|
|
|
+ this.zipLocalFiles(out, v3localFilePaths, v3localPath, num, count, total);
|
|
|
+ long end1 = Calendar.getInstance().getTimeInMillis();
|
|
|
+ log.info("打包固定文件耗时, num:{}, time:{}", num, end1 - start);
|
|
|
|
|
|
//oss文件写入
|
|
|
- count = this.zipOssFiles(out, ossFilePaths, num, count, total, resolution, imagesVersion, cacheKeys);
|
|
|
+ this.zipOssFiles(out, ossFilePaths, num, count, total, resolution, imagesVersion, cacheKeys);
|
|
|
+ long end2 = Calendar.getInstance().getTimeInMillis();
|
|
|
+ log.info("打包oss文件耗时, num:{}, time:{}", num, end2 - end1);
|
|
|
+
|
|
|
|
|
|
//写入启动命令
|
|
|
this.zipBat(out, num);
|
|
@@ -207,23 +209,27 @@ public class SceneDownloadHandlerServiceImpl {
|
|
|
|
|
|
//上传压缩包
|
|
|
String uploadPath = String.format(this.zipOssFormat, num);
|
|
|
- uploadToOssUtil.upload(zipPath, uploadPath);
|
|
|
+ uploadToOssUtil.uploadBySh(zipPath, uploadPath);
|
|
|
|
|
|
//更新进度100
|
|
|
String url = this.publicUrl + uploadPath;
|
|
|
this.updateProgress(null, num, SceneDownloadProgressStatus.DOWNLOAD_SUCCESS.code(), url);
|
|
|
|
|
|
+ // TODO: 2022/5/24 v3 停止后要开启-----------------------start
|
|
|
//更新用户场景已下载次数
|
|
|
- platformUserClient.updateDownloadNum(userId, 1);
|
|
|
-
|
|
|
- //更新下载log状态为成功
|
|
|
- sceneUserSceneClient.updateSceneDownloadLog(num, DownloadStatus.SUCCESS.code(), url, null);
|
|
|
+// platformUserClient.updateDownloadNum(userId, 1);
|
|
|
+//
|
|
|
+// //更新下载log状态为成功
|
|
|
+// sceneUserSceneClient.updateSceneDownloadLog(num, DownloadStatus.SUCCESS.code(), url, null);
|
|
|
+ // TODO: 2022/5/24 v3 停止后要开启-----------------------end
|
|
|
|
|
|
}catch (Exception e){
|
|
|
//更新进度为下载失败
|
|
|
this.updateProgress( null, num, SceneDownloadProgressStatus.DOWNLOAD_FAILED.code(), null);
|
|
|
//更新下载log状态为成功
|
|
|
- sceneUserSceneClient.updateSceneDownloadLog(num, DownloadStatus.FAILD.code(), null, ExceptionUtil.stacktraceToString(e));
|
|
|
+ // TODO: 2022/5/24 v3 停止后要开启-----------------------start
|
|
|
+// sceneUserSceneClient.updateSceneDownloadLog(num, DownloadStatus.FAILD.code(), null, ExceptionUtil.stacktraceToString(e));
|
|
|
+ // TODO: 2022/5/24 v3 停止后要开启-----------------------send
|
|
|
throw e;
|
|
|
}finally {
|
|
|
if(StrUtil.isNotBlank(zipPath)){
|
|
@@ -233,28 +239,53 @@ public class SceneDownloadHandlerServiceImpl {
|
|
|
}
|
|
|
}
|
|
|
|
|
|
- private int zipOssFiles(ZipOutputStream out, List<String> ossFilePaths, String num, int count, int total, String resolution, int imagesVersion, Set<String> cacheKeys) throws Exception{
|
|
|
- String imageNumPath = "images" + num;
|
|
|
+ private void zipOssFiles(ZipOutputStream out, List<String> ossFilePaths, String num, AtomicInteger count,
|
|
|
+ int total, String resolution, int imagesVersion, Set<String> cacheKeys) throws Exception{
|
|
|
+ String imageNumPath = String.format(UploadFilePath.IMG_VIEW_PATH, num);
|
|
|
+ ExecutorService executorService = Executors.newFixedThreadPool(this.zipNthreads);
|
|
|
+ List<Future> futureList = new ArrayList<>();
|
|
|
for (String filePath : ossFilePaths) {
|
|
|
- if(filePath.contains(imageNumPath + "/panorama/panorama_edit/")){
|
|
|
- //如果是编辑目录,只需要更新进度,不需要放进压缩包
|
|
|
- this.updateProgress(new BigDecimal(++count).divide(new BigDecimal(total), 6, BigDecimal.ROUND_HALF_UP),
|
|
|
- num, SceneDownloadProgressStatus.DOWNLOADING.code(), null);
|
|
|
- continue;
|
|
|
- }else if((filePath.contains(imageNumPath + "/panorama/") && filePath.contains("tiles/" + resolution)) || filePath.contains(imageNumPath + "/tiles/" + resolution + "/")) {
|
|
|
- this.processImage(filePath, out, resolution, imagesVersion, cacheKeys);
|
|
|
- }else{
|
|
|
- this.ProcessFiles(filePath, out, this.wwwroot, cacheKeys);
|
|
|
- }
|
|
|
+ Callable<Boolean> call = new Callable() {
|
|
|
+ @Override
|
|
|
+ public Boolean call() throws Exception {
|
|
|
+ zipOssFilesHandler(out, num, count, total, resolution,
|
|
|
+ imagesVersion, cacheKeys,filePath, imageNumPath);
|
|
|
+ return true;
|
|
|
+ }
|
|
|
+ };
|
|
|
+ futureList.add(executorService.submit(call));
|
|
|
+ }
|
|
|
+ //这里一定要加阻塞,不然会导致oss文件还没打包好,主程序已经结束返回了
|
|
|
+ for (Future future : futureList) {
|
|
|
+ future.get();
|
|
|
+ }
|
|
|
+ }
|
|
|
|
|
|
- //更新进度
|
|
|
- this.updateProgress(new BigDecimal(++count).divide(new BigDecimal(total), 6, BigDecimal.ROUND_HALF_UP),
|
|
|
- num, SceneDownloadProgressStatus.DOWNLOAD_COMPRESSING.code(), null);
|
|
|
+ private void zipOssFilesHandler(ZipOutputStream out, String num,
|
|
|
+ AtomicInteger count, int total, String resolution,
|
|
|
+ int imagesVersion, Set<String> cacheKeys,
|
|
|
+ String filePath, String imageNumPath) throws Exception{
|
|
|
+
|
|
|
+ //更新进度
|
|
|
+ this.updateProgress(new BigDecimal(count.incrementAndGet()).divide(new BigDecimal(total), 6, BigDecimal.ROUND_HALF_UP),
|
|
|
+ num, SceneDownloadProgressStatus.DOWNLOADING.code(), null);
|
|
|
+
|
|
|
+ //某个目录不需要打包
|
|
|
+ if(filePath.contains(imageNumPath + "panorama/panorama_edit/"))
|
|
|
+ return;
|
|
|
+
|
|
|
+ //切图
|
|
|
+ if((filePath.contains(imageNumPath + "panorama/") && filePath.contains("tiles/" + resolution))
|
|
|
+ || filePath.contains(imageNumPath + "tiles/" + resolution + "/")) {
|
|
|
+ this.processImage(filePath, out, resolution, imagesVersion, cacheKeys);
|
|
|
+ return;
|
|
|
}
|
|
|
- return count;
|
|
|
+
|
|
|
+ //其他文件打包
|
|
|
+ this.ProcessFiles(filePath, out, this.wwwroot, cacheKeys);
|
|
|
}
|
|
|
|
|
|
- private int zipLocalFiles(ZipOutputStream out, List<String> v3localFilePaths, String v3localPath, String num, int count, int total) throws Exception{
|
|
|
+ private void zipLocalFiles(ZipOutputStream out, List<String> v3localFilePaths, String v3localPath, String num, AtomicInteger count, int total) throws Exception{
|
|
|
for (String v3localFilePath : v3localFilePaths) {
|
|
|
try (FileInputStream in = new FileInputStream(new File(v3localFilePath));){
|
|
|
this.zipInputStream(out, v3localFilePath.replace(v3localPath, ""), in);
|
|
@@ -263,12 +294,11 @@ public class SceneDownloadHandlerServiceImpl {
|
|
|
}
|
|
|
//更新进度
|
|
|
this.updateProgress(
|
|
|
- new BigDecimal(++count).divide(new BigDecimal(total), 6, BigDecimal.ROUND_HALF_UP),
|
|
|
+ new BigDecimal(count.incrementAndGet()).divide(new BigDecimal(total), 6, BigDecimal.ROUND_HALF_UP),
|
|
|
num, SceneDownloadProgressStatus.DOWNLOAD_COMPRESSING.code(), null);
|
|
|
}
|
|
|
//写入code.txt
|
|
|
this.zipBytes(out, "code.txt", num.getBytes());
|
|
|
- return count;
|
|
|
}
|
|
|
|
|
|
private void zipBat(ZipOutputStream out, String num) throws Exception{
|
|
@@ -329,7 +359,7 @@ public class SceneDownloadHandlerServiceImpl {
|
|
|
getInfoJson.set("needKey", 0);
|
|
|
getInfoJson.set("sceneKey","");
|
|
|
//写入getInfo.json
|
|
|
- String getInfoJsonPath = root + "data/data"+ num + "/getInfo.json";
|
|
|
+ String getInfoJsonPath = root + String.format(UploadFilePath.DATA_VIEW_PATH, num) + "getInfo.json";
|
|
|
this.zipBytes(out, getInfoJsonPath, getInfoJson.toString().getBytes());
|
|
|
return getInfoJson;
|
|
|
}
|
|
@@ -368,7 +398,6 @@ public class SceneDownloadHandlerServiceImpl {
|
|
|
);
|
|
|
}
|
|
|
}
|
|
|
-
|
|
|
for (ImageTypeDetail item : items) {
|
|
|
String par = "?x-oss-process=image/resize,m_lfit,w_" + imageType.getSize() + "/crop,w_512,h_512,x_" + item.getX() + ",y_" + item.getY();
|
|
|
if(StorageType.AWS.code().equals(uploadType)){
|
|
@@ -391,8 +420,7 @@ public class SceneDownloadHandlerServiceImpl {
|
|
|
continue;
|
|
|
}
|
|
|
imgKeys.add(fky);
|
|
|
- byte[] bytes = FileUtils.getBytesFromUrl(url);
|
|
|
- this.zipBytes(out, wwwroot + fky, bytes);
|
|
|
+ this.zipBytes(out, wwwroot + fky, FileUtils.getBytesFromUrl(url));
|
|
|
}
|
|
|
|
|
|
}
|
|
@@ -405,7 +433,6 @@ public class SceneDownloadHandlerServiceImpl {
|
|
|
}
|
|
|
cacheKeys.add(key);
|
|
|
String url = this.resourceUrl + key + "?t=" + Calendar.getInstance().getTimeInMillis();
|
|
|
- byte[] data = null;
|
|
|
if(key.contains("hot.json") || key.contains("link-scene.json")){
|
|
|
String content = FileUtils.getStringFromUrl(url);
|
|
|
|
|
@@ -414,12 +441,10 @@ public class SceneDownloadHandlerServiceImpl {
|
|
|
.replace("https://spc.html","spc.html")
|
|
|
.replace("https://smobile.html", "smobile.html");
|
|
|
|
|
|
- data = content.getBytes();
|
|
|
+ zipBytes(out, prefix + key, content.getBytes());
|
|
|
}else{
|
|
|
- data = FileUtils.getBytesFromUrl(url);
|
|
|
+ zipBytes(out, prefix + key, FileUtils.getBytesFromUrl(url));
|
|
|
}
|
|
|
-
|
|
|
- zipBytes(out, prefix + key, data);
|
|
|
}
|
|
|
|
|
|
|
|
@@ -465,7 +490,7 @@ public class SceneDownloadHandlerServiceImpl {
|
|
|
}
|
|
|
}
|
|
|
|
|
|
- public void zipBytes(ZipOutputStream out, String key, byte[] bytes) throws Exception {
|
|
|
+ public synchronized void zipBytes(ZipOutputStream out, String key, byte[] bytes) throws Exception {
|
|
|
out.putNextEntry(new org.apache.tools.zip.ZipEntry(key));
|
|
|
out.write(bytes);
|
|
|
}
|