package com.fdkankan.download.service.impl; import cn.hutool.core.collection.CollUtil; import cn.hutool.core.collection.ConcurrentHashSet; import cn.hutool.core.exceptions.ExceptionUtil; import cn.hutool.core.util.StrUtil; import cn.hutool.json.JSON; import cn.hutool.json.JSONObject; import cn.hutool.json.JSONUtil; import com.fdkankan.common.constant.DownloadStatus; import com.fdkankan.common.constant.SceneDownloadProgressStatus; import com.fdkankan.common.constant.ServerCode; import com.fdkankan.common.constant.UploadFilePath; import com.fdkankan.common.response.ResultData; import com.fdkankan.common.util.FileUtils; import com.fdkankan.download.bean.CurrentDownloadNumUtil; import com.fdkankan.fyun.constant.StorageType; import com.fdkankan.fyun.oss.UploadToOssUtil; import com.fdkankan.platform.api.feign.PlatformUserClient; import com.fdkankan.redis.constant.RedisKey; import com.fdkankan.redis.util.RedisUtil; import com.fdkankan.common.bean.DownLoadProgressBean; import com.fdkankan.common.bean.DownLoadTaskBean; import com.fdkankan.scene.api.dto.SceneInfoDTO; import com.fdkankan.scene.api.feign.SceneUserSceneClient; import com.fdkankan.download.bean.ImageType; import com.fdkankan.download.bean.ImageTypeDetail; import com.google.common.collect.Lists; import java.io.File; import java.io.FileInputStream; import java.math.BigDecimal; import java.net.URLEncoder; import java.util.ArrayList; import java.util.Calendar; import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Objects; import java.util.Set; import java.util.stream.Collectors; import lombok.extern.slf4j.Slf4j; import lombok.var; import org.apache.tools.zip.ZipOutputStream; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.cloud.context.config.annotation.RefreshScope; import org.springframework.scheduling.annotation.Async; import org.springframework.stereotype.Service; import org.springframework.web.client.RestTemplate; /** *

* TODO *

* * @author dengsixing * @since 2022/2/22 **/ @RefreshScope @Slf4j @Service public class SceneDownloadHandlerServiceImpl { // @Autowired // private PlatformUserClient platformUserClient; @Autowired private SceneUserSceneClient sceneUserSceneClient; @Value("${path.v3school}") private String v3localPath; @Value("${path.zip-local}") private String zipLocalFormat; @Value("${path.zip-oss}") private String zipOssFormat; @Value("${path.zip-root}") private String wwwroot; // private static final String[] prefixArr = new String[]{ // "data/data%s/", // "voice/voice%s/", // "video/video%s/", // "images/images%s/" // }; private static final String[] prefixArr = new String[]{ UploadFilePath.DATA_VIEW_PATH, UploadFilePath.VOICE_VIEW_PATH, UploadFilePath.VIDEOS_VIEW_PATH, UploadFilePath.IMG_VIEW_PATH }; private static final List imageTypes = Lists.newArrayList(); static{ imageTypes.add(ImageType.builder().name("2k_face").size("2048").ranges(new String[]{"0", "511", "1023", "1535"}).build()); imageTypes.add(ImageType.builder().name("1k_face").size("1024").ranges(new String[]{"0", "511"}).build()); imageTypes.add(ImageType.builder().name("512_face").size("512").ranges(new String[]{"0"}).build()); } @Value("${upload.type:oss}") private String uploadType; @Value("${download.config.server-url}") private String serverUrl; @Value("${download.config.resource-url}") private String resourceUrl; @Value("${download.config.public-url}") private String publicUrl; @Value("${download.config.exe-name}") private String exeName; @Value("${download.config.exe-content}") private String exeContent; @Autowired RestTemplate restTemplate; @Autowired RedisUtil redisUtil; @Autowired UploadToOssUtil uploadToOssUtil; @Async("sceneDownLoadExecutror") public void download(DownLoadTaskBean downLoadTaskBean){ //场景码 String num = null; try { num = downLoadTaskBean.getNum(); log.info("场景下载开始 - num[{}] - threadName[{}]", num, Thread.currentThread().getName()); long startTime = Calendar.getInstance().getTimeInMillis(); //执行场景下载逻辑 this.downloadHandler(downLoadTaskBean); //耗时 long consumeTime = Calendar.getInstance().getTimeInMillis() - startTime; log.info("场景下载结束 - num[{}] - threadName[{}] - consumeTime[{}]", num, Thread.currentThread().getName(), consumeTime); }catch (Exception e){ log.error(ExceptionUtil.stacktraceToString(e)); }finally { if(StrUtil.isNotEmpty(num)){ //本地正在下载任务出队 CurrentDownloadNumUtil.removeSceneNum(num); //删除正在下载任务 redisUtil.lRemove(RedisKey.SCENE_DOWNLOAD_ING, 1, num); } } } public void downloadHandler(DownLoadTaskBean downLoadTaskBean) throws Exception{ String num = downLoadTaskBean.getNum(); Long userId = downLoadTaskBean.getUserId(); //zip包路径 String zipPath = null; //代码文件路径 //String v3localPath = "/downloads/v3local/"; // String v3localPath = "F:\\downloads\\v3local\\"; try { Set cacheKeys = new HashSet<>(); Map> allFiles = this.getAllFiles(num, v3localPath); List ossFilePaths = allFiles.get("ossFilePaths"); List v3localFilePaths = allFiles.get("v3localFilePaths"); //key总个数 int total = ossFilePaths.size() + v3localFilePaths.size(); int count = 0; //定义压缩包 // zipPath = "/downloads/scenes/" + num + ".zip"; // zipPath = "F:\\downloads\\scenes\\" + num + ".zip"; zipPath = String.format(this.zipLocalFormat, num); File zipFile = new File(zipPath); ZipOutputStream out = new ZipOutputStream(zipFile); JSONObject getInfoJson = this.zipGetInfoJson(out, this.wwwroot, num); String resolution = "2k"; if(getInfoJson.getInt("sceneSource") != null && (getInfoJson.getInt("sceneSource") == 3 || getInfoJson.getInt("sceneSource") == 4)){ resolution = "4k"; } int imagesVersion = -1; // TODO: 2022/3/29 V4版本目前没有imagesVersion字段,暂时用version字段替代 // if(getInfoJson.getInt("imagesVersion") != null){ // imagesVersion = getInfoJson.getInt("imagesVersion"); // } if(getInfoJson.getInt("version") != null){ imagesVersion = getInfoJson.getInt("version"); } //固定文件写入 count = this.zipLocalFiles(out, v3localFilePaths, v3localPath, num, count, total); //oss文件写入 count = this.zipOssFiles(out, ossFilePaths, num, count, total, resolution, imagesVersion, cacheKeys); //写入启动命令 this.zipBat(out, num); out.close(); //上传压缩包 String uploadPath = String.format(this.zipOssFormat, num); uploadToOssUtil.upload(zipPath, uploadPath); //更新进度100 String url = this.publicUrl + uploadPath; this.updateProgress(null, num, SceneDownloadProgressStatus.DOWNLOAD_SUCCESS.code(), url); // TODO: 2022/5/24 v3 停止后要开启-----------------------start //更新用户场景已下载次数 // platformUserClient.updateDownloadNum(userId, 1); // // //更新下载log状态为成功 // sceneUserSceneClient.updateSceneDownloadLog(num, DownloadStatus.SUCCESS.code(), url, null); // TODO: 2022/5/24 v3 停止后要开启-----------------------end }catch (Exception e){ //更新进度为下载失败 this.updateProgress( null, num, SceneDownloadProgressStatus.DOWNLOAD_FAILED.code(), null); //更新下载log状态为成功 // TODO: 2022/5/24 v3 停止后要开启-----------------------start // sceneUserSceneClient.updateSceneDownloadLog(num, DownloadStatus.FAILD.code(), null, ExceptionUtil.stacktraceToString(e)); // TODO: 2022/5/24 v3 停止后要开启-----------------------send throw e; }finally { if(StrUtil.isNotBlank(zipPath)){ //删除本地zip包 FileUtils.deleteFile(zipPath); } } } private int zipOssFiles(ZipOutputStream out, List ossFilePaths, String num, int count, int total, String resolution, int imagesVersion, Set cacheKeys) throws Exception{ String imageNumPath = "images" + num; for (String filePath : ossFilePaths) { if(filePath.contains(imageNumPath + "/panorama/panorama_edit/")){ //如果是编辑目录,只需要更新进度,不需要放进压缩包 this.updateProgress(new BigDecimal(++count).divide(new BigDecimal(total), 6, BigDecimal.ROUND_HALF_UP), num, SceneDownloadProgressStatus.DOWNLOADING.code(), null); continue; }else if((filePath.contains(imageNumPath + "/panorama/") && filePath.contains("tiles/" + resolution)) || filePath.contains(imageNumPath + "/tiles/" + resolution + "/")) { this.processImage(filePath, out, resolution, imagesVersion, cacheKeys); }else{ this.ProcessFiles(filePath, out, this.wwwroot, cacheKeys); } //更新进度 this.updateProgress(new BigDecimal(++count).divide(new BigDecimal(total), 6, BigDecimal.ROUND_HALF_UP), num, SceneDownloadProgressStatus.DOWNLOAD_COMPRESSING.code(), null); } return count; } private int zipLocalFiles(ZipOutputStream out, List v3localFilePaths, String v3localPath, String num, int count, int total) throws Exception{ for (String v3localFilePath : v3localFilePaths) { try (FileInputStream in = new FileInputStream(new File(v3localFilePath));){ this.zipInputStream(out, v3localFilePath.replace(v3localPath, ""), in); }catch (Exception e){ throw e; } //更新进度 this.updateProgress( new BigDecimal(++count).divide(new BigDecimal(total), 6, BigDecimal.ROUND_HALF_UP), num, SceneDownloadProgressStatus.DOWNLOAD_COMPRESSING.code(), null); } //写入code.txt this.zipBytes(out, "code.txt", num.getBytes()); return count; } private void zipBat(ZipOutputStream out, String num) throws Exception{ String batContent = String.format(this.exeContent, num); this.zipBytes(out, exeName, batContent.getBytes()); //更新进度为90% this.updateProgress(new BigDecimal("0.9").divide(new BigDecimal("0.8"), 6, BigDecimal.ROUND_HALF_UP), num, SceneDownloadProgressStatus.DOWNLOAD_COMPRESSING.code(), null); } private Map> getAllFiles(String num, String v3localPath) throws Exception{ //列出oss所有文件路径 List ossFilePaths = new ArrayList<>(); for (String prefix : prefixArr) { prefix = String.format(prefix, num); List keys = uploadToOssUtil.listKeys(prefix); if(CollUtil.isEmpty(keys)){ continue; } if(StorageType.AWS.code().equals(this.uploadType)){ keys = keys.stream().filter(key->{ if(key.contains("x-oss-process")){ return false; } return true; }).collect(Collectors.toList()); } ossFilePaths.addAll(keys); } //列出v3local所有文件路径 File file = new File(v3localPath); List v3localFilePaths = FileUtils.list(file); HashMap> map = new HashMap<>(); map.put("ossFilePaths", ossFilePaths); map.put("v3localFilePaths", v3localFilePaths); return map; } private JSONObject zipGetInfoJson(ZipOutputStream out, String root, String num) throws Exception{ ResultData sceneViewInfo = sceneUserSceneClient.getSceneViewInfo(num); if(!sceneViewInfo.getSuccess()){ throw new Exception(ServerCode.FEIGN_REQUEST_FAILD.message()); } SceneInfoDTO data = sceneViewInfo.getData(); JSONObject getInfoJson = null; if(Objects.isNull(data)){ getInfoJson = new JSONObject(); }else { getInfoJson = JSONUtil.parseObj(data); } getInfoJson.set("sceneScheme", 3); getInfoJson.set("needKey", 0); getInfoJson.set("sceneKey",""); //写入getInfo.json String getInfoJsonPath = root + "data/data"+ num + "/getInfo.json"; this.zipBytes(out, getInfoJsonPath, getInfoJson.toString().getBytes()); return getInfoJson; } private void processImage(String key, ZipOutputStream out, String resolution, int imagesVersion, Set imgKeys) throws Exception{ String fileName = key.substring(key.lastIndexOf("/")+1, key.indexOf(".")); String ext = key.substring(key.lastIndexOf(".")); String[] arr = fileName.split("_skybox"); String dir = arr[0]; String num = arr[1]; if(StrUtil.isEmpty(fileName) || StrUtil.isEmpty(ext) || (".jpg".equals(ext) && ".png".equals(ext)) || StrUtil.isEmpty(dir) || StrUtil.isEmpty(num)){ throw new Exception("本地下载图片资源不符合规则,key:" + key); } for (ImageType imageType : imageTypes) { List items = Lists.newArrayList(); String[] ranges = imageType.getRanges(); for(int i = 0; i < ranges.length; i++){ String x = ranges[i]; for(int j = 0; j < ranges.length; j++){ String y = ranges[j]; items.add( ImageTypeDetail.builder() .i(String.valueOf(i)) .j(String.valueOf(j)) .x(x) .y(y) .build() ); } } for (ImageTypeDetail item : items) { String par = "?x-oss-process=image/resize,m_lfit,w_" + imageType.getSize() + "/crop,w_512,h_512,x_" + item.getX() + ",y_" + item.getY(); if(StorageType.AWS.code().equals(uploadType)){ par += "&imagesVersion="+ imagesVersion; } var url = this. resourceUrl + key; StorageType storageType = StorageType.get(uploadType); switch (storageType){ case OSS: url += par; break; case AWS: url += URLEncoder.encode(par.replace("/", "@"), "UTF-8"); break; } var fky = key.split("/" + resolution + "/")[0] + "/" + dir + "/" + imageType.getName() + num + "_" + item.getI() + "_" + item.getJ() + ext; if(imgKeys.contains(fky)){ continue; } imgKeys.add(fky); this.zipBytes(out, wwwroot + fky, FileUtils.getBytesFromUrl(url)); } } } public void ProcessFiles(String key, ZipOutputStream out, String prefix, Set cacheKeys) throws Exception{ if(cacheKeys.contains(key)){ return; } cacheKeys.add(key); String url = this.resourceUrl + key + "?t=" + Calendar.getInstance().getTimeInMillis(); if(key.contains("hot.json") || key.contains("link-scene.json")){ String content = FileUtils.getStringFromUrl(url); content.replace(publicUrl, "") // .replace(publicUrl+"v3/", "") .replace("https://spc.html","spc.html") .replace("https://smobile.html", "smobile.html"); zipBytes(out, prefix + key, content.getBytes()); }else{ zipBytes(out, prefix + key, FileUtils.getBytesFromUrl(url)); } } public void updateProgress(BigDecimal precent, String num, Integer status, String url){ SceneDownloadProgressStatus progressStatus = SceneDownloadProgressStatus.get(status); switch (progressStatus){ case DOWNLOAD_SUCCESS: precent = new BigDecimal("100"); break; case DOWNLOAD_FAILED: precent = new BigDecimal("0"); break; default: precent = precent.multiply(new BigDecimal("0.8")).multiply(new BigDecimal("100")); } DownLoadProgressBean progress = null; String key = String.format(RedisKey.PREFIX_DOWNLOAD_PROGRESS_V4, num); String progressStr = redisUtil.get(key); if(StrUtil.isEmpty(progressStr)){ progress = DownLoadProgressBean.builder().percent(precent.intValue()).status(status).url(url).build(); }else{ progress = JSONUtil.toBean(progressStr, DownLoadProgressBean.class); //如果下载失败,进度不变 if(status == SceneDownloadProgressStatus.DOWNLOAD_FAILED.code() && progress.getPercent() != null){ precent = new BigDecimal(progress.getPercent()); } progress.setPercent(precent.intValue()); progress.setStatus(status); progress.setUrl(url); } redisUtil.set(key, JSONUtil.toJsonStr(progress)); } public void zipInputStream(ZipOutputStream out, String key, FileInputStream in) throws Exception { out.putNextEntry(new org.apache.tools.zip.ZipEntry(key)); byte[] bytes = new byte[1024]; int b = 0; while ((b = in.read(bytes)) != -1) { out.write(bytes, 0, b); } } public void zipBytes(ZipOutputStream out, String key, byte[] bytes) throws Exception { out.putNextEntry(new org.apache.tools.zip.ZipEntry(key)); out.write(bytes); } }