4dkankan-parent/4dkankan-center-scene-download/src/main/java/com/fdkankan/download/service/impl/SceneDownloadHandlerServiceImpl.java

package com.fdkankan.download.service.impl;

import cn.hutool.core.collection.CollUtil;
import cn.hutool.core.collection.ConcurrentHashSet;
import cn.hutool.core.exceptions.ExceptionUtil;
import cn.hutool.core.util.StrUtil;
import cn.hutool.json.JSON;
import cn.hutool.json.JSONObject;
import cn.hutool.json.JSONUtil;
import com.fdkankan.common.constant.DownloadStatus;
import com.fdkankan.common.constant.SceneDownloadProgressStatus;
import com.fdkankan.common.constant.ServerCode;
import com.fdkankan.common.constant.UploadFilePath;
import com.fdkankan.common.response.ResultData;
import com.fdkankan.common.util.FileUtils;
import com.fdkankan.download.bean.CurrentDownloadNumUtil;
import com.fdkankan.fyun.constant.StorageType;
import com.fdkankan.fyun.oss.UploadToOssUtil;
import com.fdkankan.platform.api.feign.PlatformUserClient;
import com.fdkankan.redis.constant.RedisKey;
import com.fdkankan.redis.util.RedisUtil;
import com.fdkankan.common.bean.DownLoadProgressBean;
import com.fdkankan.common.bean.DownLoadTaskBean;
import com.fdkankan.scene.api.dto.SceneInfoDTO;
import com.fdkankan.scene.api.feign.SceneUserSceneClient;
import com.fdkankan.download.bean.ImageType;
import com.fdkankan.download.bean.ImageTypeDetail;
import com.google.common.collect.Lists;
import java.io.File;
import java.io.FileInputStream;
import java.io.UnsupportedEncodingException;
import java.math.BigDecimal;
import java.net.URLEncoder;
import java.util.ArrayList;
import java.util.Calendar;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.stream.Collectors;
import lombok.extern.slf4j.Slf4j;
import lombok.var;
import org.apache.tools.zip.ZipOutputStream;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.cloud.context.config.annotation.RefreshScope;
import org.springframework.scheduling.annotation.Async;
import org.springframework.stereotype.Service;
import org.springframework.web.client.RestTemplate;

/**
 * <p>
 * TODO
 * </p>
 *
 * @author dengsixing
 * @since 2022/2/22
 **/
@RefreshScope
@Slf4j
@Service
public class SceneDownloadHandlerServiceImpl {

    //    @Autowired
//    private PlatformUserClient platformUserClient;
    @Autowired
    private SceneUserSceneClient sceneUserSceneClient;

    @Value("${path.v3school}")
    private String v3localPath;

    @Value("${path.zip-local}")
    private String zipLocalFormat;

    @Value("${path.zip-oss}")
    private String zipOssFormat;

    @Value("${path.zip-root}")
    private String wwwroot;
//    private static final String[] prefixArr = new String[]{
//        "data/data%s/",
//        "voice/voice%s/",
//        "video/video%s/",
//        "images/images%s/"
//    };

    private static final String[] prefixArr = new String[]{
        UploadFilePath.DATA_VIEW_PATH,
        UploadFilePath.VOICE_VIEW_PATH,
        UploadFilePath.VIDEOS_VIEW_PATH,
        UploadFilePath.IMG_VIEW_PATH,
        UploadFilePath.USER_VIEW_PATH,
    };

    private static final List<ImageType> imageTypes = Lists.newArrayList();
    static{
        imageTypes.add(ImageType.builder().name("2k_face").size("2048").ranges(new String[]{"0", "511", "1023", "1535"}).build());
        imageTypes.add(ImageType.builder().name("1k_face").size("1024").ranges(new String[]{"0", "511"}).build());
        imageTypes.add(ImageType.builder().name("512_face").size("512").ranges(new String[]{"0"}).build());
    }

    @Value("${upload.type:oss}")
    private String uploadType;
    @Value("${download.config.server-url}")
    private String serverUrl;
    @Value("${download.config.resource-url}")
    private String resourceUrl;
    @Value("${download.config.public-url}")
    private String publicUrl;
    @Value("${download.config.exe-name}")
    private String exeName;
    @Value("${download.config.exe-content}")
    private String exeContent;
    @Autowired
    RestTemplate restTemplate;

    @Autowired
    RedisUtil redisUtil;
    @Autowired
    UploadToOssUtil uploadToOssUtil;

    @Async("sceneDownLoadExecutror")
    public void download(DownLoadTaskBean downLoadTaskBean){
        //场景码
        String num = null;

        try {
            num = downLoadTaskBean.getNum();

            log.info("场景下载开始 - num[{}] - threadName[{}]", num, Thread.currentThread().getName());

            long startTime = Calendar.getInstance().getTimeInMillis();

            //执行场景下载逻辑
            this.downloadHandler(downLoadTaskBean);

            //耗时
            long consumeTime = Calendar.getInstance().getTimeInMillis() - startTime;

            log.info("场景下载结束 - num[{}] - threadName[{}] - consumeTime[{}]", num, Thread.currentThread().getName(), consumeTime);

        }catch (Exception e){
            log.error(ExceptionUtil.stacktraceToString(e));
        }finally {
            if(StrUtil.isNotEmpty(num)){
                //本地正在下载任务出队
                CurrentDownloadNumUtil.removeSceneNum(num);
                //删除正在下载任务
                redisUtil.lRemove(RedisKey.SCENE_DOWNLOAD_ING, 1, num);
            }
        }
    }

    public void downloadHandler(DownLoadTaskBean downLoadTaskBean) throws Exception{

        String num = downLoadTaskBean.getNum();
        Long userId = downLoadTaskBean.getUserId();

        //zip包路径
        String zipPath = null;

        //代码文件路径
        //String v3localPath = "/downloads/v3local/";
//        String v3localPath = "F:\\downloads\\v3local\\";

        try {
            Set<String> cacheKeys = new HashSet<>();

            Map<String, List<String>> allFiles = this.getAllFiles(num, v3localPath);
            List<String> ossFilePaths = allFiles.get("ossFilePaths");
            List<String> v3localFilePaths = allFiles.get("v3localFilePaths");

            //key总个数
            int total = ossFilePaths.size() + v3localFilePaths.size();
            int count = 0;
            //定义压缩包
            //            zipPath = "/downloads/scenes/" + num + ".zip";
//            zipPath = "F:\\downloads\\scenes\\" + num + ".zip";
            zipPath = String.format(this.zipLocalFormat, num);
            File zipFile = new File(zipPath);
            if(!zipFile.getParentFile().exists()){
                zipFile.getParentFile().mkdirs();
            }
            ZipOutputStream out = new ZipOutputStream(zipFile);

            JSONObject getInfoJson = this.zipGetInfoJson(out, this.wwwroot, num);
            String resolution = "2k";
            if(getInfoJson.getInt("sceneSource") != null &&
                (getInfoJson.getInt("sceneSource") == 3 || getInfoJson.getInt("sceneSource") == 4)){
                resolution = "4k";
            }
            int imagesVersion = -1;
            // TODO: 2022/3/29  V4版本目前没有imagesVersion字段，暂时用version字段替代
//            if(getInfoJson.getInt("imagesVersion") != null){
//                imagesVersion = getInfoJson.getInt("imagesVersion");
//            }
            if(getInfoJson.getInt("version") != null){
                imagesVersion = getInfoJson.getInt("version");
            }


            long start = Calendar.getInstance().getTimeInMillis();
            //固定文件写入
            count = this.zipLocalFiles(out, v3localFilePaths, v3localPath, num, count, total);
            long end1 = Calendar.getInstance().getTimeInMillis();
            log.info("打包固定文件耗时:{}", end1 - start);

            //oss文件写入
            count = this.zipOssFiles(out, ossFilePaths, num, count, total, resolution, imagesVersion, cacheKeys);
            long end2 = Calendar.getInstance().getTimeInMillis();
            log.info("打包oss文件耗时:{}", end2 - end1);


            //写入启动命令
            this.zipBat(out, num);

            out.close();

            //上传压缩包
            String uploadPath = String.format(this.zipOssFormat, num);
            uploadToOssUtil.uploadBySh(zipPath, uploadPath);

            //更新进度100
            String url = this.publicUrl + uploadPath;
            this.updateProgress(null, num, SceneDownloadProgressStatus.DOWNLOAD_SUCCESS.code(), url);

            // TODO: 2022/5/24 v3 停止后要开启-----------------------start
            //更新用户场景已下载次数
//            platformUserClient.updateDownloadNum(userId, 1);
//
//            //更新下载log状态为成功
//            sceneUserSceneClient.updateSceneDownloadLog(num, DownloadStatus.SUCCESS.code(), url, null);
            // TODO: 2022/5/24 v3 停止后要开启-----------------------end

        }catch (Exception e){
            //更新进度为下载失败
            this.updateProgress( null, num, SceneDownloadProgressStatus.DOWNLOAD_FAILED.code(), null);
            //更新下载log状态为成功
            // TODO: 2022/5/24 v3 停止后要开启-----------------------start
//            sceneUserSceneClient.updateSceneDownloadLog(num, DownloadStatus.FAILD.code(), null, ExceptionUtil.stacktraceToString(e));
            // TODO: 2022/5/24 v3 停止后要开启-----------------------send
            throw e;
        }finally {
            if(StrUtil.isNotBlank(zipPath)){
                //删除本地zip包
                FileUtils.deleteFile(zipPath);
            }
        }
    }

    private int zipOssFiles(ZipOutputStream out, List<String> ossFilePaths, String num, int count, int total, String resolution, int imagesVersion, Set<String> cacheKeys) throws Exception{
        String imageNumPath = String.format(UploadFilePath.IMG_VIEW_PATH, num);
        for (String filePath : ossFilePaths) {
            if(filePath.contains(imageNumPath + "panorama/panorama_edit/")){
                //如果是编辑目录，只需要更新进度，不需要放进压缩包
                this.updateProgress(new BigDecimal(++count).divide(new BigDecimal(total), 6, BigDecimal.ROUND_HALF_UP),
                    num, SceneDownloadProgressStatus.DOWNLOADING.code(), null);
                continue;
            }else if((filePath.contains(imageNumPath + "panorama/") && filePath.contains("tiles/" + resolution)) || filePath.contains(imageNumPath + "tiles/" + resolution + "/")) {
                this.processImage(filePath, out, resolution, imagesVersion, cacheKeys);
            }else{
                long start = Calendar.getInstance().getTimeInMillis();
                this.ProcessFiles(filePath, out, this.wwwroot, cacheKeys);
                log.info("非切图文件耗时,key:{},time:{}", filePath, Calendar.getInstance().getTimeInMillis() - start);
            }

            //更新进度
            this.updateProgress(new BigDecimal(++count).divide(new BigDecimal(total), 6, BigDecimal.ROUND_HALF_UP),
                num, SceneDownloadProgressStatus.DOWNLOAD_COMPRESSING.code(), null);
        }
        return count;
    }

    private int zipLocalFiles(ZipOutputStream out, List<String> v3localFilePaths, String v3localPath, String num, int count, int total) throws Exception{
        for (String v3localFilePath : v3localFilePaths) {
            try (FileInputStream in = new FileInputStream(new File(v3localFilePath));){
                this.zipInputStream(out, v3localFilePath.replace(v3localPath, ""), in);
            }catch (Exception e){
                throw e;
            }
            //更新进度
            this.updateProgress(
                new BigDecimal(++count).divide(new BigDecimal(total), 6, BigDecimal.ROUND_HALF_UP),
                num, SceneDownloadProgressStatus.DOWNLOAD_COMPRESSING.code(), null);
        }
        //写入code.txt
        this.zipBytes(out, "code.txt", num.getBytes());
        return count;
    }

    private void zipBat(ZipOutputStream out, String num) throws Exception{
        String batContent = String.format(this.exeContent, num);
        this.zipBytes(out, exeName, batContent.getBytes());

        //更新进度为90%
        this.updateProgress(new BigDecimal("0.9").divide(new BigDecimal("0.8"), 6, BigDecimal.ROUND_HALF_UP), num,
            SceneDownloadProgressStatus.DOWNLOAD_COMPRESSING.code(), null);
    }

    private Map<String, List<String>> getAllFiles(String num, String v3localPath) throws Exception{
        //列出oss所有文件路径
        List<String> ossFilePaths = new ArrayList<>();
        for (String prefix : prefixArr) {
            prefix = String.format(prefix, num);
            List<String> keys = uploadToOssUtil.listKeys(prefix);
            if(CollUtil.isEmpty(keys)){
                continue;
            }
            if(StorageType.AWS.code().equals(this.uploadType)){
                keys = keys.stream().filter(key->{
                    if(key.contains("x-oss-process")){
                        return false;
                    }
                    return true;
                }).collect(Collectors.toList());
            }
            ossFilePaths.addAll(keys);
        }

        //列出v3local所有文件路径
        File file = new File(v3localPath);
        List<String> v3localFilePaths = FileUtils.list(file);

        HashMap<String, List<String>> map = new HashMap<>();
        map.put("ossFilePaths", ossFilePaths);
        map.put("v3localFilePaths", v3localFilePaths);

        return map;
    }

    private JSONObject zipGetInfoJson(ZipOutputStream out, String root, String num) throws Exception{

        ResultData<SceneInfoDTO> sceneViewInfo = sceneUserSceneClient.getSceneViewInfo(num);
        if(!sceneViewInfo.getSuccess()){
            throw new Exception(ServerCode.FEIGN_REQUEST_FAILD.message());
        }
        SceneInfoDTO data = sceneViewInfo.getData();
        JSONObject getInfoJson = null;
        if(Objects.isNull(data)){
            getInfoJson = new JSONObject();
        }else {
            getInfoJson = JSONUtil.parseObj(data);
        }

        getInfoJson.set("sceneScheme", 3);
        getInfoJson.set("needKey", 0);
        getInfoJson.set("sceneKey","");
        //写入getInfo.json
        String getInfoJsonPath = root + String.format(UploadFilePath.DATA_VIEW_PATH, num) + "getInfo.json";
        this.zipBytes(out, getInfoJsonPath, getInfoJson.toString().getBytes());
        return getInfoJson;
    }

    private void processImage(String key, ZipOutputStream out, String resolution, int imagesVersion, Set<String> imgKeys) throws Exception{

        String fileName = key.substring(key.lastIndexOf("/")+1, key.indexOf("."));
        String ext = key.substring(key.lastIndexOf("."));
        String[] arr = fileName.split("_skybox");
        String dir = arr[0];
        String num = arr[1];

        if(StrUtil.isEmpty(fileName)
            || StrUtil.isEmpty(ext)
            || (".jpg".equals(ext) && ".png".equals(ext))
            ||  StrUtil.isEmpty(dir)
            || StrUtil.isEmpty(num)){
            throw new Exception("本地下载图片资源不符合规则，key：" + key);
        }

        for (ImageType imageType : imageTypes) {

            List<ImageTypeDetail> items = Lists.newArrayList();
            String[] ranges = imageType.getRanges();
            for(int i = 0; i < ranges.length; i++){
                String x = ranges[i];
                for(int j = 0; j < ranges.length; j++){
                    String y = ranges[j];
                    items.add(
                        ImageTypeDetail.builder()
                            .i(String.valueOf(i))
                            .j(String.valueOf(j))
                            .x(x)
                            .y(y)
                            .build()
                    );
                }
            }

            items.parallelStream().forEach(item->{
                String par = "?x-oss-process=image/resize,m_lfit,w_" + imageType.getSize() + "/crop,w_512,h_512,x_" + item.getX() + ",y_" + item.getY();
                if(StorageType.AWS.code().equals(uploadType)){
                    par += "&imagesVersion="+ imagesVersion;
                }

                var url = this.
                    resourceUrl + key;
                StorageType storageType = StorageType.get(uploadType);
                switch (storageType){
                    case OSS:
                        url += par;
                        break;
                    case AWS:
                        try {
                            url += URLEncoder.encode(par.replace("/", "@"), "UTF-8");
                        } catch (UnsupportedEncodingException e) {
                            e.printStackTrace();
                        }
                        break;
                }
                var fky = key.split("/" + resolution + "/")[0] + "/" + dir + "/" + imageType.getName() +  num + "_" + item.getI()  + "_" + item.getJ() + ext;
                if(!imgKeys.contains(fky)){
                    imgKeys.add(fky);
                    long start = Calendar.getInstance().getTimeInMillis();
                    try {
                        this.zipBytes(out, wwwroot + fky, FileUtils.getBytesFromUrl(url));
                    } catch (Exception e) {
                        e.printStackTrace();
                    }
                    log.info("切图耗时，url:{},time:{}", url, Calendar.getInstance().getTimeInMillis() - start);
                }
            });
//            for (ImageTypeDetail item : items) {
//                String par = "?x-oss-process=image/resize,m_lfit,w_" + imageType.getSize() + "/crop,w_512,h_512,x_" + item.getX() + ",y_" + item.getY();
//                if(StorageType.AWS.code().equals(uploadType)){
//                    par += "&imagesVersion="+ imagesVersion;
//                }
//
//                var url = this.
//                    resourceUrl + key;
//                StorageType storageType = StorageType.get(uploadType);
//                switch (storageType){
//                    case OSS:
//                        url += par;
//                        break;
//                    case AWS:
//                        url += URLEncoder.encode(par.replace("/", "@"), "UTF-8");
//                        break;
//                }
//                var fky = key.split("/" + resolution + "/")[0] + "/" + dir + "/" + imageType.getName() +  num + "_" + item.getI()  + "_" + item.getJ() + ext;
//                if(imgKeys.contains(fky)){
//                    continue;
//                }
//                imgKeys.add(fky);
//                long start = Calendar.getInstance().getTimeInMillis();
//                this.zipBytes(out, wwwroot + fky, FileUtils.getBytesFromUrl(url));
//                log.info("切图耗时，url:{},time:{}", url, Calendar.getInstance().getTimeInMillis() - start);
//            }

        }

    }

    public void ProcessFiles(String key, ZipOutputStream out, String prefix, Set<String> cacheKeys) throws Exception{
        if(cacheKeys.contains(key)){
            return;
        }
        cacheKeys.add(key);
        String url = this.resourceUrl + key + "?t=" + Calendar.getInstance().getTimeInMillis();
        if(key.contains("hot.json") || key.contains("link-scene.json")){
            String content = FileUtils.getStringFromUrl(url);

            content.replace(publicUrl, "")
//                .replace(publicUrl+"v3/", "")
                .replace("https://spc.html","spc.html")
                .replace("https://smobile.html", "smobile.html");

            zipBytes(out, prefix + key, content.getBytes());
        }else{
            zipBytes(out, prefix + key, FileUtils.getBytesFromUrl(url));
        }
    }


    public void updateProgress(BigDecimal precent, String num, Integer status, String url){

        SceneDownloadProgressStatus progressStatus = SceneDownloadProgressStatus.get(status);
        switch (progressStatus){
            case DOWNLOAD_SUCCESS:
                precent = new BigDecimal("100");
                break;
            case DOWNLOAD_FAILED:
                precent = new BigDecimal("0");
                break;
            default:
                precent = precent.multiply(new BigDecimal("0.8")).multiply(new BigDecimal("100"));
        }

        DownLoadProgressBean progress = null;
        String key = String.format(RedisKey.PREFIX_DOWNLOAD_PROGRESS_V4, num);
        String progressStr = redisUtil.get(key);
        if(StrUtil.isEmpty(progressStr)){
            progress =  DownLoadProgressBean.builder().percent(precent.intValue()).status(status).url(url).build();
        }else{
            progress = JSONUtil.toBean(progressStr, DownLoadProgressBean.class);
            //如果下载失败，进度不变
            if(status == SceneDownloadProgressStatus.DOWNLOAD_FAILED.code() && progress.getPercent() != null){
                precent = new BigDecimal(progress.getPercent());
            }
            progress.setPercent(precent.intValue());
            progress.setStatus(status);
            progress.setUrl(url);
        }
        redisUtil.set(key, JSONUtil.toJsonStr(progress));

    }

    public void zipInputStream(ZipOutputStream out, String key, FileInputStream in) throws Exception {
        out.putNextEntry(new org.apache.tools.zip.ZipEntry(key));
        byte[] bytes = new byte[1024];
        int b = 0;
        while ((b = in.read(bytes)) != -1) {
            out.write(bytes, 0, b);
        }
    }

    public void zipBytes(ZipOutputStream out, String key, byte[] bytes) throws Exception {
        out.putNextEntry(new org.apache.tools.zip.ZipEntry(key));
        out.write(bytes);
    }


}