4dkankan-center-scene/src/main/java/com/fdkankan/scene/service/impl/SceneCleanOrigServiceImpl.java

package com.fdkankan.scene.service.impl;

import cn.hutool.core.collection.CollUtil;
import cn.hutool.core.exceptions.ExceptionUtil;
import cn.hutool.core.util.StrUtil;
import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONObject;
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
import com.fdkankan.common.constant.CommonStatus;
import com.fdkankan.common.constant.CommonSuccessStatus;
import com.fdkankan.common.constant.ErrorCode;
import com.fdkankan.common.constant.OperationType;
import com.fdkankan.common.exception.BusinessException;
import com.fdkankan.fyun.config.FYunFileConfig;
import com.fdkankan.fyun.constant.FYunTypeEnum;
import com.fdkankan.fyun.face.FYunFileServiceInterface;
import com.fdkankan.model.constants.ConstantFilePath;
import com.fdkankan.model.constants.UploadFilePath;
import com.fdkankan.model.utils.SceneUtil;
import com.fdkankan.redis.constant.RedisKey;
import com.fdkankan.redis.util.RedisUtil;
import com.fdkankan.scene.bean.SceneBean;
import com.fdkankan.scene.entity.Camera;
import com.fdkankan.scene.entity.SceneCleanOrig;
import com.fdkankan.scene.mapper.ISceneCleanOrigMapper;
import com.fdkankan.scene.mapper.IScenePlusExtMapper;
import com.fdkankan.scene.service.*;
import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.cloud.context.config.annotation.RefreshScope;
import org.springframework.stereotype.Service;
import sun.font.TextRecord;

import java.util.*;
import java.util.stream.Collectors;

/**
 * <p>
 * 删除oss原始资源记录 服务实现类
 * </p>
 *
 * @author 
 * @since 2023-03-29
 */
@RefreshScope
@Slf4j
@Service
public class SceneCleanOrigServiceImpl extends ServiceImpl<ISceneCleanOrigMapper, SceneCleanOrig> implements ISceneCleanOrigService {

    @Value("${scene.cleanOrig.month:#{120}}")
    private Integer cleanOrigMonth;
    @Value("${scene.coldStorage.month:#{120}}")
    private Integer coldStorageMonth;
    @Value("${scene.cleanDeleted.month:#{120}}")
    private Integer cleanDeletedMonth;
    @Value("#{'${scene.cleanTestCamera.snCode:}'.split(',')}")
    private List<String> testSnCodeList;
    @Value("${scene.cleanTestCamera.month:#{120}}")
    private Integer cleanTestCameraMonth;
    @Value("${fyun.bucket}")
    private String bucket;
    @Value("${fyun.coldBucket:#{null}}")
    private String coldBucket;
    @Autowired
    private ICameraService cameraService;
    @Autowired
    private ISceneColdStorageService sceneColdStorageService;
    @Autowired
    private FYunFileConfig fYunFileConfig;


    @Autowired
    private ISceneProService sceneProService;
    @Autowired
    private IScenePlusService scenePlusService;
    @Autowired
    private IScenePlusExtService scenePlusExtService;
    @Autowired
    private FYunFileServiceInterface fYunFileService;
    @Autowired
    private RedisUtil redisUtil;
    @Autowired
    private ISceneColdStorageLogService sceneColdStorageLogService;

    @Override
    public void cleanOrigV4() {
        //查询所有计算时间超过限定时间的场景,计算成功、未被删除、最后一次计算后未被删除过的
        List<SceneBean> sceneBeans = scenePlusService.listCleanOrigScene(cleanOrigMonth);
        this.cleanOrig(sceneBeans);
    }

    @Override
    public void cleanOrigV3() {
        //查询所有计算时间超过限定时间的场景,计算成功、未被删除
        List<SceneBean> sceneBeans = sceneProService.listCleanOrigScene(cleanOrigMonth);
        this.cleanOrig(sceneBeans);

    }

    private void cleanOrig(List<SceneBean> sceneBeans){
        if(CollUtil.isEmpty(sceneBeans)){
            return;
        }
        sceneBeans.parallelStream().forEach(scene->{
            boolean lock = this.lock(scene.getDataSource());
            try {
                if(lock) {
                    this.cleanOrigHandler(scene);
                    this.saveLog(scene.getNum(), 1, CommonSuccessStatus.SUCCESS.code(), null);
                }
            }catch (Exception e){
                log.error("删除原始资源失败,num : " + scene.getNum(), e);
                this.saveLog(scene.getNum(), 1, CommonSuccessStatus.FAIL.code(), ExceptionUtil.stacktraceToString(e, 3000));
            }finally {
                this.releaseLock(scene.getDataSource());
            }
        });
    }

    private void cleanOrigHandler(SceneBean scene){
        String dataSource = scene.getDataSource();
        if(StrUtil.isNotEmpty(dataSource)){
            String homePath = dataSource.replace(ConstantFilePath.BUILD_MODEL_PATH, ConstantFilePath.OSS_PREFIX);
            //由于国内测试和生产用的bucket是同一个，这里需要做一个安全校验，保证不会删错
            String fileContent = fYunFileService.getFileContent(homePath.concat("/").concat("data.fdage"));
            if(StrUtil.isNotBlank(fileContent)){
                JSONObject jsonObject = JSON.parseObject(fileContent);
                String snCode = jsonObject.getJSONObject("cam").getString("uuid");
                String uuidTime = jsonObject.getString("uuidtime");
                if(StrUtil.isEmpty(snCode)
                        || StrUtil.isEmpty(uuidTime)
                        || !homePath.contains(snCode)
                        || !homePath.contains(uuidTime)){
                    throw new RuntimeException("dataSource与data.fdage文件不匹配");
                }else{
                    fYunFileService.deleteFolder(homePath);
                }
            }
        }
    }




    private void saveLog(String num, int type, int status, String reason){
        //清除旧的日志
        this.remove(new LambdaQueryWrapper<SceneCleanOrig>().eq(SceneCleanOrig::getNum, num));
        SceneCleanOrig sceneCleanOrig = new SceneCleanOrig();
        sceneCleanOrig.setNum(num);
        sceneCleanOrig.setType(type);
        sceneCleanOrig.setState(status);
        sceneCleanOrig.setReason(reason);
        this.saveOrUpdate(sceneCleanOrig);
    }

    private boolean lock(String dataSource){
        Map<String, String> property = SceneUtil.getPropertyFromDataSource(dataSource);
        String homePath = property.get("homePath");
        String uuid = property.get("uuid");

        String uploadLock = redisUtil.get(String.format(RedisKey.SCENE_OSS_HOME_DIR_UPLOAD, uuid));
        //场景正在上传，不删除
        if(StrUtil.isNotEmpty(uploadLock)){
            return false;
        }
        redisUtil.set(String.format(RedisKey.SCENE_OSS_HOME_DIR_DELETE, uuid), homePath, 8*60*60);
        return true;
    }

    private void releaseLock(String dataSource){
        Map<String, String> property = SceneUtil.getPropertyFromDataSource(dataSource);
        String uuid = property.get("uuid");
        redisUtil.del(String.format(RedisKey.SCENE_OSS_HOME_DIR_DELETE, uuid));
    }


    @Override
    public void cleanOss4DeletedSceneV3() {
        List<SceneBean> sceneBeans = sceneProService.listCleanOss4DeletedScene(cleanDeletedMonth);
        this.cleanOrig4Delete(sceneBeans, false, 2);
    }

    @Override
    public void cleanOss4DeletedSceneV4() {
        //查询所有计算时间超过限定时间的场景,计算成功、未被删除、最后一次计算后未被删除过的
        List<SceneBean> sceneBeans = scenePlusService.listCleanOss4DeletedScene(cleanDeletedMonth);
        this.cleanOrig4Delete(sceneBeans, true, 2);
    }

    /**
     * 删除已删除场景的原始资源及caches目录（v3场景不需要删除caches目录）
     * @param sceneBeans
     * @param deleteCaches 是否需要删除caches目录
     */
    private void cleanOrig4Delete(List<SceneBean> sceneBeans, boolean deleteCaches, Integer type){
        if(CollUtil.isEmpty(sceneBeans)){
            return;
        }
        sceneBeans.parallelStream().forEach(scene->{
            try {
                //删除caches文件
                if(deleteCaches){
                    this.deleteResultCaches(scene.getNum());
                }
                //删除原始资源
                this.cleanOrigHandler(scene);

                this.saveLog(scene.getNum(), type, CommonSuccessStatus.SUCCESS.code(), null);
            }catch (Exception e){
                log.error("删除已删除场景资源失败,num : " + scene.getNum(), e);
                this.saveLog(scene.getNum(), type, CommonSuccessStatus.FAIL.code(), ExceptionUtil.stacktraceToString(e, 3000));
            }
        });
    }

    private void deleteResultCaches(String num){
        String cachesPath = String.format(UploadFilePath.scene_result_data_path, num).concat("caches");
        if(CollUtil.isEmpty(fYunFileService.listRemoteFiles(cachesPath))){
            return;
        }
        fYunFileService.deleteFolder(cachesPath);
    }

    @Override
    public void cleanOss4TestCameraV3() {
        List<Camera> cameras = cameraService.listBySnCodes(testSnCodeList);
        if(CollUtil.isEmpty(cameras)){
            return;
        }
        Set<Long> cameraIds = cameras.stream().map(Camera::getId).collect(Collectors.toSet());
        List<SceneBean> sceneBeans = sceneProService.listCleanOss4TestCamera(cameraIds, cleanTestCameraMonth);
        this.cleanOrig4Delete(sceneBeans, false, 3);
    }

    @Override
    public void cleanOss4TestCameraV4() {
        List<Camera> cameras = cameraService.listBySnCodes(testSnCodeList);
        if(CollUtil.isEmpty(cameras)){
            return;
        }
        Set<Long> cameraIds = cameras.stream().map(Camera::getId).collect(Collectors.toSet());
        List<SceneBean> sceneBeans = scenePlusService.listCleanOss4TestCamera(cameraIds, cleanTestCameraMonth);
        this.cleanOrig4Delete(sceneBeans, true, 3);
    }

    @Override
    public void coldStorageHomeV3() {
        //查询所有计算时间超过限定时间的场景,计算成功、未被删除
        List<SceneBean> sceneBeans = sceneProService.listColdStorageScene(coldStorageMonth);
        this.coldStorage(sceneBeans);
    }

    @Override
    public void coldStorageHomeV4() {
        //查询所有计算时间超过限定时间的场景,计算成功、未被删除
        List<SceneBean> sceneBeans = scenePlusService.listColdStorageScene(coldStorageMonth);
        this.coldStorage(sceneBeans);
    }

    private void coldStorage(List<SceneBean> sceneBeans){
        if(CollUtil.isEmpty(sceneBeans)){
            return;
        }
        sceneBeans.parallelStream().forEach(scene->{
            boolean lock = this.lock(scene.getDataSource());
            try {
                if(lock) {
                    this.coldStorageHandler(scene);
                    sceneColdStorageLogService.saveLog(scene.getNum(), scene.getDataSource(), 1, 1, null);
                    sceneColdStorageService.save(scene.getNum(), 1, coldBucket, bucket);
                }
            }catch (Exception e){
                log.error("冷归档失败,num:{}" + scene.getNum(), e);
                sceneColdStorageLogService.saveLog(scene.getNum(), scene.getDataSource(),1, CommonSuccessStatus.FAIL.code(), ExceptionUtil.stacktraceToString(e, 3000));
            }finally {
                this.releaseLock(scene.getDataSource());
            }
        });
    }

    private void coldStorageHandler(SceneBean scene){
        String dataSource = scene.getDataSource();
        if(StrUtil.isEmpty(dataSource) || dataSource.length() < 10) {
            return;
        }
        String homePath = dataSource.replace(ConstantFilePath.BUILD_MODEL_PATH, ConstantFilePath.OSS_PREFIX);

        //将文件复制到冷归档bucket
        if(fYunFileConfig.getFyunType().equals(FYunTypeEnum.AWS.code())){
            fYunFileService.copyFileToArchive(bucket, homePath, coldBucket, homePath);
        }else{
            fYunFileService.copyFileBetweenBucket(bucket, homePath, coldBucket, homePath);
        }

        List<String> origList = fYunFileService.listRemoteFiles(bucket, homePath);
        List<String> coldList = fYunFileService.listRemoteFiles(coldBucket, homePath);
        if(origList.size() != coldList.size()){
            throw new RuntimeException("复制文件到冷归档bucket失败");
        }

        //删除标准bucket文件
        fYunFileService.deleteFolder(homePath);
    }
}