package com.fdkankan.scene.service.impl; import cn.hutool.core.collection.CollUtil; import cn.hutool.core.exceptions.ExceptionUtil; import cn.hutool.core.util.StrUtil; import com.alibaba.fastjson.JSON; import com.alibaba.fastjson.JSONObject; import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper; import com.fdkankan.common.constant.CommonStatus; import com.fdkankan.common.constant.CommonSuccessStatus; import com.fdkankan.common.constant.ErrorCode; import com.fdkankan.common.constant.OperationType; import com.fdkankan.common.exception.BusinessException; import com.fdkankan.fyun.config.FYunFileConfig; import com.fdkankan.fyun.constant.FYunTypeEnum; import com.fdkankan.fyun.face.FYunFileServiceInterface; import com.fdkankan.model.constants.ConstantFilePath; import com.fdkankan.model.constants.UploadFilePath; import com.fdkankan.model.utils.SceneUtil; import com.fdkankan.redis.constant.RedisKey; import com.fdkankan.redis.util.RedisUtil; import com.fdkankan.scene.bean.SceneBean; import com.fdkankan.scene.entity.Camera; import com.fdkankan.scene.entity.SceneCleanOrig; import com.fdkankan.scene.mapper.ISceneCleanOrigMapper; import com.fdkankan.scene.mapper.IScenePlusExtMapper; import com.fdkankan.scene.service.*; import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.cloud.context.config.annotation.RefreshScope; import org.springframework.stereotype.Service; import sun.font.TextRecord; import java.util.*; import java.util.stream.Collectors; /** *

* 删除oss原始资源记录 服务实现类 *

* * @author * @since 2023-03-29 */ @RefreshScope @Slf4j @Service public class SceneCleanOrigServiceImpl extends ServiceImpl implements ISceneCleanOrigService { @Value("${scene.cleanOrig.month:#{120}}") private Integer cleanOrigMonth; @Value("${scene.coldStorage.month:#{120}}") private Integer coldStorageMonth; @Value("${scene.cleanDeleted.month:#{120}}") private Integer cleanDeletedMonth; @Value("#{'${scene.cleanTestCamera.snCode:}'.split(',')}") private List testSnCodeList; @Value("${scene.cleanTestCamera.month:#{120}}") private Integer cleanTestCameraMonth; @Value("${fyun.bucket}") private String bucket; @Value("${fyun.coldBucket:#{null}}") private String coldBucket; @Autowired private ICameraService cameraService; @Autowired private ISceneColdStorageService sceneColdStorageService; @Autowired private FYunFileConfig fYunFileConfig; @Autowired private ISceneProService sceneProService; @Autowired private IScenePlusService scenePlusService; @Autowired private IScenePlusExtService scenePlusExtService; @Autowired private FYunFileServiceInterface fYunFileService; @Autowired private RedisUtil redisUtil; @Autowired private ISceneColdStorageLogService sceneColdStorageLogService; @Override public void cleanOrigV4() { //查询所有计算时间超过限定时间的场景,计算成功、未被删除、最后一次计算后未被删除过的 List sceneBeans = scenePlusService.listCleanOrigScene(cleanOrigMonth); this.cleanOrig(sceneBeans); } @Override public void cleanOrigV3() { //查询所有计算时间超过限定时间的场景,计算成功、未被删除 List sceneBeans = sceneProService.listCleanOrigScene(cleanOrigMonth); this.cleanOrig(sceneBeans); } private void cleanOrig(List sceneBeans){ if(CollUtil.isEmpty(sceneBeans)){ return; } sceneBeans.parallelStream().forEach(scene->{ boolean lock = this.lock(scene.getDataSource()); try { if(lock) { this.cleanOrigHandler(scene); this.saveLog(scene.getNum(), 1, CommonSuccessStatus.SUCCESS.code(), null); } }catch (Exception e){ log.error("删除原始资源失败,num : " + scene.getNum(), e); this.saveLog(scene.getNum(), 1, CommonSuccessStatus.FAIL.code(), ExceptionUtil.stacktraceToString(e, 3000)); }finally { this.releaseLock(scene.getDataSource()); } }); } private void cleanOrigHandler(SceneBean scene){ String dataSource = scene.getDataSource(); if(StrUtil.isNotEmpty(dataSource)){ String homePath = dataSource.replace(ConstantFilePath.BUILD_MODEL_PATH, ConstantFilePath.OSS_PREFIX); //由于国内测试和生产用的bucket是同一个,这里需要做一个安全校验,保证不会删错 String fileContent = fYunFileService.getFileContent(homePath.concat("/").concat("data.fdage")); if(StrUtil.isNotBlank(fileContent)){ JSONObject jsonObject = JSON.parseObject(fileContent); String snCode = jsonObject.getJSONObject("cam").getString("uuid"); String uuidTime = jsonObject.getString("uuidtime"); if(StrUtil.isEmpty(snCode) || StrUtil.isEmpty(uuidTime) || !homePath.contains(snCode) || !homePath.contains(uuidTime)){ throw new RuntimeException("dataSource与data.fdage文件不匹配"); }else{ fYunFileService.deleteFolder(homePath); } } } } private void saveLog(String num, int type, int status, String reason){ //清除旧的日志 this.remove(new LambdaQueryWrapper().eq(SceneCleanOrig::getNum, num)); SceneCleanOrig sceneCleanOrig = new SceneCleanOrig(); sceneCleanOrig.setNum(num); sceneCleanOrig.setType(type); sceneCleanOrig.setState(status); sceneCleanOrig.setReason(reason); this.saveOrUpdate(sceneCleanOrig); } private boolean lock(String dataSource){ Map property = SceneUtil.getPropertyFromDataSource(dataSource); String homePath = property.get("homePath"); String uuid = property.get("uuid"); String uploadLock = redisUtil.get(String.format(RedisKey.SCENE_OSS_HOME_DIR_UPLOAD, uuid)); //场景正在上传,不删除 if(StrUtil.isNotEmpty(uploadLock)){ return false; } redisUtil.set(String.format(RedisKey.SCENE_OSS_HOME_DIR_DELETE, uuid), homePath, 8*60*60); return true; } private void releaseLock(String dataSource){ Map property = SceneUtil.getPropertyFromDataSource(dataSource); String uuid = property.get("uuid"); redisUtil.del(String.format(RedisKey.SCENE_OSS_HOME_DIR_DELETE, uuid)); } @Override public void cleanOss4DeletedSceneV3() { List sceneBeans = sceneProService.listCleanOss4DeletedScene(cleanDeletedMonth); this.cleanOrig4Delete(sceneBeans, false, 2); } @Override public void cleanOss4DeletedSceneV4() { //查询所有计算时间超过限定时间的场景,计算成功、未被删除、最后一次计算后未被删除过的 List sceneBeans = scenePlusService.listCleanOss4DeletedScene(cleanDeletedMonth); this.cleanOrig4Delete(sceneBeans, true, 2); } /** * 删除已删除场景的原始资源及caches目录(v3场景不需要删除caches目录) * @param sceneBeans * @param deleteCaches 是否需要删除caches目录 */ private void cleanOrig4Delete(List sceneBeans, boolean deleteCaches, Integer type){ if(CollUtil.isEmpty(sceneBeans)){ return; } sceneBeans.parallelStream().forEach(scene->{ try { //删除caches文件 if(deleteCaches){ this.deleteResultCaches(scene.getNum()); } //删除原始资源 this.cleanOrigHandler(scene); this.saveLog(scene.getNum(), type, CommonSuccessStatus.SUCCESS.code(), null); }catch (Exception e){ log.error("删除已删除场景资源失败,num : " + scene.getNum(), e); this.saveLog(scene.getNum(), type, CommonSuccessStatus.FAIL.code(), ExceptionUtil.stacktraceToString(e, 3000)); } }); } private void deleteResultCaches(String num){ String cachesPath = String.format(UploadFilePath.scene_result_data_path, num).concat("caches"); if(CollUtil.isEmpty(fYunFileService.listRemoteFiles(cachesPath))){ return; } fYunFileService.deleteFolder(cachesPath); } @Override public void cleanOss4TestCameraV3() { List cameras = cameraService.listBySnCodes(testSnCodeList); if(CollUtil.isEmpty(cameras)){ return; } Set cameraIds = cameras.stream().map(Camera::getId).collect(Collectors.toSet()); List sceneBeans = sceneProService.listCleanOss4TestCamera(cameraIds, cleanTestCameraMonth); this.cleanOrig4Delete(sceneBeans, false, 3); } @Override public void cleanOss4TestCameraV4() { List cameras = cameraService.listBySnCodes(testSnCodeList); if(CollUtil.isEmpty(cameras)){ return; } Set cameraIds = cameras.stream().map(Camera::getId).collect(Collectors.toSet()); List sceneBeans = scenePlusService.listCleanOss4TestCamera(cameraIds, cleanTestCameraMonth); this.cleanOrig4Delete(sceneBeans, true, 3); } @Override public void coldStorageHomeV3() { //查询所有计算时间超过限定时间的场景,计算成功、未被删除 List sceneBeans = sceneProService.listColdStorageScene(coldStorageMonth); this.coldStorage(sceneBeans); } @Override public void coldStorageHomeV4() { //查询所有计算时间超过限定时间的场景,计算成功、未被删除 List sceneBeans = scenePlusService.listColdStorageScene(coldStorageMonth); this.coldStorage(sceneBeans); } private void coldStorage(List sceneBeans){ if(CollUtil.isEmpty(sceneBeans)){ return; } sceneBeans.parallelStream().forEach(scene->{ boolean lock = this.lock(scene.getDataSource()); try { if(lock) { this.coldStorageHandler(scene); sceneColdStorageLogService.saveLog(scene.getNum(), scene.getDataSource(), 1, 1, null); sceneColdStorageService.save(scene.getNum(), 1, coldBucket, bucket); } }catch (Exception e){ log.error("冷归档失败,num:{}" + scene.getNum(), e); sceneColdStorageLogService.saveLog(scene.getNum(), scene.getDataSource(),1, CommonSuccessStatus.FAIL.code(), ExceptionUtil.stacktraceToString(e, 3000)); }finally { this.releaseLock(scene.getDataSource()); } }); } private void coldStorageHandler(SceneBean scene){ String dataSource = scene.getDataSource(); if(StrUtil.isEmpty(dataSource) || dataSource.length() < 10) { return; } String homePath = dataSource.replace(ConstantFilePath.BUILD_MODEL_PATH, ConstantFilePath.OSS_PREFIX); //将文件复制到冷归档bucket if(fYunFileConfig.getFyunType().equals(FYunTypeEnum.AWS.code())){ fYunFileService.copyFileToArchive(bucket, homePath, coldBucket, homePath); }else{ fYunFileService.copyFileBetweenBucket(bucket, homePath, coldBucket, homePath); } List origList = fYunFileService.listRemoteFiles(bucket, homePath); List coldList = fYunFileService.listRemoteFiles(coldBucket, homePath); if(origList.size() != coldList.size()){ throw new RuntimeException("复制文件到冷归档bucket失败"); } //删除标准bucket文件 fYunFileService.deleteFolder(homePath); } }