package com.fdkankan.scene.service.impl;
import cn.hutool.core.collection.CollUtil;
import cn.hutool.core.exceptions.ExceptionUtil;
import cn.hutool.core.util.StrUtil;
import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONObject;
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
import com.fdkankan.common.constant.CommonStatus;
import com.fdkankan.common.constant.CommonSuccessStatus;
import com.fdkankan.common.constant.ErrorCode;
import com.fdkankan.common.constant.OperationType;
import com.fdkankan.common.exception.BusinessException;
import com.fdkankan.fyun.config.FYunFileConfig;
import com.fdkankan.fyun.constant.FYunTypeEnum;
import com.fdkankan.fyun.face.FYunFileServiceInterface;
import com.fdkankan.model.constants.ConstantFilePath;
import com.fdkankan.model.constants.UploadFilePath;
import com.fdkankan.model.utils.SceneUtil;
import com.fdkankan.redis.constant.RedisKey;
import com.fdkankan.redis.util.RedisUtil;
import com.fdkankan.scene.bean.SceneBean;
import com.fdkankan.scene.entity.Camera;
import com.fdkankan.scene.entity.SceneCleanOrig;
import com.fdkankan.scene.mapper.ISceneCleanOrigMapper;
import com.fdkankan.scene.mapper.IScenePlusExtMapper;
import com.fdkankan.scene.service.*;
import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.cloud.context.config.annotation.RefreshScope;
import org.springframework.stereotype.Service;
import sun.font.TextRecord;
import java.util.*;
import java.util.stream.Collectors;
/**
*
* 删除oss原始资源记录 服务实现类
*
*
* @author
* @since 2023-03-29
*/
@RefreshScope
@Slf4j
@Service
public class SceneCleanOrigServiceImpl extends ServiceImpl implements ISceneCleanOrigService {
@Value("${scene.cleanOrig.month:#{120}}")
private Integer cleanOrigMonth;
@Value("${scene.coldStorage.month:#{120}}")
private Integer coldStorageMonth;
@Value("${scene.cleanDeleted.month:#{120}}")
private Integer cleanDeletedMonth;
@Value("#{'${scene.cleanTestCamera.snCode:}'.split(',')}")
private List testSnCodeList;
@Value("${scene.cleanTestCamera.month:#{120}}")
private Integer cleanTestCameraMonth;
@Value("${fyun.bucket}")
private String bucket;
@Value("${fyun.coldBucket:#{null}}")
private String coldBucket;
@Autowired
private ICameraService cameraService;
@Autowired
private ISceneColdStorageService sceneColdStorageService;
@Autowired
private FYunFileConfig fYunFileConfig;
@Autowired
private ISceneProService sceneProService;
@Autowired
private IScenePlusService scenePlusService;
@Autowired
private IScenePlusExtService scenePlusExtService;
@Autowired
private FYunFileServiceInterface fYunFileService;
@Autowired
private RedisUtil redisUtil;
@Autowired
private ISceneColdStorageLogService sceneColdStorageLogService;
@Override
public void cleanOrigV4() {
//查询所有计算时间超过限定时间的场景,计算成功、未被删除、最后一次计算后未被删除过的
List sceneBeans = scenePlusService.listCleanOrigScene(cleanOrigMonth);
this.cleanOrig(sceneBeans);
}
@Override
public void cleanOrigV3() {
//查询所有计算时间超过限定时间的场景,计算成功、未被删除
List sceneBeans = sceneProService.listCleanOrigScene(cleanOrigMonth);
this.cleanOrig(sceneBeans);
}
private void cleanOrig(List sceneBeans){
if(CollUtil.isEmpty(sceneBeans)){
return;
}
sceneBeans.parallelStream().forEach(scene->{
boolean lock = this.lock(scene.getDataSource());
try {
if(lock) {
this.cleanOrigHandler(scene);
this.saveLog(scene.getNum(), 1, CommonSuccessStatus.SUCCESS.code(), null);
}
}catch (Exception e){
log.error("删除原始资源失败,num : " + scene.getNum(), e);
this.saveLog(scene.getNum(), 1, CommonSuccessStatus.FAIL.code(), ExceptionUtil.stacktraceToString(e, 3000));
}finally {
this.releaseLock(scene.getDataSource());
}
});
}
private void cleanOrigHandler(SceneBean scene){
String dataSource = scene.getDataSource();
if(StrUtil.isNotEmpty(dataSource)){
String homePath = dataSource.replace(ConstantFilePath.BUILD_MODEL_PATH, ConstantFilePath.OSS_PREFIX);
//由于国内测试和生产用的bucket是同一个,这里需要做一个安全校验,保证不会删错
String fileContent = fYunFileService.getFileContent(homePath.concat("/").concat("data.fdage"));
if(StrUtil.isNotBlank(fileContent)){
JSONObject jsonObject = JSON.parseObject(fileContent);
String snCode = jsonObject.getJSONObject("cam").getString("uuid");
String uuidTime = jsonObject.getString("uuidtime");
if(StrUtil.isEmpty(snCode)
|| StrUtil.isEmpty(uuidTime)
|| !homePath.contains(snCode)
|| !homePath.contains(uuidTime)){
throw new RuntimeException("dataSource与data.fdage文件不匹配");
}else{
fYunFileService.deleteFolder(homePath);
}
}
}
}
private void saveLog(String num, int type, int status, String reason){
//清除旧的日志
this.remove(new LambdaQueryWrapper().eq(SceneCleanOrig::getNum, num));
SceneCleanOrig sceneCleanOrig = new SceneCleanOrig();
sceneCleanOrig.setNum(num);
sceneCleanOrig.setType(type);
sceneCleanOrig.setState(status);
sceneCleanOrig.setReason(reason);
this.saveOrUpdate(sceneCleanOrig);
}
private boolean lock(String dataSource){
Map property = SceneUtil.getPropertyFromDataSource(dataSource);
String homePath = property.get("homePath");
String uuid = property.get("uuid");
String uploadLock = redisUtil.get(String.format(RedisKey.SCENE_OSS_HOME_DIR_UPLOAD, uuid));
//场景正在上传,不删除
if(StrUtil.isNotEmpty(uploadLock)){
return false;
}
redisUtil.set(String.format(RedisKey.SCENE_OSS_HOME_DIR_DELETE, uuid), homePath, 8*60*60);
return true;
}
private void releaseLock(String dataSource){
Map property = SceneUtil.getPropertyFromDataSource(dataSource);
String uuid = property.get("uuid");
redisUtil.del(String.format(RedisKey.SCENE_OSS_HOME_DIR_DELETE, uuid));
}
@Override
public void cleanOss4DeletedSceneV3() {
List sceneBeans = sceneProService.listCleanOss4DeletedScene(cleanDeletedMonth);
this.cleanOrig4Delete(sceneBeans, false, 2);
}
@Override
public void cleanOss4DeletedSceneV4() {
//查询所有计算时间超过限定时间的场景,计算成功、未被删除、最后一次计算后未被删除过的
List sceneBeans = scenePlusService.listCleanOss4DeletedScene(cleanDeletedMonth);
this.cleanOrig4Delete(sceneBeans, true, 2);
}
/**
* 删除已删除场景的原始资源及caches目录(v3场景不需要删除caches目录)
* @param sceneBeans
* @param deleteCaches 是否需要删除caches目录
*/
private void cleanOrig4Delete(List sceneBeans, boolean deleteCaches, Integer type){
if(CollUtil.isEmpty(sceneBeans)){
return;
}
sceneBeans.parallelStream().forEach(scene->{
try {
//删除caches文件
if(deleteCaches){
this.deleteResultCaches(scene.getNum());
}
//删除原始资源
this.cleanOrigHandler(scene);
this.saveLog(scene.getNum(), type, CommonSuccessStatus.SUCCESS.code(), null);
}catch (Exception e){
log.error("删除已删除场景资源失败,num : " + scene.getNum(), e);
this.saveLog(scene.getNum(), type, CommonSuccessStatus.FAIL.code(), ExceptionUtil.stacktraceToString(e, 3000));
}
});
}
private void deleteResultCaches(String num){
String cachesPath = String.format(UploadFilePath.scene_result_data_path, num).concat("caches");
if(CollUtil.isEmpty(fYunFileService.listRemoteFiles(cachesPath))){
return;
}
fYunFileService.deleteFolder(cachesPath);
}
@Override
public void cleanOss4TestCameraV3() {
List cameras = cameraService.listBySnCodes(testSnCodeList);
if(CollUtil.isEmpty(cameras)){
return;
}
Set cameraIds = cameras.stream().map(Camera::getId).collect(Collectors.toSet());
List sceneBeans = sceneProService.listCleanOss4TestCamera(cameraIds, cleanTestCameraMonth);
this.cleanOrig4Delete(sceneBeans, false, 3);
}
@Override
public void cleanOss4TestCameraV4() {
List cameras = cameraService.listBySnCodes(testSnCodeList);
if(CollUtil.isEmpty(cameras)){
return;
}
Set cameraIds = cameras.stream().map(Camera::getId).collect(Collectors.toSet());
List sceneBeans = scenePlusService.listCleanOss4TestCamera(cameraIds, cleanTestCameraMonth);
this.cleanOrig4Delete(sceneBeans, true, 3);
}
@Override
public void coldStorageHomeV3() {
//查询所有计算时间超过限定时间的场景,计算成功、未被删除
List sceneBeans = sceneProService.listColdStorageScene(coldStorageMonth);
this.coldStorage(sceneBeans);
}
@Override
public void coldStorageHomeV4() {
//查询所有计算时间超过限定时间的场景,计算成功、未被删除
List sceneBeans = scenePlusService.listColdStorageScene(coldStorageMonth);
this.coldStorage(sceneBeans);
}
private void coldStorage(List sceneBeans){
if(CollUtil.isEmpty(sceneBeans)){
return;
}
sceneBeans.parallelStream().forEach(scene->{
boolean lock = this.lock(scene.getDataSource());
try {
if(lock) {
this.coldStorageHandler(scene);
sceneColdStorageLogService.saveLog(scene.getNum(), scene.getDataSource(), 1, 1, null);
sceneColdStorageService.save(scene.getNum(), 1, coldBucket, bucket);
}
}catch (Exception e){
log.error("冷归档失败,num:{}" + scene.getNum(), e);
sceneColdStorageLogService.saveLog(scene.getNum(), scene.getDataSource(),1, CommonSuccessStatus.FAIL.code(), ExceptionUtil.stacktraceToString(e, 3000));
}finally {
this.releaseLock(scene.getDataSource());
}
});
}
private void coldStorageHandler(SceneBean scene){
String dataSource = scene.getDataSource();
if(StrUtil.isEmpty(dataSource) || dataSource.length() < 10) {
return;
}
String homePath = dataSource.replace(ConstantFilePath.BUILD_MODEL_PATH, ConstantFilePath.OSS_PREFIX);
//将文件复制到冷归档bucket
if(fYunFileConfig.getFyunType().equals(FYunTypeEnum.AWS.code())){
fYunFileService.copyFileToArchive(bucket, homePath, coldBucket, homePath);
}else{
fYunFileService.copyFileBetweenBucket(bucket, homePath, coldBucket, homePath);
}
List origList = fYunFileService.listRemoteFiles(bucket, homePath);
List coldList = fYunFileService.listRemoteFiles(coldBucket, homePath);
if(origList.size() != coldList.size()){
throw new RuntimeException("复制文件到冷归档bucket失败");
}
//删除标准bucket文件
fYunFileService.deleteFolder(homePath);
}
}