package com.fdkankan.fyun.oss; import cn.hutool.core.collection.CollUtil; import cn.hutool.core.io.IoUtil; import com.alibaba.fastjson.JSON; import com.fdkankan.common.util.DateExtUtil; import com.fdkankan.fyun.constant.FYunTypeEnum; import com.fdkankan.fyun.face.AbstractFYunFileService; import com.qcloud.cos.COSClient; import com.qcloud.cos.ClientConfig; import com.qcloud.cos.auth.BasicCOSCredentials; import com.qcloud.cos.auth.COSCredentials; import com.qcloud.cos.http.HttpMethodName; import com.qcloud.cos.http.HttpProtocol; import com.qcloud.cos.model.*; import com.qcloud.cos.region.Region; import com.qcloud.cos.utils.IOUtils; import com.qcloud.cos.utils.Md5Utils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.stereotype.Component; import org.springframework.util.CollectionUtils; import org.springframework.util.ObjectUtils; import sun.misc.Cleaner; import java.io.*; import java.math.BigDecimal; import java.net.URL; import java.util.*; import java.util.stream.Collectors; @Component @ConditionalOnProperty(name = "fyun.type", havingValue = "cos") public class CosFileService extends AbstractFYunFileService { private Logger log = LoggerFactory.getLogger(this.getClass().getName()); @Autowired private COSClient cosClient; public static COSClient getClient(){ // 1 初始化用户身份信息(secretId, secretKey)。 // SECRETID 和 SECRETKEY 请登录访问管理控制台 https://console.cloud.tencent.com/cam/capi 进行查看和管理 COSCredentials cred = new BasicCOSCredentials("AKIDlz42sV8sV3pW6UiiieXQuU0QrFbq9Qmx", "A8zmMoz1ufCYuCSmvxulV8hAXnx6EOTX"); // 2 设置 bucket 的地域, COS 地域的简称请参见 https://cloud.tencent.com/document/product/436/6224 // clientConfig 中包含了设置 region, https(默认 http), 超时, 代理等 set 方法, 使用可参见源码或者常见问题 Java SDK 部分。 Region region = new Region("ap-guangzhou"); ClientConfig clientConfig = new ClientConfig(region); // 这里建议设置使用 https 协议 // 从 5.6.54 版本开始,默认使用了 https clientConfig.setHttpProtocol(HttpProtocol.https); // 3 生成 cos 客户端。 COSClient cosClient = new COSClient(cred, clientConfig); return cosClient; } @Override public String uploadFile(String bucket, byte[] data, String remoteFilePath) { String contentMd5 = Md5Utils.md5AsBase64(data); InputStream ins = new ByteArrayInputStream(data); ObjectMetadata metadata = new ObjectMetadata(); metadata.setContentType("text/plain"); metadata.setContentLength(data.length); metadata.setContentMD5(contentMd5); cosClient.putObject(bucket, remoteFilePath, ins, metadata); return null; } @Override public String uploadFile(String bucket, String filePath, String remoteFilePath) { return uploadFile(bucket, filePath, remoteFilePath, null); } @Override public String uploadFile(String bucket, InputStream inputStream, String remoteFilePath) { try { cosClient.putObject(bucket, remoteFilePath, inputStream, new ObjectMetadata()); log.info("文件流上传成功,目标路径:remoteFilePath:{}", remoteFilePath); } catch (Exception e) { log.error("oss上传文件失败,remoteFilePath:"+remoteFilePath, e); } return null; } @Override public String uploadFile(String bucket, String filePath, String remoteFilePath, Map headers) { File file = new File(filePath); if (!file.exists()) { log.warn("要上传的文件不存在,filePath" + filePath); return null; } try (InputStream ins = new FileInputStream(file)){ ObjectMetadata metadata = new ObjectMetadata(); if (filePath.contains(".jpg")) { metadata.setContentType("image/jpeg"); } if (filePath.contains(".mp4")) { metadata.setContentType("video/mp4"); } if (filePath.contains(".mp3")) { metadata.setContentType("audio/mp3"); } if (org.apache.commons.lang3.ObjectUtils.isNotEmpty(headers)) { for (Map.Entry header : headers.entrySet()) { metadata.setHeader(header.getKey(), header.getValue()); } } cosClient.putObject(bucket, remoteFilePath, ins, metadata); log.info("文件上传成功,path:{}", filePath); } catch (Exception e) { log.error("cos上传文件失败,filePath:"+filePath, e); } return null; } @Override public String uploadFileByCommand(String bucket, String filePath, String remoteFilePath) { try { String optType = new File(filePath).isDirectory() ? "folder" : "file"; String command = String.format(fYunConstants.UPLOAD_SH, bucket, filePath, remoteFilePath, FYunTypeEnum.OSS.code(), optType); log.info("开始上传文件, ossPath:{}, srcPath:{}", remoteFilePath, filePath); callshell(command); log.info("上传文件完毕, ossPath:{}, srcPath:{}", remoteFilePath, filePath); } catch (Exception e) { log.error(String.format("上传文件失败, ossPath:%s, srcPath:%s", remoteFilePath, filePath), e); } return null; } @Override public void downloadFileByCommand(String bucket, String filePath, String remoteFilePath) { try { String optType = remoteFilePath.contains(".") ? "file" : "folder"; String command = String.format(fYunConstants.DOWNLOAD_SH, bucket, remoteFilePath, filePath, FYunTypeEnum.COS.code(), optType); log.info("开始下载文件, ossPath:{}, srcPath:{}", remoteFilePath, filePath); callshell(command); log.info("下载文件完毕, ossPath:{}, srcPath:{}", remoteFilePath, filePath); } catch (Exception e) { log.error(String.format("下载文件失败, ossPath:%s, srcPath:%s", remoteFilePath, filePath), e); } } @Override public void downloadByCommand(String bucket, String filePath, String remoteFilePath, boolean isDir) { try { String optType = isDir ? "folder" : "file"; String command = String.format(fYunConstants.DOWNLOAD_SH, bucket, remoteFilePath, filePath, FYunTypeEnum.COS.code(), optType); log.info("开始下载文件, ossPath:{}, srcPath:{}", remoteFilePath, filePath); callshell(command); log.info("下载文件完毕, ossPath:{}, srcPath:{}", remoteFilePath, filePath); } catch (Exception e) { log.error(String.format("下载文件失败, ossPath:%s, srcPath:%s", remoteFilePath, filePath), e); } } @Override public void deleteFile(String bucket, String remoteFilePath) throws IOException { try { cosClient.deleteObject(bucket, remoteFilePath); } catch (Exception e) { log.error("OSS删除文件失败,key:" + remoteFilePath, e); } } @Override public void deleteFolder(String bucket, String remoteFolderPath) { try { if (!remoteFolderPath.endsWith("/")) { remoteFolderPath = remoteFolderPath + "/"; } log.info("开始删除文件夹:{}", remoteFolderPath); boolean flag = true; String nextMaker = null; ListObjectsRequest listObjectsRequest = new ListObjectsRequest(); // 设置 bucket 名称 listObjectsRequest.setBucketName(bucket); // prefix 表示列出的对象名以 prefix 为前缀 // 这里填要列出的目录的相对 bucket 的路径 listObjectsRequest.setPrefix(remoteFolderPath); // 设置最大遍历出多少个对象, 一次 listobject 最大支持1000 listObjectsRequest.setMaxKeys(1000); DeleteObjectsRequest request = new DeleteObjectsRequest(bucket); do { //获取下一页的起始点,它的下一项 ObjectListing objectListing = cosClient.listObjects(listObjectsRequest); List keys = objectListing.getObjectSummaries().parallelStream() .map(cosObjectSummary -> new DeleteObjectsRequest.KeyVersion(cosObjectSummary.getKey())).collect(Collectors.toList()); if (!CollectionUtils.isEmpty(keys)) { request.setKeys(keys); cosClient.deleteObjects(request); } nextMaker = objectListing.getNextMarker(); listObjectsRequest.setMarker(nextMaker); //全部执行完后,为false flag = objectListing.isTruncated(); } while (flag); } catch (Exception e) { log.error("coss删除目录失败,key:" + remoteFolderPath, e); } } @Override public void uploadMulFiles(String bucket, Map filepaths) { try { for (Map.Entry entry : filepaths.entrySet()) { uploadFile(bucket, entry.getKey(), entry.getValue(), null); } } catch (Exception e) { log.error("OSS批量上传文件失败!"); } } @Override public List listRemoteFiles(String bucket, String sourcePath) { List keyList = new ArrayList<>(); try { boolean flag = true; String nextMaker = null; ListObjectsRequest listObjectsRequest = new ListObjectsRequest(); listObjectsRequest.setBucketName(bucket); //指定下一级文件 listObjectsRequest.setPrefix(sourcePath); //设置分页的页容量 listObjectsRequest.setMaxKeys(1000); do { //获取下一页的起始点,它的下一项 ObjectListing objectListing = cosClient.listObjects(listObjectsRequest); List collect = objectListing.getObjectSummaries().parallelStream() .map(COSObjectSummary::getKey).filter(entity -> !entity.endsWith("/")).collect(Collectors.toList()); if (!CollectionUtils.isEmpty(collect)) { keyList.addAll(collect); } nextMaker = objectListing.getNextMarker(); listObjectsRequest.setMarker(nextMaker); //全部执行完后,为false flag = objectListing.isTruncated(); } while (flag); } catch (Exception e) { log.error("获取文件列表失败,path:" + sourcePath, e); } return keyList; } @Override public void copyFileBetweenBucket(String sourceBucketName, String sourcePath, String targetBucketName, String targetPath) { try { List files = listRemoteFiles(sourceBucketName, sourcePath); if (ObjectUtils.isEmpty(files)) { return; } files.stream().forEach(file -> { cosClient.copyObject(sourceBucketName, file, targetBucketName, file.replace(sourcePath, targetPath)); }); } catch (Exception e) { log.error("复制文件或目录失败,key:" + sourcePath, e); } } @Override public void copyFileBetweenBucketParallel(String sourceBucketName, String sourcePath, String targetBucketName, String targetPath) { try { List files = listRemoteFiles(sourceBucketName, sourcePath); if (ObjectUtils.isEmpty(files)) { return; } files.parallelStream().forEach(file -> { cosClient.copyObject(sourceBucketName, file, targetBucketName, file.replace(sourcePath, targetPath)); }); } catch (Exception e) { log.error("复制文件或目录失败,key:" + sourcePath, e); } } @Override public void copyFilesBetweenBucket(String sourceBucketName, String targetBucketName, Map pathMap) { if (ObjectUtils.isEmpty(pathMap)) { return; } try { for (Map.Entry entry : pathMap.entrySet()) { copyFileBetweenBucket(sourceBucketName, entry.getKey(), targetBucketName, entry.getValue()); } } catch (Exception e) { log.error(String.format("批量复制文件失败, sourceBucketName:%s, targetBucketName:%s", sourceBucketName, targetBucketName), e); } } @Override public String getFileContent(String bucketName, String remoteFilePath) { try (COSObject object = cosClient.getObject(bucketName, remoteFilePath)){ InputStream objectContent = object.getObjectContent(); StringBuilder contentJson = new StringBuilder(); try (BufferedReader reader = new BufferedReader(new InputStreamReader(objectContent))) { while (true) { String line = reader.readLine(); if (line == null) break; contentJson.append(line); } } catch (IOException e) { throw e; } return contentJson.toString(); } catch (Exception e) { log.error("获取文件内容失败:key:"+remoteFilePath, e); } return null; } @Override public boolean fileExist(String bucket, String key) { try { return cosClient.doesObjectExist(bucket, key); } catch (Exception e) { log.error("判断文件是否存在失败,key:"+key, e); } return false; } @Override public void downloadFile(String bucket, String remoteFilePath, String localPath) { try { File localFile = new File(localPath); if (!localFile.getParentFile().exists()) { localFile.getParentFile().mkdirs(); } if(localFile.isDirectory()){ String fileName = remoteFilePath.substring(remoteFilePath.lastIndexOf("/")+1); log.info("未配置文件名,使用默认文件名:{}",fileName); localPath = localPath.concat(File.separator).concat(fileName); } GetObjectRequest getObjectRequest = new GetObjectRequest(bucket, remoteFilePath); cosClient.getObject(getObjectRequest, new File(localPath)); } catch (Throwable throwable) { log.error("文件下载失败,key:"+remoteFilePath, throwable); } } @Override public URL getPresignedUrl(String bucket, String url) { java.util.Date expiration = new java.util.Date(); long expTimeMillis = expiration.getTime(); expTimeMillis += 1000 * 60 * 60 * 8; expiration.setTime(expTimeMillis); GeneratePresignedUrlRequest generatePresignedUrlRequest = new GeneratePresignedUrlRequest(bucket, url); generatePresignedUrlRequest.setMethod(HttpMethodName.PUT); generatePresignedUrlRequest.setExpiration(expiration); return cosClient.generatePresignedUrl(generatePresignedUrlRequest); } @Override public long getSubFileNums(String bucket, String url) { long totalSubFileNum = 0; try { if (!url.endsWith("/")) { url = url + "/"; } boolean flag = true; String nextMaker = null; ListObjectsRequest listObjectsRequest = new ListObjectsRequest(); listObjectsRequest.setBucketName(bucket); //指定下一级文件 listObjectsRequest.setPrefix(url); //设置分页的页容量 listObjectsRequest.setMaxKeys(1000); do { //获取下一页的起始点,它的下一项 listObjectsRequest.setMarker(nextMaker); ObjectListing objectListing = cosClient.listObjects(listObjectsRequest); List collect = objectListing.getObjectSummaries().parallelStream() .filter(summary -> !summary.getKey().endsWith("/")).map(COSObjectSummary::getKey).collect(Collectors.toList()); if (!CollectionUtils.isEmpty(collect)) { totalSubFileNum = totalSubFileNum + collect.size(); } nextMaker = objectListing.getNextMarker(); //全部执行完后,为false flag = objectListing.isTruncated(); } while (flag); } catch (Exception e) { log.error("获取文件数量失败,path:" + url, e); } return totalSubFileNum; } @Override public Boolean checkStore(String bucket,String filePath){ ObjectMetadata objectMetadata = cosClient.getObjectMetadata(bucket, filePath); return !isRestoreCompleted(objectMetadata); } @Override public void restoreFolder(String bucket,String folderName){ ObjectMetadata objectMetadata ; List objectList = this.listRemoteFiles(bucket, folderName); if(CollUtil.isEmpty(objectList)){ return; } for (String objectName : objectList) { objectMetadata = cosClient.getObjectMetadata(bucket, objectName); // 校验Object是否为归档类型Object。 StorageClass storageClass = objectMetadata.getStorageClassEnum(); if (storageClass == StorageClass.Archive) { // 解冻Object。 cosClient.restoreObject(bucket, objectName,1); } } } private boolean isRestoreCompleted(ObjectMetadata objectMetadata){ Date restoreExpirationTime = objectMetadata.getRestoreExpirationTime(); if(Objects.nonNull(restoreExpirationTime) && restoreExpirationTime.after(Calendar.getInstance().getTime())){ return true; } return false; } @Override public Integer getRestoreFolderProcess(String bucket,String folderName){ ObjectMetadata objectMetadata ; List objectList = this.listRemoteFiles(bucket, folderName); if(CollUtil.isEmpty(objectList)){ return 100; } List restoreFileList = new ArrayList<>(); for (String objectName : objectList) { objectMetadata = cosClient.getObjectMetadata(bucket, objectName); if(this.isRestoreCompleted(objectMetadata)){ restoreFileList.add(objectName); } } if(objectList.size() <= restoreFileList.size() ){ return 100; } BigDecimal rite = new BigDecimal(restoreFileList.size()).divide(new BigDecimal(objectList.size()),2,BigDecimal.ROUND_HALF_UP); BigDecimal multiply = rite.multiply(new BigDecimal(100)); return multiply.intValue(); } @Override public void restoreFolder(String bucket, String folderName, Integer priority) { List objectList = this.listRemoteFiles(bucket, folderName); if(CollUtil.isEmpty(objectList)){ return; } objectList.parallelStream().forEach(objectName -> { this.restoreFile(bucket, objectName, priority); }); } @Override public void restoreFile(String bucket, String objectName, Integer priority){ ObjectMetadata objectMetadata = cosClient.getObjectMetadata(bucket, objectName); // 校验Object是否为归档类型Object。 StorageClass storageClass = objectMetadata.getStorageClassEnum(); if (storageClass == StorageClass.Archive) { Tier tier = null; switch (priority){ case 1 : tier = Tier.Expedited; break; case 2 : tier = Tier.Standard; break; case 3 : tier = Tier.Bulk; break; default: tier = Tier.Standard; } // 设置 restore 得到的临时副本过期天数为1天 RestoreObjectRequest restoreObjectRequest = new RestoreObjectRequest(bucket, objectName, 1); // 设置恢复模式为 Standard,其他的可选模式包括 Expedited 和 Bulk。若恢复归档存储类型数据,则支持上述三种恢复模式,选择不同恢复模式,在费用和恢复速度上不一样。若恢复深度归档存储类型数据,则仅支持 Standard 和 Bulk 恢复模式 CASJobParameters casJobParameters = new CASJobParameters(); casJobParameters.setTier(tier); restoreObjectRequest.setCASJobParameters(casJobParameters); //开始解冻 cosClient.restoreObject(restoreObjectRequest); } } @Override public Long getSpace(String bucket, String key) { Long total = 0L; boolean flag = true; String nextMaker = null; ListObjectsRequest listObjectsRequest = new ListObjectsRequest(); listObjectsRequest.setBucketName(bucket); //指定下一级文件 listObjectsRequest.setPrefix(key); //设置分页的页容量 listObjectsRequest.setMaxKeys(1000); do { //获取下一页的起始点,它的下一项 listObjectsRequest.setMarker(nextMaker); ObjectListing objectListing = cosClient.listObjects(listObjectsRequest); Long space = objectListing.getObjectSummaries().parallelStream() .mapToLong(COSObjectSummary::getSize).sum(); total += space; nextMaker = objectListing.getNextMarker(); //全部执行完后,为false flag = objectListing.isTruncated(); } while (flag); return total; } @Override public void copyFileToArchive(String sourceBucketName, String sourcePath, String targetBucketName, String targetPath) { try { List files = listRemoteFiles(sourceBucketName, sourcePath); if (ObjectUtils.isEmpty(files)) { return; } files.stream().forEach(file -> { CopyObjectRequest request = new CopyObjectRequest(sourceBucketName, file, targetBucketName, file.replace(sourcePath, targetPath)); request.setStorageClass(StorageClass.Archive); cosClient.copyObject(request); }); } catch (Exception e) { log.error("复制文件或目录失败,key:" + sourcePath, e); } } }