obs

4dkankan-utils-fyun-obs/pom.xml

@@ -0,0 +1,28 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+    <parent>
+        <artifactId>4dkankan-utils</artifactId>
+        <groupId>com.fdkankan</groupId>
+        <version>3.0.0-SNAPSHOT</version>
+    </parent>
+    <artifactId>4dkankan-utils-fyun-obs</artifactId>
+    <properties>
+        <java.version>1.8</java.version>
+    </properties>
+    <dependencies>
+        <!--        uploadToOssUtil-->
+        <dependency>
+            <groupId>com.huaweicloud</groupId>
+            <artifactId>esdk-obs-java-bundle</artifactId>
+            <version>3.22.12</version>
+        </dependency>
+        <dependency>
+            <groupId>com.fdkankan</groupId>
+            <artifactId>4dkankan-utils-fyun-parent</artifactId>
+            <version>3.0.0-SNAPSHOT</version>
+        </dependency>
+    </dependencies>
+
+</project>

4dkankan-utils-fyun-obs/src/main/java/com/fdkankan/fyun/oss/ObsFileService.java

@@ -0,0 +1,466 @@
+package com.fdkankan.fyun.oss;
+
+import cn.hutool.core.collection.CollUtil;
+import com.fdkankan.fyun.constant.FYunTypeEnum;
+import com.fdkankan.fyun.face.AbstractFYunFileService;
+import com.obs.services.ObsClient;
+import com.obs.services.model.*;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
+import org.springframework.stereotype.Component;
+import org.springframework.util.CollectionUtils;
+import org.springframework.util.ObjectUtils;
+
+import java.io.*;
+import java.math.BigDecimal;
+import java.net.URL;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+
+@Component
+@ConditionalOnProperty(name = "fyun.type", havingValue = "obs")
+public class ObsFileService extends AbstractFYunFileService {
+
+    private Logger log = LoggerFactory.getLogger(this.getClass().getName());
+
+    @Autowired
+    private ObsClient obsClient;
+
+    @Override
+    public String uploadFile(String bucket, byte[] data, String remoteFilePath) {
+        try {
+            obsClient.putObject(bucket, remoteFilePath, new ByteArrayInputStream(data));
+        } catch (Exception e) {
+            log.error("oss上传文件失败,remoteFilePath:" + remoteFilePath, e);
+        }
+        return null;
+    }
+
+    @Override
+    public String uploadFile(String bucket, String filePath, String remoteFilePath) {
+        return uploadFile(bucket, filePath, remoteFilePath, null);
+    }
+
+    @Override
+    public String uploadFile(String bucket, InputStream inputStream, String remoteFilePath) {
+        try {
+            obsClient.putObject(bucket, remoteFilePath, inputStream);
+            log.info("文件流上传成功，目标路径：remoteFilePath:{}", remoteFilePath);
+        } catch (Exception e) {
+            log.error("oss上传文件失败,remoteFilePath:"+remoteFilePath, e);
+        }
+        return null;
+    }
+
+    @Override
+    public String uploadFile(String bucket, String filePath, String remoteFilePath, Map<String, String> headers) {
+        try {
+            File file = new File(filePath);
+            if (!file.exists()) {
+                log.warn("要上传的文件不存在,filePath" + filePath);
+                return null;
+            }
+            ObjectMetadata metadata = new ObjectMetadata();
+            if (filePath.contains(".jpg")) {
+                metadata.setContentType("image/jpeg");
+            }
+            if (filePath.contains(".mp4")) {
+                metadata.setContentType("video/mp4");
+            }
+            if (filePath.contains(".mp3")) {
+                metadata.setContentType("audio/mp3");
+            }
+            if (CollUtil.isNotEmpty(headers)) {
+                Map<String, Object> collect = headers.keySet().stream().collect(Collectors.toMap(v -> v, v -> headers.get(v)));
+                metadata.setMetadata(collect);
+            }
+            obsClient.putObject(bucket, remoteFilePath, file, metadata);
+            log.info("文件上传成功，path:{}", filePath);
+        } catch (Exception e) {
+            log.error("oss上传文件失败,filePath:"+filePath, e);
+        }
+        return null;
+    }
+
+    @Override
+    public String uploadFileByCommand(String bucket, String filePath, String remoteFilePath) {
+        try {
+            String optType = new File(filePath).isDirectory() ? "folder" : "file";
+            String command = String.format(fYunConstants.UPLOAD_SH, bucket, filePath, remoteFilePath, FYunTypeEnum.OSS.code(), optType);
+            log.info("开始上传文件, ossPath:{}, srcPath:{}", remoteFilePath, filePath);
+            callshell(command);
+            log.info("上传文件完毕, ossPath:{}, srcPath:{}", remoteFilePath, filePath);
+        } catch (Exception e) {
+            log.error(String.format("上传文件失败, ossPath:%s, srcPath:%s", remoteFilePath, filePath), e);
+        }
+        return null;
+    }
+
+    @Override
+    public void downloadFileByCommand(String bucket, String filePath, String remoteFilePath) {
+        try {
+            String optType = remoteFilePath.contains(".") ? "file" : "folder";
+            String command = String.format(fYunConstants.DOWNLOAD_SH, bucket, remoteFilePath, filePath, FYunTypeEnum.OSS.code(), optType);
+            log.info("开始下载文件, ossPath:{}, srcPath:{}", remoteFilePath, filePath);
+            callshell(command);
+            log.info("下载文件完毕, ossPath:{}, srcPath:{}", remoteFilePath, filePath);
+        } catch (Exception e) {
+            log.error(String.format("下载文件失败, ossPath:%s, srcPath:%s", remoteFilePath, filePath), e);
+        }
+    }
+
+    @Override
+    public void deleteFile(String bucket, String remoteFilePath) throws IOException {
+        try {
+            obsClient.deleteObject(bucket, remoteFilePath);
+        } catch (Exception e) {
+            log.error("OSS删除文件失败,key:" + remoteFilePath, e);
+        }
+    }
+
+    @Override
+    public void deleteFolder(String bucket, String remoteFolderPath) {
+        try {
+
+            if (!remoteFolderPath.endsWith("/")) {
+                remoteFolderPath = remoteFolderPath + "/";
+            }
+            log.info("开始删除文件夹：{}", remoteFolderPath);
+            boolean flag = true;
+            String nextMaker = null;
+            ListObjectsRequest listObjectsRequest = new ListObjectsRequest(bucket);
+            listObjectsRequest.setPrefix(remoteFolderPath);
+            listObjectsRequest.setMaxKeys(500);
+
+            do {
+                DeleteObjectsRequest request = new DeleteObjectsRequest(bucket);
+                //获取下一页的起始点，它的下一项
+                listObjectsRequest.setMarker(nextMaker);
+                ObjectListing objectListing = obsClient.listObjects(listObjectsRequest);
+                objectListing.getObjectSummaries().parallelStream().forEach(v->{
+                    request.addKeyAndVersion(v.getObjectKey());
+                });
+                obsClient.deleteObjects(request);
+                nextMaker = objectListing.getNextMarker();
+                //全部执行完后，为false
+                flag = objectListing.isTruncated();
+            } while (flag);
+        } catch (Exception e) {
+            log.error("OSS删除文件失败,key:" + remoteFolderPath, e);
+        }
+    }
+
+    @Override
+    public void uploadMulFiles(String bucket, Map<String, String> filepaths) {
+        try {
+            for (Map.Entry<String, String> entry : filepaths.entrySet()) {
+                uploadFile(bucket, entry.getKey(), entry.getValue(), null);
+            }
+        } catch (Exception e) {
+            log.error("OSS批量上传文件失败！");
+        }
+    }
+
+    @Override
+    public List<String> listRemoteFiles(String bucket, String sourcePath) {
+        List<String> keyList = new ArrayList<>();
+        try {
+            boolean flag = true;
+            String nextMaker = null;
+            ListObjectsRequest listObjectsRequest = new ListObjectsRequest(bucket);
+            //指定下一级文件
+            listObjectsRequest.setPrefix(sourcePath);
+            //设置分页的页容量
+            listObjectsRequest.setMaxKeys(200);
+            do {
+                //获取下一页的起始点，它的下一项
+                listObjectsRequest.setMarker(nextMaker);
+                ObjectListing objectListing = obsClient.listObjects(listObjectsRequest);
+                List<String> collect = objectListing.getObjects().parallelStream()
+                        .map(ObsObject::getObjectKey).collect(Collectors.toList());
+                if (!CollectionUtils.isEmpty(collect)) {
+                    keyList.addAll(collect);
+                }
+                nextMaker = objectListing.getNextMarker();
+                //全部执行完后，为false
+                flag = objectListing.isTruncated();
+            } while (flag);
+        } catch (Exception e) {
+            log.error("获取文件列表失败，path:" + sourcePath, e);
+        }
+        return keyList;
+    }
+
+    @Override
+    public void copyFileBetweenBucket(String sourceBucketName, String sourcePath, String targetBucketName, String targetPath) {
+        try {
+            List<String> files = listRemoteFiles(sourceBucketName, sourcePath);
+            if (ObjectUtils.isEmpty(files)) {
+                return;
+            }
+            files.stream().forEach(file -> {
+                obsClient.copyObject(sourceBucketName, file, targetBucketName, file.replace(sourcePath, targetPath));
+            });
+        } catch (Exception e) {
+            log.error("列举文件目录失败,key:" + sourcePath, e);
+        }
+    }
+
+    @Override
+    public void copyFilesBetweenBucket(String sourceBucketName, String targetBucketName, Map<String, String> pathMap) {
+        if (ObjectUtils.isEmpty(pathMap)) {
+            return;
+        }
+        try {
+            for (Map.Entry<String, String> entry : pathMap.entrySet()) {
+                copyFileBetweenBucket(sourceBucketName, entry.getKey(), targetBucketName, entry.getValue());
+            }
+        } catch (Exception e) {
+            log.error(String.format("批量复制文件失败, sourceBucketName:%s, targetBucketName:%s", sourceBucketName, targetBucketName), e);
+        }
+    }
+
+    @Override
+    public String getFileContent(String bucketName, String remoteFilePath) {
+        ObsObject ossObject = obsClient.getObject(bucketName, remoteFilePath);
+        try (InputStream objectContent = ossObject.getObjectContent()){
+            StringBuilder contentJson = new StringBuilder();
+            try (BufferedReader reader = new BufferedReader(new InputStreamReader(objectContent))) {
+                while (true) {
+                    String line = reader.readLine();
+                    if (line == null) break;
+                    contentJson.append(line);
+                }
+            } catch (IOException e) {
+                throw e;
+            }
+            return contentJson.toString();
+        } catch (Exception e) {
+            log.error("获取文件内容失败：key:"+remoteFilePath, e);
+        }
+        return null;
+    }
+
+    @Override
+    public boolean fileExist(String bucket, String objectName) {
+        try {
+            return obsClient.doesObjectExist(bucket, objectName);
+        } catch (Exception e) {
+            log.error("判断文件是否存在失败,key:"+objectName, e);
+        }
+        return false;
+    }
+
+    @Override
+    public void downloadFile(String bucket, String remoteFilePath, String localPath) {
+        try {
+            File localFile = new File(localPath);
+            if (!localFile.getParentFile().exists()) {
+                localFile.getParentFile().mkdirs();
+            }
+            if(localFile.isDirectory()){
+                String fileName = remoteFilePath.substring(remoteFilePath.lastIndexOf("/")+1);
+                log.info("未配置文件名，使用默认文件名：{}",fileName);
+                localPath = localPath.concat(File.separator).concat(fileName);
+            }
+            DownloadFileRequest request = new DownloadFileRequest(bucket, remoteFilePath);
+            request.setDownloadFile(localPath);
+            // 默认5个任务并发下载
+            request.setTaskNum(5);
+            // 启动断点续传
+            request.setEnableCheckpoint(true);
+            obsClient.downloadFile(request);
+        } catch (Throwable throwable) {
+            log.error("文件下载失败,key:"+remoteFilePath, throwable);
+        }
+    }
+
+    @Override
+    public URL getPresignedUrl(String bucket, String url) {
+//        java.util.Date expiration = new java.util.Date();
+//        long expTimeMillis = expiration.getTime();
+//        expTimeMillis += 1000 * 60 * 60 * 8;
+//        expiration.setTime(expTimeMillis);
+//        GeneratePresignedUrlRequest generatePresignedUrlRequest = new GeneratePresignedUrlRequest(bucket, url);
+//        generatePresignedUrlRequest.setMethod(HttpMethod.PUT);
+//        generatePresignedUrlRequest.setExpiration(expiration);
+//        return ossClient.generatePresignedUrl(generatePresignedUrlRequest);
+        return null;
+    }
+
+    @Override
+    public long getSubFileNums(String bucket, String url) {
+        long totalSubFileNum = 0;
+        try {
+            boolean flag = true;
+            String nextMaker = null;
+            ListObjectsRequest listObjectsRequest = new ListObjectsRequest(bucket);
+            //指定下一级文件
+            listObjectsRequest.setPrefix(url);
+            //设置分页的页容量
+            listObjectsRequest.setMaxKeys(200);
+            do {
+                //获取下一页的起始点，它的下一项
+                listObjectsRequest.setMarker(nextMaker);
+                ObjectListing objectListing = obsClient.listObjects(listObjectsRequest);
+                List<String> collect = objectListing.getObjects().parallelStream()
+                        .map(ObsObject::getObjectKey).collect(Collectors.toList());
+                if (!CollectionUtils.isEmpty(collect)) {
+                    totalSubFileNum = totalSubFileNum + collect.size();
+                }
+                nextMaker = objectListing.getNextMarker();
+                //全部执行完后，为false
+                flag = objectListing.isTruncated();
+            } while (flag);
+        } catch (Exception e) {
+            log.error("获取文件数量失败，path:" + url, e);
+        }
+        return totalSubFileNum;
+    }
+
+    @Override
+    public Boolean checkStore(String bucket,String filePath){
+        ObjectMetadata objectMetadata =  obsClient.getObjectMetadata(bucket, filePath);
+        return this.isRestoreCompleted(objectMetadata);
+    }
+
+    private boolean isRestoreCompleted(ObjectMetadata objectMetadata){
+        String restoreStr = (String)objectMetadata.getValue("x-obs-restore");
+        boolean restore = Boolean.valueOf(restoreStr);
+        if(!restore){
+            return true;
+        }
+        return false;
+    }
+
+    @Override
+    public void restoreFolder(String bucket,String folderName){
+        ObjectMetadata objectMetadata ;
+
+        List<String> objectList = this.listRemoteFiles(bucket, folderName);
+        if(CollUtil.isEmpty(objectList)){
+            return;
+        }
+        for (String objectName : objectList) {
+            objectMetadata =  obsClient.getObjectMetadata(bucket, objectName);
+            // 校验Object是否为归档类型Object。
+            StorageClassEnum storageClass = objectMetadata.getObjectStorageClass();
+            if (storageClass == StorageClassEnum.COLD) {
+                // 解冻Object。
+                RestoreObjectRequest request = new RestoreObjectRequest(bucket, objectName, 1);
+                obsClient.restoreObject(request);
+            }
+        }
+    }
+
+    @Override
+    public Integer getRestoreFolderProcess(String bucket,String folderName){
+        ObjectMetadata objectMetadata ;
+
+        List<String> objectList = this.listRemoteFiles(bucket, folderName);
+        if(CollUtil.isEmpty(objectList)){
+            return 100;
+        }
+        List<String> restoreFileList = new ArrayList<>();
+        for (String objectName : objectList) {
+            objectMetadata = obsClient.getObjectMetadata(bucket, objectName);
+            if(this.isRestoreCompleted(objectMetadata)){
+                restoreFileList.add(objectName);
+            }
+        }
+        if(objectList.size() <= restoreFileList.size() ){
+            return 100;
+        }
+        BigDecimal rite = new BigDecimal(restoreFileList.size()).divide(new BigDecimal(objectList.size()),2,BigDecimal.ROUND_HALF_UP);
+        BigDecimal multiply = rite.multiply(new BigDecimal(100));
+        return multiply.intValue();
+    }
+
+    @Override
+    public void restoreFolder(String bucket, String folderName, Integer priority) {
+
+        List<String> objectList = this.listRemoteFiles(bucket, folderName);
+        if(CollUtil.isEmpty(objectList)){
+            return;
+        }
+        objectList.parallelStream().forEach(objectName -> {
+            this.restoreFile(bucket, objectName, priority);
+        });
+    }
+
+    @Override
+    public void restoreFile(String bucket, String objectName, Integer priority){
+        ObjectMetadata objectMetadata = obsClient.getObjectMetadata(bucket, objectName);
+
+        // 校验Object是否为归档类型Object。
+        StorageClassEnum storageClass = objectMetadata.getObjectStorageClass();
+        if (storageClass == StorageClassEnum.COLD) {
+            // 设置解冻冷归档Object的优先级。
+            // RestoreTier.RESTORE_TIER_EXPEDITED 表示1小时内完成解冻。
+            // RestoreTier.RESTORE_TIER_STANDARD 表示2~5小时内完成解冻。
+            // RestoreTier.RESTORE_TIER_BULK 表示5~12小时内完成解冻。
+            RestoreTierEnum restoreTier = null;
+            switch (priority){
+                case 1 :
+                    restoreTier = RestoreTierEnum.EXPEDITED;
+                    break;
+                case 2 :
+                    restoreTier = RestoreTierEnum.STANDARD;
+                    break;
+                default:
+                    restoreTier = RestoreTierEnum.BULK;
+            }
+            // 配置解冻参数，以设置5小时内解冻完成，解冻状态保持2天为例。
+            // 第一个参数表示保持解冻状态的天数，默认是1天，此参数适用于解冻Archive（归档）与ColdArchive（冷归档）类型Object。
+            // 第二个参数jobParameters表示解冻优先级，只适用于解冻ColdArchive类型Object。
+            //开始解冻
+            // 解冻Object。
+            RestoreObjectRequest request = new RestoreObjectRequest(bucket, objectName, 1);
+            request.setRestoreTier(restoreTier);
+            obsClient.restoreObject(request);
+//            // 等待解冻完成。
+//            do {
+//                try {
+//                    Thread.sleep(1000);
+//                } catch (InterruptedException e) {
+//                    e.printStackTrace();
+//                }
+//                objectMetadata = ossClient.getObjectMetadata(bucket, objectName);
+//            } while (!objectMetadata.isRestoreCompleted());
+        }
+    }
+
+    @Override
+    public Long getSpace(String bucket, String key) {
+
+        Long total = 0L;
+        boolean flag = true;
+        String nextMaker = null;
+        ListObjectsRequest listObjectsRequest = new ListObjectsRequest(bucket);
+        //指定下一级文件
+        listObjectsRequest.setPrefix(key);
+        //设置分页的页容量
+        listObjectsRequest.setMaxKeys(200);
+        do {
+            //获取下一页的起始点，它的下一项
+            listObjectsRequest.setMarker(nextMaker);
+            ObjectListing objectListing = obsClient.listObjects(listObjectsRequest);
+//            Long space = objectListing.getObjectSummaries().parallelStream()
+//                    .mapToLong(S3Object::getMetadat).sum();
+            List<S3Object> objectSummaries = objectListing.getObjectSummaries();
+            for (S3Object objectSummary : objectSummaries) {
+                Long contentLength = objectSummary.getMetadata().getContentLength();
+                total += contentLength;
+            }
+            nextMaker = objectListing.getNextMarker();
+            //全部执行完后，为false
+            flag = objectListing.isTruncated();
+        } while (flag);
+        return total;
+    }
+
+}

4dkankan-utils-fyun-obs/src/main/java/com/fdkankan/fyun/oss/config/ObsConfig.java

@@ -0,0 +1,25 @@
+package com.fdkankan.fyun.oss.config;
+
+import com.fdkankan.fyun.config.FYunFileConfig;
+import com.obs.services.ObsClient;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
+import org.springframework.context.annotation.Bean;
+import org.springframework.stereotype.Component;
+
+@Component
+@ConditionalOnProperty(name = "fyun.type", havingValue = "obs")
+public class ObsConfig {
+
+    @Autowired
+    private FYunFileConfig fYunFileConfig;
+
+    /**
+     * new OSSClient操作完成后，请通过shutdown进行关闭，保证new OSSClient和ossClient.shutdown成对使用。
+     * @return
+     */
+    @Bean
+    public ObsClient ossClient(){
+        return new ObsClient(fYunFileConfig.getKey(), fYunFileConfig.getSecret(), fYunFileConfig.getEndPoint());
+    }
+}

4dkankan-utils-fyun-s3/src/main/java/com/fdkankan/fyun/s3/S3FileService.java

@@ -261,10 +261,8 @@ public class S3FileService extends AbstractFYunFileService {
 
     @Override
     public String getFileContent(String bucketName, String remoteFilePath){
-        try {
-			GetObjectRequest request  = new GetObjectRequest(bucketName,remoteFilePath);
-			S3Object object = s3.getObject(request);
-			S3ObjectInputStream inputStream = object.getObjectContent();
+        try (S3Object object = s3.getObject(bucketName,remoteFilePath);
+             S3ObjectInputStream inputStream = object.getObjectContent()){
 			StringBuilder content = new StringBuilder();
 			try(BufferedReader reader = new BufferedReader(new InputStreamReader(inputStream))){
 				while (true) {
@@ -275,7 +273,6 @@ public class S3FileService extends AbstractFYunFileService {
 			} catch (IOException e) {
 				log.error("读取aws文件流失败", e);
 			}
-            object.close();
 			return content.toString();
         } catch (Exception e) {
             log.error("获取文件内容失败：{}", remoteFilePath);

pom.xml

@@ -22,6 +22,7 @@
         <module>4dkankan-utils-fyun-oss</module>
         <module>4dkankan-utils-fyun-s3</module>
         <module>4dkankan-utils-fyun-cos</module>
+        <module>4dkankan-utils-fyun-obs</module>
         <module>4dkankan-utils-fyun-local</module>
         <module>4dkankan-utils-model</module>
         <module>4dkankan-utils-wechat</module>