|
@@ -0,0 +1,302 @@
|
|
|
+package com.fdkankan.fyun.s3;
|
|
|
+
|
|
|
+import cn.hutool.core.collection.CollUtil;
|
|
|
+import com.amazonaws.services.s3.AmazonS3;
|
|
|
+import com.amazonaws.services.s3.model.*;
|
|
|
+import com.fdkankan.common.util.CreateObjUtil;
|
|
|
+import com.fdkankan.fyun.constant.FYunConstants;
|
|
|
+import com.fdkankan.fyun.face.AbstractFYunFileService;
|
|
|
+import org.apache.commons.lang3.StringUtils;
|
|
|
+import org.slf4j.Logger;
|
|
|
+import org.slf4j.LoggerFactory;
|
|
|
+import org.springframework.beans.factory.annotation.Autowired;
|
|
|
+import org.springframework.stereotype.Component;
|
|
|
+import org.springframework.util.ObjectUtils;
|
|
|
+
|
|
|
+import java.io.*;
|
|
|
+import java.util.ArrayList;
|
|
|
+import java.util.List;
|
|
|
+import java.util.Map;
|
|
|
+import java.util.stream.Collectors;
|
|
|
+
|
|
|
+@Component
|
|
|
+public class S3FileService extends AbstractFYunFileService {
|
|
|
+
|
|
|
+ private Logger log = LoggerFactory.getLogger(this.getClass().getName());
|
|
|
+
|
|
|
+ @Autowired
|
|
|
+ private AmazonS3 s3;
|
|
|
+
|
|
|
+ @Override
|
|
|
+ public void uploadFile(String bucket, byte[] data, String remoteFilePath) throws Exception {
|
|
|
+ try {
|
|
|
+ ObjectMetadata metadata = new ObjectMetadata();
|
|
|
+ PutObjectRequest request = new PutObjectRequest(bucket, remoteFilePath, new ByteArrayInputStream(data), metadata);
|
|
|
+ request.withCannedAcl(CannedAccessControlList.PublicRead);
|
|
|
+ s3.putObject(request);
|
|
|
+ } catch (Exception e) {
|
|
|
+ log.error("s3上传文件失败", e);
|
|
|
+ } finally {
|
|
|
+ if (s3 != null) {
|
|
|
+ s3.shutdown();
|
|
|
+ }
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+ @Override
|
|
|
+ public void uploadFile(String bucket, String filePath, String remoteFilePath) throws Exception {
|
|
|
+ uploadFile(bucket, filePath, remoteFilePath,true);
|
|
|
+ }
|
|
|
+
|
|
|
+ private void uploadFile(String bucket, String filePath, String remoteFilePath, Boolean shutdown) throws Exception {
|
|
|
+ try {
|
|
|
+ File file = new File(filePath);
|
|
|
+ if (!file.exists()) {
|
|
|
+ log.info("要上传s3的文件不存在");
|
|
|
+ return;
|
|
|
+ }
|
|
|
+
|
|
|
+ // 设置文件并设置公读
|
|
|
+ ObjectMetadata metadata = new ObjectMetadata();
|
|
|
+ if (filePath.contains(".jpg")) {
|
|
|
+ metadata.setContentType("image/jpeg");
|
|
|
+ }
|
|
|
+ if (filePath.contains(".png")) {
|
|
|
+ metadata.setContentType("image/png");
|
|
|
+ }
|
|
|
+ PutObjectRequest request = new PutObjectRequest(bucket, remoteFilePath, file);
|
|
|
+ request.withCannedAcl(CannedAccessControlList.PublicRead);
|
|
|
+ request.withMetadata(metadata);
|
|
|
+
|
|
|
+ // 上传文件
|
|
|
+ PutObjectResult putObjectResult = s3.putObject(request);
|
|
|
+ if (StringUtils.isNotEmpty(putObjectResult.getETag())) {
|
|
|
+ log.info("s3上传文件成功:" + remoteFilePath);
|
|
|
+ }
|
|
|
+ } catch (Exception e) {
|
|
|
+ throw e;
|
|
|
+ } finally {
|
|
|
+ if ((ObjectUtils.isEmpty(shutdown) || shutdown) && !ObjectUtils.isEmpty(s3)) {
|
|
|
+ s3.shutdown();
|
|
|
+ }
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+ @Override
|
|
|
+ public void uploadFileByCommand(String bucket, String filePath, String remoteFilePath) throws Exception {
|
|
|
+ String ossPath = bucket + "/" + remoteFilePath;
|
|
|
+ try {
|
|
|
+ String command = String.format(FYunConstants.UPLOAD_SH, ossPath, filePath);
|
|
|
+ log.info("开始上传文件, ossPath:{}, srcPath:{}", ossPath, filePath);
|
|
|
+ CreateObjUtil.callshell(command);
|
|
|
+ } catch (Exception e) {
|
|
|
+ log.error("上传文件失败, ossPath:{}, srcPath:{}", ossPath, filePath);
|
|
|
+ e.printStackTrace();
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+ @Override
|
|
|
+ public void deleteFile(String bucket, String remoteFilePath) throws IOException {
|
|
|
+ if (remoteFilePath.startsWith("/")) {
|
|
|
+ remoteFilePath = remoteFilePath.substring(1);
|
|
|
+ }
|
|
|
+ try {
|
|
|
+ s3.deleteObject(bucket, remoteFilePath);
|
|
|
+ } catch (Exception e) {
|
|
|
+ log.error("s3删除文件失败,key=" + remoteFilePath, e);
|
|
|
+ } finally {
|
|
|
+ if (s3 != null) {
|
|
|
+ s3.shutdown();
|
|
|
+ }
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+ @Override
|
|
|
+ public void deleteFolder(String bucket, String remoteFolderPath) throws Exception {
|
|
|
+ try {
|
|
|
+ int maxKeys = 200;
|
|
|
+ String nextMaker = null;
|
|
|
+
|
|
|
+ ListObjectsRequest listObjectsRequest = new ListObjectsRequest();
|
|
|
+ listObjectsRequest.setBucketName(bucket);
|
|
|
+ listObjectsRequest.setPrefix(remoteFolderPath);
|
|
|
+ listObjectsRequest.setMaxKeys(maxKeys);
|
|
|
+
|
|
|
+ com.amazonaws.services.s3.model.ObjectListing objectListing;
|
|
|
+ do {
|
|
|
+ listObjectsRequest.setMarker(nextMaker);
|
|
|
+ objectListing = s3.listObjects(listObjectsRequest);
|
|
|
+ List<S3ObjectSummary> objectSummaries = objectListing.getObjectSummaries();
|
|
|
+ List<DeleteObjectsRequest.KeyVersion> keys = objectSummaries.stream().map(summary -> new DeleteObjectsRequest.KeyVersion(summary.getKey())).collect(Collectors.toList());
|
|
|
+
|
|
|
+ DeleteObjectsRequest multiObjectDeleteRequest = new DeleteObjectsRequest(bucket)
|
|
|
+ .withKeys(keys).withQuiet(false);
|
|
|
+ DeleteObjectsResult delObjRes = s3.deleteObjects(multiObjectDeleteRequest);
|
|
|
+ int successfulDeletes = delObjRes.getDeletedObjects().size();
|
|
|
+ log.info("删除aws文件成功,删除文件数;{}", successfulDeletes);
|
|
|
+ nextMaker = objectListing.getNextMarker();
|
|
|
+ } while (objectListing.isTruncated());
|
|
|
+ } catch (Exception e) {
|
|
|
+ log.error("删除was文件失败,path=" + remoteFolderPath, e);
|
|
|
+ } finally {
|
|
|
+ if (s3 != null) {
|
|
|
+ s3.shutdown();
|
|
|
+ }
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+ @Override
|
|
|
+ public void uploadMulFiles(String bucket, Map<String, String> filepaths) throws Exception {
|
|
|
+ try {
|
|
|
+ for (Map.Entry<String, String> entry : filepaths.entrySet()) {
|
|
|
+ uploadFile(bucket, entry.getKey(), entry.getValue(), false);
|
|
|
+ }
|
|
|
+ } catch (Exception e) {
|
|
|
+ log.error("OSS批量上传文件失败!");
|
|
|
+ } finally {
|
|
|
+ if (!ObjectUtils.isEmpty(s3)) {
|
|
|
+ s3.shutdown();
|
|
|
+ }
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+ @Override
|
|
|
+ public List<String> listRemoteFiles(String bucket, String sourcePath) throws Exception {
|
|
|
+ return listRemoteFiles(bucket, sourcePath, true);
|
|
|
+ }
|
|
|
+
|
|
|
+ private List<String> listRemoteFiles(String bucket, String sourcePath, Boolean shutdown) throws Exception {
|
|
|
+ List<String> keyList = new ArrayList<>();
|
|
|
+ try {
|
|
|
+ boolean flag = true;
|
|
|
+ String nextMaker = null;
|
|
|
+ ListObjectsRequest listObjectsRequest = new ListObjectsRequest();
|
|
|
+ listObjectsRequest.setBucketName(bucket);
|
|
|
+ listObjectsRequest.setPrefix(sourcePath);
|
|
|
+ listObjectsRequest.setMaxKeys(200);
|
|
|
+
|
|
|
+ do {
|
|
|
+ listObjectsRequest.setMarker(nextMaker);
|
|
|
+ ObjectListing objectListing = s3.listObjects(listObjectsRequest);
|
|
|
+ List<S3ObjectSummary> objectSummaries = objectListing.getObjectSummaries();
|
|
|
+ List<String> collect = objectSummaries.stream().map(S3ObjectSummary::getKey).collect(Collectors.toList());
|
|
|
+ if (CollUtil.isNotEmpty(collect)) {
|
|
|
+ keyList.addAll(collect);
|
|
|
+ }
|
|
|
+ nextMaker = objectListing.getNextMarker();
|
|
|
+ flag = objectListing.isTruncated();
|
|
|
+ } while (flag);
|
|
|
+ } catch (Exception e) {
|
|
|
+ log.error("获取文件列表失败,path=" + sourcePath, e);
|
|
|
+ e.printStackTrace();
|
|
|
+ } finally {
|
|
|
+ if ((ObjectUtils.isEmpty(shutdown) || shutdown) && !ObjectUtils.isEmpty(s3)) {
|
|
|
+ s3.shutdown();
|
|
|
+ }
|
|
|
+ }
|
|
|
+ return keyList;
|
|
|
+ }
|
|
|
+
|
|
|
+ @Override
|
|
|
+ public void copyFileBetweenFyun(String sourceBucketName, String sourcePath, String targetBucketName, String targetPath) throws Exception {
|
|
|
+ copyFileBetweenFyun(sourceBucketName, sourcePath, targetBucketName, targetPath, true);
|
|
|
+ }
|
|
|
+
|
|
|
+ private void copyFileBetweenFyun(String sourceBucketName, String sourcePath, String targetBucketName, String targetPath, Boolean shutdown) throws Exception {
|
|
|
+ try {
|
|
|
+ List<String> files = listRemoteFiles(sourceBucketName, sourcePath, false);
|
|
|
+ if (ObjectUtils.isEmpty(files)) {
|
|
|
+ log.error("源文件夹为空:{}", sourcePath);
|
|
|
+ }
|
|
|
+ files.stream().forEach(file -> {
|
|
|
+ CopyObjectRequest request = new CopyObjectRequest(sourceBucketName, file, targetBucketName, file.replace(sourcePath, targetPath));
|
|
|
+ request.withCannedAccessControlList(CannedAccessControlList.PublicRead);
|
|
|
+ s3.copyObject(request);
|
|
|
+ });
|
|
|
+ } catch (Exception e) {
|
|
|
+ log.error("列举文件目录失败,key=" + sourcePath);
|
|
|
+ } finally {
|
|
|
+ if ((ObjectUtils.isEmpty(shutdown) || shutdown) && !ObjectUtils.isEmpty(s3)) {
|
|
|
+ s3.shutdown();
|
|
|
+ }
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+ @Override
|
|
|
+ public void copyFilesBetweenFyun(String sourceBucketName, String targetBucketName, Map<String, String> pathMap) throws Exception {
|
|
|
+ if (ObjectUtils.isEmpty(pathMap)) {
|
|
|
+ return;
|
|
|
+ }
|
|
|
+ try {
|
|
|
+ for (Map.Entry<String, String> entry : pathMap.entrySet()) {
|
|
|
+ copyFileBetweenFyun(sourceBucketName, entry.getKey(), targetBucketName, entry.getValue(), false);
|
|
|
+ }
|
|
|
+ } catch (Exception e) {
|
|
|
+ log.error("批量复制文件失败!");
|
|
|
+ } finally {
|
|
|
+ if (!ObjectUtils.isEmpty(s3)) {
|
|
|
+ s3.shutdown();
|
|
|
+ }
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+ @Override
|
|
|
+ public String getFileContent(String bucketName, String remoteFilePath) throws Exception {
|
|
|
+ try {
|
|
|
+ GetObjectRequest request = new GetObjectRequest(bucketName,remoteFilePath);
|
|
|
+ S3Object object = s3.getObject(request);
|
|
|
+ S3ObjectInputStream inputStream = object.getObjectContent();
|
|
|
+ StringBuilder content = new StringBuilder();
|
|
|
+ try(BufferedReader reader = new BufferedReader(new InputStreamReader(inputStream))){
|
|
|
+ while (true) {
|
|
|
+ String line = reader.readLine();
|
|
|
+ if (line == null) break;
|
|
|
+ content.append(line);
|
|
|
+ }
|
|
|
+ } catch (IOException e) {
|
|
|
+ log.error("读取aws文件流失败", e);
|
|
|
+ }
|
|
|
+ return content.toString();
|
|
|
+ } catch (Exception e) {
|
|
|
+ log.error("获取文件内容失败:{}", remoteFilePath);
|
|
|
+ return null;
|
|
|
+ } finally {
|
|
|
+ if (!ObjectUtils.isEmpty(s3)) {
|
|
|
+ s3.shutdown();
|
|
|
+ }
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+ @Override
|
|
|
+ public boolean fileExist(String bucket, String objectName) throws Exception {
|
|
|
+ try {
|
|
|
+ return s3.doesObjectExist(bucket, objectName);
|
|
|
+ } catch (Exception e) {
|
|
|
+ log.error("判断文件是否存在失败:{}", objectName);
|
|
|
+ return false;
|
|
|
+ } finally {
|
|
|
+ if (!ObjectUtils.isEmpty(s3)) {
|
|
|
+ s3.shutdown();
|
|
|
+ }
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+ @Override
|
|
|
+ public void downloadFile(String bucket, String remoteFilePath, String localPath) throws Exception {
|
|
|
+ try {
|
|
|
+ File localFile = new File(localPath);
|
|
|
+ if (!localFile.getParentFile().exists()) {
|
|
|
+ localFile.getParentFile().mkdirs();
|
|
|
+ }
|
|
|
+ GetObjectRequest request = new GetObjectRequest(bucket, remoteFilePath);
|
|
|
+ s3.getObject(request,new File(localPath));
|
|
|
+ } catch (Throwable throwable) {
|
|
|
+ log.error("文件下载失败:{}", remoteFilePath);
|
|
|
+ throwable.printStackTrace();
|
|
|
+ } finally {
|
|
|
+ if (!ObjectUtils.isEmpty(s3)) {
|
|
|
+ s3.shutdown();
|
|
|
+ }
|
|
|
+ }
|
|
|
+ }
|
|
|
+}
|