UploadToOssUtil.java 23 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855
  1. package com.fdkankan.fyun.oss;
  2. import cn.hutool.core.collection.CollUtil;
  3. import cn.hutool.core.io.FileUtil;
  4. import com.alibaba.fastjson.JSON;
  5. import com.aliyun.oss.OSSClient;
  6. import com.aliyun.oss.model.*;
  7. import com.amazonaws.HttpMethod;
  8. import com.amazonaws.auth.AWSStaticCredentialsProvider;
  9. import com.amazonaws.auth.BasicAWSCredentials;
  10. import com.amazonaws.regions.Regions;
  11. import com.amazonaws.services.s3.AmazonS3;
  12. import com.amazonaws.services.s3.AmazonS3ClientBuilder;
  13. import com.amazonaws.services.s3.model.CannedAccessControlList;
  14. import com.amazonaws.services.s3.model.GeneratePresignedUrlRequest;
  15. import com.amazonaws.services.s3.model.GetObjectRequest;
  16. import com.amazonaws.services.s3.model.PutObjectRequest;
  17. import com.amazonaws.services.s3.model.S3Object;
  18. import com.amazonaws.services.s3.model.S3ObjectInputStream;
  19. import com.amazonaws.services.s3.model.S3ObjectSummary;
  20. import com.fdkankan.fyun.constant.StorageType;
  21. import com.qiniu.common.Zone;
  22. import com.qiniu.storage.Configuration;
  23. import com.qiniu.storage.UploadManager;
  24. import java.util.stream.Collectors;
  25. import lombok.extern.slf4j.Slf4j;
  26. import org.apache.commons.fileupload.FileItem;
  27. import org.apache.commons.fileupload.FileItemFactory;
  28. import org.apache.commons.fileupload.disk.DiskFileItemFactory;
  29. import org.apache.commons.io.FileUtils;
  30. import org.apache.commons.lang3.StringUtils;
  31. import org.springframework.beans.factory.annotation.Value;
  32. import org.springframework.stereotype.Component;
  33. import org.springframework.web.multipart.MultipartFile;
  34. import org.springframework.web.multipart.commons.CommonsMultipartFile;
  35. import java.io.*;
  36. import java.net.FileNameMap;
  37. import java.net.URL;
  38. import java.net.URLConnection;
  39. import java.util.ArrayList;
  40. import java.util.HashMap;
  41. import java.util.List;
  42. import java.util.Map;
  43. @Slf4j
  44. @Component
  45. public class UploadToOssUtil {
  46. Zone zone = Zone.autoZone();
  47. Configuration config = new Configuration(zone);
  48. UploadManager uploadManager = new UploadManager(config);
  49. @Value("${oss.point:http://oss-cn-shenzhen-internal.aliyuncs.com}")
  50. private String point;
  51. @Value("${oss.key:LTAIUrvuHqj8pvry}")
  52. private String key;
  53. @Value("${oss.secrey:JLOVl0k8Ke0aaM8nLMMiUAZ3EiiqI4}")
  54. private String secrey;
  55. @Value("${oss.bucket:4dkankan}")
  56. private String bucket;
  57. @Value("${oss.sdk:4dscene}")
  58. private String bucketSdk;
  59. @Value("${upload.type:oss}")
  60. private String type;
  61. @Value("${aws.s3key:AKIAWCV5QFZ3ZNELKYUY}")
  62. private String s3key;
  63. @Value("${aws.s3secrey:epS5ghyR4LJ7rxk/qJO9ZYh6m9Oz6g5haKDu4yws}")
  64. private String s3secrey;
  65. @Value("${aws.s3bucket:4dkankan}")
  66. private String s3bucket;
  67. @Value("${local.path:/home/4dkankan}")
  68. private String localPath;
  69. //上传的数据是byte[],key是上传后的文件名
  70. public void upload(byte[] data,String key1) throws IOException{
  71. log.info("开始上传文件 源路径:{},目标路径:{},type:{}" , new String(data, "UTF-8"),key1,type);
  72. StorageType storageType = StorageType.get(type);
  73. switch (storageType){
  74. case OSS:
  75. uploadOss(data,key1);
  76. break;
  77. case AWS:
  78. uploadAws(data,key1);
  79. break;
  80. case LOCAL:
  81. uploadLocal(data,key1);
  82. break;
  83. }
  84. }
  85. public void upload(String filePath, String key1) {
  86. log.info("开始上传文件 源路径:{},目标路径:{},type:{}" , filePath,key1,type);
  87. StorageType storageType = StorageType.get(type);
  88. switch (storageType){
  89. case OSS:
  90. uploadOss(filePath,key1);
  91. break;
  92. case AWS:
  93. uploadAws(filePath,key1);
  94. break;
  95. case LOCAL:
  96. uploadLocal(filePath,key1);
  97. break;
  98. }
  99. }
  100. public void uploadSdk(String filePath, String key1) {
  101. log.info("开始上传文件 源路径:{},目标路径:{},type:{}" , filePath,key1,type);
  102. switch (type){
  103. case "oss":uploadSdkOss(filePath,key1); break;
  104. case "aws": uploadAws(filePath,key1); break;
  105. case "local":uploadLocal(filePath,key1); break;
  106. }
  107. }
  108. public void upload2(String filePath, String key1) {
  109. log.info("开始上传文件 源路径:{},目标路径:{},type:{}" , filePath,key1,type);
  110. switch (type){
  111. case "oss":upload2Oss(filePath,key1); break;
  112. case "aws": uploadAws(filePath,key1); break;
  113. case "local":uploadLocal(filePath,key1); break;
  114. }
  115. }
  116. public void delete(String key1) throws IOException{
  117. switch (type){
  118. case "oss":deleteOss(key1); break;
  119. case "aws": deleteS3Object(key1); break;
  120. case "local":FileUtil.del(key1); break;
  121. }
  122. }
  123. public int deleteFile(String prefix){
  124. switch (type){
  125. case "oss":deleteOssFile(prefix); break;
  126. case "aws": deleteS3Object(prefix); break;
  127. case "local":FileUtil.del(prefix); break;
  128. }
  129. return 1;
  130. }
  131. public void deleteOss(String key1){
  132. OSSClient ossClient = new OSSClient(point, key, secrey);
  133. try {
  134. ossClient.deleteObject(bucket, key1);
  135. } catch (Exception e) {
  136. e.printStackTrace();
  137. }
  138. }
  139. public void deleteOssFile(String prefix){
  140. OSSClient ossClient = new OSSClient(point, key, secrey);
  141. ObjectListing objectListing = ossClient.listObjects(bucket, prefix);
  142. List<OSSObjectSummary> sums = objectListing.getObjectSummaries();
  143. try {
  144. for (OSSObjectSummary s : sums) {
  145. delete(s.getKey());
  146. }
  147. } catch (IOException e) {
  148. e.printStackTrace();
  149. }
  150. }
  151. public void uploadOss(byte[] data,String key1){
  152. OSSClient ossClient = new OSSClient(point, key, secrey);
  153. try {
  154. ossClient.putObject(bucket, key1, new ByteArrayInputStream(data));
  155. } catch (Exception e) {
  156. log.error(e.toString()+key1);
  157. }
  158. }
  159. public void uploadAws(byte[] data,String key1){
  160. }
  161. public void uploadLocal(byte[] data,String key1){
  162. InputStream in = new ByteArrayInputStream(data);
  163. File file = new File(key1);
  164. String path = key1.substring(0, key1.lastIndexOf("/"));
  165. if (!file.exists()) {
  166. new File(path).mkdir();
  167. }
  168. FileOutputStream fos = null;
  169. try {
  170. fos = new FileOutputStream(file);
  171. int len = 0;
  172. byte[] buf = new byte[1024];
  173. while ((len = in.read(buf)) != -1) {
  174. fos.write(buf, 0, len);
  175. }
  176. fos.flush();
  177. } catch (Exception e) {
  178. e.printStackTrace();
  179. } finally {
  180. if (null != fos) {
  181. try {
  182. fos.close();
  183. } catch (IOException e) {
  184. e.printStackTrace();
  185. }
  186. }
  187. }
  188. }
  189. public void uploadOss(String filePath, String key1){
  190. OSSClient ossClient = new OSSClient(point, key, secrey);
  191. try {
  192. File file = new File(filePath);
  193. if (!file.exists()) {
  194. log.error("要上传的文件不存在:" + filePath);
  195. return;
  196. }
  197. ObjectMetadata metadata = new ObjectMetadata();
  198. if(filePath.contains(".jpg")){
  199. metadata.setContentType("image/jpeg");
  200. }
  201. ossClient.putObject(bucket, key1, new File(filePath), metadata);
  202. } catch (Exception e) {
  203. log.error(e.toString() + filePath);
  204. } finally {
  205. ossClient.shutdown();
  206. }
  207. }
  208. public void uploadAws(String filePath, String key1){
  209. try{
  210. uploadS3File(filePath, key1);
  211. }catch (Exception e){
  212. e.printStackTrace();
  213. }
  214. }
  215. public void uploadLocal(String filePath, String key1){
  216. try {
  217. File srcFile = new File(filePath);
  218. File file = new File(localPath + key1);
  219. FileUtils.copyFile(srcFile,file);
  220. }catch (Exception e){
  221. e.printStackTrace();
  222. }
  223. }
  224. public void uploadSdkOss(String filePath, String key1){
  225. OSSClient ossClient = new OSSClient(point, key, secrey);
  226. try {
  227. File file = new File(filePath);
  228. if (!file.exists()) {
  229. log.error("要上传的文件不存在:" + filePath);
  230. return;
  231. }
  232. ObjectMetadata metadata = new ObjectMetadata();
  233. if(filePath.contains(".jpg")){
  234. metadata.setContentType("image/jpeg");
  235. }
  236. ossClient.putObject(bucketSdk, key1, new File(filePath), metadata);
  237. } catch (Exception e) {
  238. log.error(e.toString() + filePath);
  239. }
  240. }
  241. public void upload2Oss(String filePath, String key1){
  242. OSSClient ossClient = new OSSClient(point, key, secrey);
  243. try {
  244. ObjectMetadata metadata = new ObjectMetadata();
  245. if(filePath.contains(".jpg")){
  246. metadata.setContentType("image/jpeg");
  247. }
  248. if(filePath.contains(".mp4")){
  249. metadata.setContentType("video/mp4");
  250. }
  251. if(filePath.contains(".mp3")){
  252. metadata.setContentType("audio/mp3");
  253. }
  254. ossClient.putObject(bucket, key1, new File(filePath), metadata);
  255. } catch (Exception e) {
  256. log.error(e.toString() + filePath);
  257. }
  258. }
  259. //上传的数据是文件夹,参数是文件夹路径,key是上传后的文件名
  260. public void uploadMulFiles(Map<String, String> filepaths) {
  261. if (filepaths == null) {
  262. return;
  263. }
  264. Long start = System.currentTimeMillis();
  265. log.info("开始批量上传文件:");
  266. if (filepaths.size() > 50) {
  267. filepaths.entrySet().parallelStream().forEach(entry->{
  268. upload2(entry.getKey(), entry.getValue());
  269. });
  270. } else {
  271. filepaths.entrySet().parallelStream().forEach(entry->{
  272. upload(entry.getKey(), entry.getValue());
  273. });
  274. }
  275. log.info("批量上传文件结束,用时:{}" ,(System.currentTimeMillis() - start));
  276. }
  277. public Map<String, String> getUploadS3Url(List<String> urls){
  278. if(urls == null || urls.size() <= 0){
  279. return null;
  280. }
  281. BasicAWSCredentials awsCred = new BasicAWSCredentials(s3key, s3secrey);
  282. AmazonS3 s3Client = AmazonS3ClientBuilder.standard()
  283. .withCredentials(new AWSStaticCredentialsProvider(awsCred))
  284. .withRegion(Regions.EU_WEST_2)
  285. .build();
  286. // Set the pre-signed URL to expire after one hour.
  287. java.util.Date expiration = new java.util.Date();
  288. long expTimeMillis = expiration.getTime();
  289. expTimeMillis += 1000 * 60 * 60 * 8;
  290. expiration.setTime(expTimeMillis);
  291. //生成预签名URL
  292. log.info("生成预签名URL");
  293. GeneratePresignedUrlRequest generatePresignedUrlRequest = null;
  294. URL url = null;
  295. Map<String, String> map = new HashMap();
  296. for(String path : urls){
  297. // if(path.contains(".jpg") || path.contains("png")){
  298. // generatePresignedUrlRequest = new GeneratePresignedUrlRequest(s3bucket, path)
  299. // .withMethod(HttpMethod.PUT)
  300. // .withExpiration(expiration)
  301. // .withContentType("image/jpeg");
  302. // }else {
  303. generatePresignedUrlRequest = new GeneratePresignedUrlRequest(s3bucket, path)
  304. .withMethod(HttpMethod.PUT)
  305. .withExpiration(expiration);
  306. // }
  307. url = s3Client.generatePresignedUrl(generatePresignedUrlRequest);
  308. map.put(path, url.toString());
  309. }
  310. return map;
  311. }
  312. public String upload5(String filePath, String key1) {
  313. OSSClient ossClient = new OSSClient(point, key, secrey);
  314. PutObjectResult result = null;
  315. try {
  316. File file = new File(filePath);
  317. if (!file.exists()) {
  318. log.error("要上传的文件不存在:" + filePath);
  319. }
  320. result = ossClient.putObject(bucket, key1, new File(filePath));
  321. } catch (Exception e) {
  322. log.error(e.toString() + filePath);
  323. }
  324. log.info(" getETag : " + result.getETag());
  325. log.info("1 : " + result.toString());
  326. log.info("2 : " + result.getRequestId());
  327. log.info("3 : " + result.getClientCRC());
  328. log.info("4 : " + result.getResponse());
  329. log.info("5 : " + result.getServerCRC());
  330. return result.getETag();
  331. }
  332. //海外亚马逊s3
  333. /**
  334. * s3上传文件流
  335. *
  336. * @param file 文件
  337. * @param updatePath 上传路径[ eg: xxx/xxx ]
  338. */
  339. public String updateS3LoadFile(MultipartFile file, String updatePath) {
  340. if (isEmpty(file)) {
  341. return null;
  342. }
  343. /**
  344. * 创建s3对象
  345. */
  346. BasicAWSCredentials awsCreds = new BasicAWSCredentials(s3key, s3secrey);
  347. AmazonS3 s3 = AmazonS3ClientBuilder.standard()
  348. .withCredentials(new AWSStaticCredentialsProvider(awsCreds))
  349. .withRegion(Regions.EU_WEST_2)
  350. .build();
  351. try {
  352. // 创建临时文件,程序运行结束,会自动删除
  353. File localFile = File.createTempFile("temp", null);
  354. // 把文件写入内存中
  355. file.transferTo(localFile);
  356. // 指定要上传到服务器上的路径
  357. String key = updatePath;
  358. // 设置文件并设置公读
  359. PutObjectRequest request = new PutObjectRequest(s3bucket, key, localFile);
  360. request.withCannedAcl(CannedAccessControlList.PublicRead);
  361. // 上传文件
  362. com.amazonaws.services.s3.model.PutObjectResult putObjectResult = s3.putObject(request);
  363. if (StringUtils.isNotEmpty(putObjectResult.getETag())) {
  364. System.out.println("success");
  365. return key;
  366. }
  367. return null;
  368. } catch (IOException e) {
  369. }
  370. return null;
  371. }
  372. /**
  373. * s3上传文件
  374. * @param filePath
  375. * @param key1
  376. * @throws IOException
  377. */
  378. private void uploadS3File(String filePath, String key1) throws Exception {
  379. /**
  380. * 创建s3对象
  381. */
  382. BasicAWSCredentials awsCreds = new BasicAWSCredentials(s3key, s3secrey);
  383. AmazonS3 s3 = AmazonS3ClientBuilder.standard()
  384. .withCredentials(new AWSStaticCredentialsProvider(awsCreds))
  385. .withRegion(Regions.EU_WEST_2)
  386. .build();
  387. try{
  388. File file = new File(filePath);
  389. if(!file.exists()){
  390. log.info("要上传s3的文件不存在");
  391. return;
  392. }
  393. // 设置文件并设置公读
  394. com.amazonaws.services.s3.model.ObjectMetadata metadata = new com.amazonaws.services.s3.model.ObjectMetadata();
  395. if(filePath.contains(".jpg")){
  396. metadata.setContentType("image/jpeg");
  397. }
  398. if(filePath.contains(".png")){
  399. metadata.setContentType("image/png");
  400. }
  401. PutObjectRequest request = new PutObjectRequest(s3bucket, key1, file);
  402. request.withCannedAcl(CannedAccessControlList.PublicRead);
  403. request.withMetadata(metadata);
  404. // 上传文件
  405. com.amazonaws.services.s3.model.PutObjectResult putObjectResult = s3.putObject(request);
  406. if (StringUtils.isNotEmpty(putObjectResult.getETag())) {
  407. log.info("s3上传文件成功:" + key1);
  408. }
  409. }catch (Exception e){
  410. throw e;
  411. }finally {
  412. s3.shutdown();
  413. }
  414. }
  415. /**
  416. * 删除单个文件
  417. *
  418. * @param filePath 文件路径[ eg: /head/xxxx.jpg ]
  419. * @return
  420. */
  421. public void deleteS3Object(String filePath) {
  422. /**
  423. * 创建s3对象
  424. */
  425. BasicAWSCredentials awsCreds = new BasicAWSCredentials(s3key, s3secrey);
  426. AmazonS3 s3 = AmazonS3ClientBuilder.standard()
  427. .withCredentials(new AWSStaticCredentialsProvider(awsCreds))
  428. .withRegion(Regions.EU_WEST_2)
  429. .build();
  430. if (filePath.startsWith("/")) {
  431. filePath = filePath.substring(1);
  432. }
  433. try {
  434. s3.deleteObject(s3bucket, filePath);
  435. } catch (Exception e) {
  436. }
  437. }
  438. /**
  439. * 获取文件类型
  440. */
  441. public static String getContentType(String filePath){
  442. FileNameMap fileNameMap = URLConnection.getFileNameMap();
  443. String contentType = fileNameMap.getContentTypeFor(filePath);
  444. System.out.println(contentType);
  445. return contentType;
  446. }
  447. /**
  448. * 检查文件是否为空
  449. *
  450. * @param imageFile
  451. * @return
  452. */
  453. private static boolean isEmpty(MultipartFile imageFile) {
  454. if (imageFile == null || imageFile.getSize() <= 0) {
  455. return true;
  456. }
  457. return false;
  458. }
  459. private static MultipartFile getMulFileByPath(String picPath) {
  460. FileItem fileItem = createFileItem(picPath);
  461. MultipartFile mfile = new CommonsMultipartFile(fileItem);
  462. return mfile;
  463. }
  464. private static FileItem createFileItem(String filePath) {
  465. FileItemFactory factory = new DiskFileItemFactory(16, null);
  466. String textFieldName = "textField";
  467. int num = filePath.lastIndexOf(".");
  468. String extFile = filePath.substring(num);
  469. FileItem item = factory.createItem(textFieldName, "text/plain", true,
  470. "MyFileName" + extFile);
  471. File newfile = new File(filePath);
  472. int bytesRead = 0;
  473. byte[] buffer = new byte[8192];
  474. try
  475. {
  476. FileInputStream fis = new FileInputStream(newfile);
  477. OutputStream os = item.getOutputStream();
  478. while ((bytesRead = fis.read(buffer, 0, 8192))
  479. != -1)
  480. {
  481. os.write(buffer, 0, bytesRead);
  482. }
  483. os.close();
  484. fis.close();
  485. }
  486. catch (IOException e)
  487. {
  488. e.printStackTrace();
  489. }
  490. return item;
  491. }
  492. /**
  493. * 下载文件
  494. * @param bucketName 桶名
  495. * @param remoteFileName 文件名
  496. * @param path 下载路径
  497. */
  498. public boolean downFromS3(String bucketName, String remoteFileName, String path) {
  499. try {
  500. /**
  501. * 创建s3对象
  502. */
  503. BasicAWSCredentials awsCreds = new BasicAWSCredentials(s3key, s3secrey);
  504. AmazonS3 s3 = AmazonS3ClientBuilder.standard()
  505. .withCredentials(new AWSStaticCredentialsProvider(awsCreds))
  506. .withRegion(Regions.EU_WEST_2)
  507. .build();
  508. GetObjectRequest request = new GetObjectRequest(bucketName,remoteFileName);
  509. s3.getObject(request,new File(path));
  510. return true;
  511. } catch (Exception ase) {
  512. log.error("amazonS3下载文件异常 " + ase.getMessage(), ase);
  513. }
  514. return false;
  515. }
  516. public List<String> listKeys(String sourcePath){
  517. StorageType storageType = StorageType.get(type);
  518. switch (storageType){
  519. case OSS:
  520. return this.listKeysFromAli(sourcePath);
  521. case AWS:
  522. return this.listKeysFromAws(sourcePath);
  523. case LOCAL:
  524. return this.listKeysFromLocal(sourcePath);
  525. }
  526. return null;
  527. }
  528. /**
  529. * 获得文件列表-阿里云
  530. * @return
  531. */
  532. public List<String> listKeysFromAli(String sourcePath) {
  533. List<String> keyList = new ArrayList<>();
  534. OSSClient ossClient = new OSSClient(point, key, secrey);
  535. boolean flag = true;
  536. String nextMaker = null;
  537. ListObjectsRequest listObjectsRequest = new ListObjectsRequest(this.bucket);
  538. //指定下一级文件
  539. listObjectsRequest.setPrefix(sourcePath);
  540. //设置分页的页容量
  541. listObjectsRequest.setMaxKeys(200);
  542. do
  543. {
  544. //获取下一页的起始点,它的下一项
  545. listObjectsRequest.setMarker(nextMaker);
  546. ObjectListing objectListing = ossClient.listObjects(listObjectsRequest);
  547. List<OSSObjectSummary> objectSummaries = objectListing.getObjectSummaries();
  548. List<String> collect = objectSummaries.stream().map(summary -> {
  549. return summary.getKey();
  550. }).collect(Collectors.toList());
  551. if(CollUtil.isNotEmpty(collect)){
  552. keyList.addAll(collect);
  553. }
  554. nextMaker = objectListing.getNextMarker();
  555. //全部执行完后,为false
  556. flag = objectListing.isTruncated();
  557. } while (flag);
  558. ossClient.shutdown();
  559. return keyList;
  560. }
  561. /**
  562. * 获得文件列表-亚马逊
  563. * @return
  564. */
  565. public List<String> listKeysFromAws(String sourcePath) {
  566. List<String> keyList = new ArrayList<>();
  567. BasicAWSCredentials awsCreds = new BasicAWSCredentials(s3key, s3secrey);
  568. AmazonS3 s3 = AmazonS3ClientBuilder.standard()
  569. .withCredentials(new AWSStaticCredentialsProvider(awsCreds))
  570. .withRegion(Regions.EU_WEST_2)
  571. .build();
  572. boolean flag = true;
  573. String nextMaker = null;
  574. com.amazonaws.services.s3.model.ListObjectsRequest listObjectsRequest = new com.amazonaws.services.s3.model.ListObjectsRequest();
  575. listObjectsRequest.setBucketName(this.bucket);
  576. listObjectsRequest.setPrefix(sourcePath);
  577. listObjectsRequest.setMaxKeys(200);
  578. do{
  579. listObjectsRequest.setMarker(nextMaker);
  580. com.amazonaws.services.s3.model.ObjectListing objectListing = s3.listObjects(listObjectsRequest);
  581. List<S3ObjectSummary> objectSummaries = objectListing.getObjectSummaries();
  582. List<String> collect =objectSummaries.stream().map(summary->{
  583. return summary.getKey();
  584. }).collect(Collectors.toList());
  585. if(CollUtil.isNotEmpty(collect)){
  586. keyList.addAll(collect);
  587. }
  588. nextMaker = objectListing.getNextMarker();
  589. flag = objectListing.isTruncated();
  590. }while (flag);
  591. s3.shutdown();
  592. return keyList;
  593. }
  594. /**
  595. * 获得文件列表-阿里云
  596. * @return
  597. */
  598. public List<String> listKeysFromLocal(String sourcePath) {
  599. List<String> keyList = new ArrayList<>();
  600. return keyList;
  601. }
  602. /**
  603. * <p>
  604. 拷贝
  605. * </p>
  606. * @author dengsixing
  607. * @date 2022/1/18
  608. * @param sourcePath
  609. * @param targetPath
  610. **/
  611. public void copyFiles(String sourcePath, String targetPath) throws IOException {
  612. StorageType storageType = StorageType.get(type);
  613. switch (storageType){
  614. case OSS: this.copyFilesFromAli(sourcePath, targetPath);
  615. case AWS: this.copyFilesFromAws(sourcePath, targetPath);
  616. case LOCAL: this.copyFilesFromLocal(sourcePath, targetPath);
  617. }
  618. }
  619. /**
  620. * <p>
  621. 拷贝-阿里云
  622. * </p>
  623. * @author dengsixing
  624. * @date 2022/1/18
  625. * @param sourcePath
  626. * @param targetPath
  627. **/
  628. public void copyFilesFromAli(String sourcePath, String targetPath) throws IOException {
  629. //获取源文件列表
  630. List<String> sourceKeyList = this.listKeysFromAli(sourcePath);
  631. if(CollUtil.isEmpty(sourceKeyList)){
  632. return;
  633. }
  634. // 创建OSSClient实例。
  635. OSSClient ossClient = new OSSClient(point, key, secrey);
  636. // 复制文件
  637. sourceKeyList.parallelStream().forEach(key -> {
  638. log.info("开始复制:" + key);
  639. ossClient.copyObject(this.bucket, key, this.bucket, key.replace(sourcePath, targetPath));
  640. log.info("复制成功:" + key);
  641. });
  642. ossClient.shutdown();
  643. }
  644. /**
  645. * <p>
  646. 拷贝-亚马逊
  647. * </p>
  648. * @author dengsixing
  649. * @date 2022/1/18
  650. * @param sourcePath
  651. * @param targetPath
  652. **/
  653. public void copyFilesFromAws(String sourcePath, String targetPath) throws IOException {
  654. // TODO: 2022/1/21
  655. }
  656. /**
  657. * <p>
  658. 拷贝-本地
  659. * </p>
  660. * @author dengsixing
  661. * @date 2022/1/18
  662. * @param sourcePath
  663. * @param targetPath
  664. **/
  665. public void copyFilesFromLocal(String sourcePath, String targetPath) throws IOException {
  666. // TODO: 2022/1/21
  667. }
  668. /**
  669. * 获取文件内容
  670. * @param bucketName
  671. * @param objectName
  672. * @return
  673. */
  674. public String getObjectContent(String bucketName, String objectName){
  675. StorageType storageType = StorageType.get(type);
  676. switch (storageType){
  677. case OSS:
  678. return this.getObjectContentFromAli(bucketName, objectName);
  679. case AWS:
  680. return this.getObjectContentFromAws(bucketName, objectName);
  681. case LOCAL:
  682. return this.getObjectContentFromLocal(objectName);
  683. }
  684. return null;
  685. }
  686. /**
  687. * 获取文件内容-阿里云
  688. * @param bucketName
  689. * @param objectName
  690. * @return
  691. */
  692. public String getObjectContentFromAli(String bucketName, String objectName){
  693. //创建oss客户端
  694. OSSClient ossClient = new OSSClient(point, key, secrey);
  695. InputStream objectContent = null;
  696. StringBuilder contentJson = new StringBuilder();
  697. try {
  698. // ossObject包含文件所在的存储空间名称、文件名称、文件元信息以及一个输入流。
  699. OSSObject ossObject = ossClient.getObject(bucketName, objectName);
  700. objectContent = ossObject.getObjectContent();
  701. try(BufferedReader reader = new BufferedReader(new InputStreamReader(objectContent))){
  702. while (true) {
  703. String line = reader.readLine();
  704. if (line == null) break;
  705. contentJson.append(line);
  706. }
  707. } catch (IOException e) {
  708. log.error("读取scene.json文件流失败", e);
  709. }
  710. ossClient.shutdown();
  711. }catch (Exception e){
  712. log.error("oos找不到文件,文件路径:{}", objectName);
  713. }
  714. return contentJson.toString();
  715. }
  716. /**
  717. * 获取文件内容-亚马逊
  718. * @param bucketName
  719. * @param objectName
  720. * @return
  721. */
  722. public String getObjectContentFromAws(String bucketName, String objectName){
  723. try {
  724. /**
  725. * 创建s3对象
  726. */
  727. BasicAWSCredentials awsCreds = new BasicAWSCredentials(s3key, s3secrey);
  728. AmazonS3 s3 = AmazonS3ClientBuilder.standard()
  729. .withCredentials(new AWSStaticCredentialsProvider(awsCreds))
  730. .withRegion(Regions.EU_WEST_2)
  731. .build();
  732. GetObjectRequest request = new GetObjectRequest(bucketName,objectName);
  733. S3Object object = s3.getObject(request);
  734. S3ObjectInputStream inputStream = object.getObjectContent();
  735. StringBuilder content = new StringBuilder();
  736. try(BufferedReader reader = new BufferedReader(new InputStreamReader(inputStream))){
  737. while (true) {
  738. String line = reader.readLine();
  739. if (line == null) break;
  740. content.append(line);
  741. }
  742. } catch (IOException e) {
  743. log.error("读取aws文件流失败", e);
  744. }
  745. return content.toString();
  746. } catch (Exception ase) {
  747. log.error("amazonS3下载文件异常 " + ase.getMessage(), ase);
  748. }
  749. return null;
  750. }
  751. /**
  752. * 获取文件内容-本地
  753. * @param objectName
  754. * @return
  755. */
  756. public String getObjectContentFromLocal(String objectName){
  757. // TODO: 2022/1/21
  758. return null;
  759. }
  760. }