UploadToOssUtil.java 28 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031
  1. package com.fdkankan.fyun.oss;
  2. import cn.hutool.core.collection.CollUtil;
  3. import cn.hutool.core.io.FileUtil;
  4. import cn.hutool.core.util.StrUtil;
  5. import com.alibaba.fastjson.JSON;
  6. import com.aliyun.oss.OSSClient;
  7. import com.aliyun.oss.model.*;
  8. import com.amazonaws.HttpMethod;
  9. import com.amazonaws.auth.AWSStaticCredentialsProvider;
  10. import com.amazonaws.auth.BasicAWSCredentials;
  11. import com.amazonaws.regions.Regions;
  12. import com.amazonaws.services.s3.AmazonS3;
  13. import com.amazonaws.services.s3.AmazonS3ClientBuilder;
  14. import com.amazonaws.services.s3.model.CannedAccessControlList;
  15. import com.amazonaws.services.s3.model.GeneratePresignedUrlRequest;
  16. import com.amazonaws.services.s3.model.GetObjectRequest;
  17. import com.amazonaws.services.s3.model.PutObjectRequest;
  18. import com.amazonaws.services.s3.model.S3Object;
  19. import com.amazonaws.services.s3.model.S3ObjectInputStream;
  20. import com.amazonaws.services.s3.model.S3ObjectSummary;
  21. import com.fdkankan.fyun.constant.StorageType;
  22. import com.qiniu.common.Zone;
  23. import com.qiniu.storage.Configuration;
  24. import com.qiniu.storage.UploadManager;
  25. import java.util.stream.Collectors;
  26. import lombok.extern.slf4j.Slf4j;
  27. import org.apache.commons.fileupload.FileItem;
  28. import org.apache.commons.fileupload.FileItemFactory;
  29. import org.apache.commons.fileupload.disk.DiskFileItemFactory;
  30. import org.apache.commons.io.FileUtils;
  31. import org.apache.commons.lang3.StringUtils;
  32. import org.springframework.beans.factory.annotation.Value;
  33. import org.springframework.stereotype.Component;
  34. import org.springframework.web.multipart.MultipartFile;
  35. import org.springframework.web.multipart.commons.CommonsMultipartFile;
  36. import java.io.*;
  37. import java.net.FileNameMap;
  38. import java.net.URL;
  39. import java.net.URLConnection;
  40. import java.util.ArrayList;
  41. import java.util.HashMap;
  42. import java.util.List;
  43. import java.util.Map;
  44. @Slf4j
  45. @Component
  46. public class UploadToOssUtil {
  47. Zone zone = Zone.autoZone();
  48. Configuration config = new Configuration(zone);
  49. UploadManager uploadManager = new UploadManager(config);
  50. @Value("${oss.point:http://oss-cn-shenzhen-internal.aliyuncs.com}")
  51. private String point;
  52. @Value("${oss.key:LTAIUrvuHqj8pvry}")
  53. private String key;
  54. @Value("${oss.secrey:JLOVl0k8Ke0aaM8nLMMiUAZ3EiiqI4}")
  55. private String secrey;
  56. @Value("${oss.bucket:4dkankan}")
  57. private String bucket;
  58. @Value("${oss.sdk:4dscene}")
  59. private String bucketSdk;
  60. @Value("${upload.type:oss}")
  61. private String type;
  62. @Value("${aws.s3key:AKIAWCV5QFZ3ZNELKYUY}")
  63. private String s3key;
  64. @Value("${aws.s3secrey:epS5ghyR4LJ7rxk/qJO9ZYh6m9Oz6g5haKDu4yws}")
  65. private String s3secrey;
  66. @Value("${aws.s3bucket:4dkankan}")
  67. private String s3bucket;
  68. @Value("${local.path:/home/4dkankan}")
  69. private String localPath;
  70. //上传的数据是byte[],key是上传后的文件名
  71. public void upload(byte[] data,String key1) throws IOException{
  72. log.info("开始上传文件 源路径:{},目标路径:{},type:{}" , new String(data, "UTF-8"),key1,type);
  73. StorageType storageType = StorageType.get(type);
  74. switch (storageType){
  75. case OSS:
  76. uploadOss(data,key1);
  77. break;
  78. case AWS:
  79. uploadAws(data,key1);
  80. break;
  81. case LOCAL:
  82. uploadLocal(data,key1);
  83. break;
  84. }
  85. }
  86. public void upload(String filePath, String key1) {
  87. log.info("开始上传文件 源路径:{},目标路径:{},type:{}" , filePath,key1,type);
  88. StorageType storageType = StorageType.get(type);
  89. switch (storageType){
  90. case OSS:
  91. uploadOss(filePath,key1);
  92. break;
  93. case AWS:
  94. uploadAws(filePath,key1);
  95. break;
  96. case LOCAL:
  97. uploadLocal(filePath,key1);
  98. break;
  99. }
  100. }
  101. public void uploadSdk(String filePath, String key1) {
  102. log.info("开始上传文件 源路径:{},目标路径:{},type:{}" , filePath,key1,type);
  103. switch (type){
  104. case "oss":uploadSdkOss(filePath,key1); break;
  105. case "aws": uploadAws(filePath,key1); break;
  106. case "local":uploadLocal(filePath,key1); break;
  107. }
  108. }
  109. public void upload2(String filePath, String key1) {
  110. log.info("开始上传文件 源路径:{},目标路径:{},type:{}" , filePath,key1,type);
  111. switch (type){
  112. case "oss":upload2Oss(filePath,key1); break;
  113. case "aws": uploadAws(filePath,key1); break;
  114. case "local":uploadLocal(filePath,key1); break;
  115. }
  116. }
  117. public void delete(String key1) throws IOException{
  118. switch (type){
  119. case "oss":deleteOss(key1); break;
  120. case "aws": deleteS3Object(key1); break;
  121. case "local":FileUtil.del(key1); break;
  122. }
  123. }
  124. public int deleteFile(String prefix){
  125. switch (type){
  126. case "oss":deleteOssFile(prefix); break;
  127. case "aws": deleteS3Object(prefix); break;
  128. case "local":FileUtil.del(prefix); break;
  129. }
  130. return 1;
  131. }
  132. public void deleteOss(String key1){
  133. OSSClient ossClient = new OSSClient(point, key, secrey);
  134. try {
  135. ossClient.deleteObject(bucket, key1);
  136. } catch (Exception e) {
  137. e.printStackTrace();
  138. }
  139. }
  140. public void deleteOssFile(String prefix){
  141. OSSClient ossClient = new OSSClient(point, key, secrey);
  142. ObjectListing objectListing = ossClient.listObjects(bucket, prefix);
  143. List<OSSObjectSummary> sums = objectListing.getObjectSummaries();
  144. try {
  145. for (OSSObjectSummary s : sums) {
  146. delete(s.getKey());
  147. }
  148. } catch (IOException e) {
  149. e.printStackTrace();
  150. }
  151. }
  152. public void uploadOss(byte[] data,String key1){
  153. OSSClient ossClient = new OSSClient(point, key, secrey);
  154. try {
  155. ossClient.putObject(bucket, key1, new ByteArrayInputStream(data));
  156. } catch (Exception e) {
  157. log.error(e.toString()+key1);
  158. }
  159. }
  160. public void uploadAws(byte[] data,String key1){
  161. }
  162. public void uploadLocal(byte[] data,String key1){
  163. InputStream in = new ByteArrayInputStream(data);
  164. File file = new File(key1);
  165. String path = key1.substring(0, key1.lastIndexOf("/"));
  166. if (!file.exists()) {
  167. new File(path).mkdir();
  168. }
  169. FileOutputStream fos = null;
  170. try {
  171. fos = new FileOutputStream(file);
  172. int len = 0;
  173. byte[] buf = new byte[1024];
  174. while ((len = in.read(buf)) != -1) {
  175. fos.write(buf, 0, len);
  176. }
  177. fos.flush();
  178. } catch (Exception e) {
  179. e.printStackTrace();
  180. } finally {
  181. if (null != fos) {
  182. try {
  183. fos.close();
  184. } catch (IOException e) {
  185. e.printStackTrace();
  186. }
  187. }
  188. }
  189. }
  190. public void uploadOss(String filePath, String key1){
  191. OSSClient ossClient = new OSSClient(point, key, secrey);
  192. try {
  193. File file = new File(filePath);
  194. if (!file.exists()) {
  195. log.error("要上传的文件不存在:" + filePath);
  196. return;
  197. }
  198. ObjectMetadata metadata = new ObjectMetadata();
  199. if(filePath.contains(".jpg")){
  200. metadata.setContentType("image/jpeg");
  201. }
  202. ossClient.putObject(bucket, key1, new File(filePath), metadata);
  203. } catch (Exception e) {
  204. log.error(e.toString() + filePath);
  205. } finally {
  206. ossClient.shutdown();
  207. }
  208. }
  209. public void uploadAws(String filePath, String key1){
  210. try{
  211. uploadS3File(filePath, key1);
  212. }catch (Exception e){
  213. e.printStackTrace();
  214. }
  215. }
  216. public void uploadLocal(String filePath, String key1){
  217. try {
  218. File srcFile = new File(filePath);
  219. File file = new File(localPath + key1);
  220. FileUtils.copyFile(srcFile,file);
  221. }catch (Exception e){
  222. e.printStackTrace();
  223. }
  224. }
  225. public void uploadSdkOss(String filePath, String key1){
  226. OSSClient ossClient = new OSSClient(point, key, secrey);
  227. try {
  228. File file = new File(filePath);
  229. if (!file.exists()) {
  230. log.error("要上传的文件不存在:" + filePath);
  231. return;
  232. }
  233. ObjectMetadata metadata = new ObjectMetadata();
  234. if(filePath.contains(".jpg")){
  235. metadata.setContentType("image/jpeg");
  236. }
  237. ossClient.putObject(bucketSdk, key1, new File(filePath), metadata);
  238. } catch (Exception e) {
  239. log.error(e.toString() + filePath);
  240. }
  241. }
  242. public void upload2Oss(String filePath, String key1){
  243. OSSClient ossClient = new OSSClient(point, key, secrey);
  244. try {
  245. ObjectMetadata metadata = new ObjectMetadata();
  246. if(filePath.contains(".jpg")){
  247. metadata.setContentType("image/jpeg");
  248. }
  249. if(filePath.contains(".mp4")){
  250. metadata.setContentType("video/mp4");
  251. }
  252. if(filePath.contains(".mp3")){
  253. metadata.setContentType("audio/mp3");
  254. }
  255. ossClient.putObject(bucket, key1, new File(filePath), metadata);
  256. } catch (Exception e) {
  257. log.error(e.toString() + filePath);
  258. }
  259. }
  260. //上传的数据是文件夹,参数是文件夹路径,key是上传后的文件名
  261. public void uploadMulFiles(Map<String, String> filepaths) {
  262. if (filepaths == null) {
  263. return;
  264. }
  265. Long start = System.currentTimeMillis();
  266. log.info("开始批量上传文件:");
  267. if (filepaths.size() > 50) {
  268. filepaths.entrySet().parallelStream().forEach(entry->{
  269. upload2(entry.getKey(), entry.getValue());
  270. });
  271. } else {
  272. filepaths.entrySet().parallelStream().forEach(entry->{
  273. upload(entry.getKey(), entry.getValue());
  274. });
  275. }
  276. log.info("批量上传文件结束,用时:{}" ,(System.currentTimeMillis() - start));
  277. }
  278. public Map<String, String> getUploadS3Url(List<String> urls){
  279. if(urls == null || urls.size() <= 0){
  280. return null;
  281. }
  282. BasicAWSCredentials awsCred = new BasicAWSCredentials(s3key, s3secrey);
  283. AmazonS3 s3Client = AmazonS3ClientBuilder.standard()
  284. .withCredentials(new AWSStaticCredentialsProvider(awsCred))
  285. .withRegion(Regions.EU_WEST_2)
  286. .build();
  287. // Set the pre-signed URL to expire after one hour.
  288. java.util.Date expiration = new java.util.Date();
  289. long expTimeMillis = expiration.getTime();
  290. expTimeMillis += 1000 * 60 * 60 * 8;
  291. expiration.setTime(expTimeMillis);
  292. //生成预签名URL
  293. log.info("生成预签名URL");
  294. GeneratePresignedUrlRequest generatePresignedUrlRequest = null;
  295. URL url = null;
  296. Map<String, String> map = new HashMap();
  297. for(String path : urls){
  298. // if(path.contains(".jpg") || path.contains("png")){
  299. // generatePresignedUrlRequest = new GeneratePresignedUrlRequest(s3bucket, path)
  300. // .withMethod(HttpMethod.PUT)
  301. // .withExpiration(expiration)
  302. // .withContentType("image/jpeg");
  303. // }else {
  304. generatePresignedUrlRequest = new GeneratePresignedUrlRequest(s3bucket, path)
  305. .withMethod(HttpMethod.PUT)
  306. .withExpiration(expiration);
  307. // }
  308. url = s3Client.generatePresignedUrl(generatePresignedUrlRequest);
  309. map.put(path, url.toString());
  310. }
  311. return map;
  312. }
  313. public String upload5(String filePath, String key1) {
  314. OSSClient ossClient = new OSSClient(point, key, secrey);
  315. PutObjectResult result = null;
  316. try {
  317. File file = new File(filePath);
  318. if (!file.exists()) {
  319. log.error("要上传的文件不存在:" + filePath);
  320. }
  321. result = ossClient.putObject(bucket, key1, new File(filePath));
  322. } catch (Exception e) {
  323. log.error(e.toString() + filePath);
  324. }
  325. log.info(" getETag : " + result.getETag());
  326. log.info("1 : " + result.toString());
  327. log.info("2 : " + result.getRequestId());
  328. log.info("3 : " + result.getClientCRC());
  329. log.info("4 : " + result.getResponse());
  330. log.info("5 : " + result.getServerCRC());
  331. return result.getETag();
  332. }
  333. //海外亚马逊s3
  334. /**
  335. * s3上传文件流
  336. *
  337. * @param file 文件
  338. * @param updatePath 上传路径[ eg: xxx/xxx ]
  339. */
  340. public String updateS3LoadFile(MultipartFile file, String updatePath) {
  341. if (isEmpty(file)) {
  342. return null;
  343. }
  344. /**
  345. * 创建s3对象
  346. */
  347. BasicAWSCredentials awsCreds = new BasicAWSCredentials(s3key, s3secrey);
  348. AmazonS3 s3 = AmazonS3ClientBuilder.standard()
  349. .withCredentials(new AWSStaticCredentialsProvider(awsCreds))
  350. .withRegion(Regions.EU_WEST_2)
  351. .build();
  352. try {
  353. // 创建临时文件,程序运行结束,会自动删除
  354. File localFile = File.createTempFile("temp", null);
  355. // 把文件写入内存中
  356. file.transferTo(localFile);
  357. // 指定要上传到服务器上的路径
  358. String key = updatePath;
  359. // 设置文件并设置公读
  360. PutObjectRequest request = new PutObjectRequest(s3bucket, key, localFile);
  361. request.withCannedAcl(CannedAccessControlList.PublicRead);
  362. // 上传文件
  363. com.amazonaws.services.s3.model.PutObjectResult putObjectResult = s3.putObject(request);
  364. if (StringUtils.isNotEmpty(putObjectResult.getETag())) {
  365. System.out.println("success");
  366. return key;
  367. }
  368. return null;
  369. } catch (IOException e) {
  370. }
  371. return null;
  372. }
  373. /**
  374. * s3上传文件
  375. * @param filePath
  376. * @param key1
  377. * @throws IOException
  378. */
  379. private void uploadS3File(String filePath, String key1) throws Exception {
  380. /**
  381. * 创建s3对象
  382. */
  383. BasicAWSCredentials awsCreds = new BasicAWSCredentials(s3key, s3secrey);
  384. AmazonS3 s3 = AmazonS3ClientBuilder.standard()
  385. .withCredentials(new AWSStaticCredentialsProvider(awsCreds))
  386. .withRegion(Regions.EU_WEST_2)
  387. .build();
  388. try{
  389. File file = new File(filePath);
  390. if(!file.exists()){
  391. log.info("要上传s3的文件不存在");
  392. return;
  393. }
  394. // 设置文件并设置公读
  395. com.amazonaws.services.s3.model.ObjectMetadata metadata = new com.amazonaws.services.s3.model.ObjectMetadata();
  396. if(filePath.contains(".jpg")){
  397. metadata.setContentType("image/jpeg");
  398. }
  399. if(filePath.contains(".png")){
  400. metadata.setContentType("image/png");
  401. }
  402. PutObjectRequest request = new PutObjectRequest(s3bucket, key1, file);
  403. request.withCannedAcl(CannedAccessControlList.PublicRead);
  404. request.withMetadata(metadata);
  405. // 上传文件
  406. com.amazonaws.services.s3.model.PutObjectResult putObjectResult = s3.putObject(request);
  407. if (StringUtils.isNotEmpty(putObjectResult.getETag())) {
  408. log.info("s3上传文件成功:" + key1);
  409. }
  410. }catch (Exception e){
  411. throw e;
  412. }finally {
  413. s3.shutdown();
  414. }
  415. }
  416. /**
  417. * 删除单个文件
  418. *
  419. * @param filePath 文件路径[ eg: /head/xxxx.jpg ]
  420. * @return
  421. */
  422. public void deleteS3Object(String filePath) {
  423. /**
  424. * 创建s3对象
  425. */
  426. BasicAWSCredentials awsCreds = new BasicAWSCredentials(s3key, s3secrey);
  427. AmazonS3 s3 = AmazonS3ClientBuilder.standard()
  428. .withCredentials(new AWSStaticCredentialsProvider(awsCreds))
  429. .withRegion(Regions.EU_WEST_2)
  430. .build();
  431. if (filePath.startsWith("/")) {
  432. filePath = filePath.substring(1);
  433. }
  434. try {
  435. s3.deleteObject(s3bucket, filePath);
  436. } catch (Exception e) {
  437. }
  438. }
  439. /**
  440. * 获取文件类型
  441. */
  442. public static String getContentType(String filePath){
  443. FileNameMap fileNameMap = URLConnection.getFileNameMap();
  444. String contentType = fileNameMap.getContentTypeFor(filePath);
  445. System.out.println(contentType);
  446. return contentType;
  447. }
  448. /**
  449. * 检查文件是否为空
  450. *
  451. * @param imageFile
  452. * @return
  453. */
  454. private static boolean isEmpty(MultipartFile imageFile) {
  455. if (imageFile == null || imageFile.getSize() <= 0) {
  456. return true;
  457. }
  458. return false;
  459. }
  460. private static MultipartFile getMulFileByPath(String picPath) {
  461. FileItem fileItem = createFileItem(picPath);
  462. MultipartFile mfile = new CommonsMultipartFile(fileItem);
  463. return mfile;
  464. }
  465. private static FileItem createFileItem(String filePath) {
  466. FileItemFactory factory = new DiskFileItemFactory(16, null);
  467. String textFieldName = "textField";
  468. int num = filePath.lastIndexOf(".");
  469. String extFile = filePath.substring(num);
  470. FileItem item = factory.createItem(textFieldName, "text/plain", true,
  471. "MyFileName" + extFile);
  472. File newfile = new File(filePath);
  473. int bytesRead = 0;
  474. byte[] buffer = new byte[8192];
  475. try
  476. {
  477. FileInputStream fis = new FileInputStream(newfile);
  478. OutputStream os = item.getOutputStream();
  479. while ((bytesRead = fis.read(buffer, 0, 8192))
  480. != -1)
  481. {
  482. os.write(buffer, 0, bytesRead);
  483. }
  484. os.close();
  485. fis.close();
  486. }
  487. catch (IOException e)
  488. {
  489. e.printStackTrace();
  490. }
  491. return item;
  492. }
  493. public List<String> listKeys(String sourcePath){
  494. StorageType storageType = StorageType.get(type);
  495. switch (storageType){
  496. case OSS:
  497. return this.listKeysFromAli(sourcePath);
  498. case AWS:
  499. return this.listKeysFromAws(sourcePath);
  500. case LOCAL:
  501. return this.listKeysFromLocal(sourcePath);
  502. }
  503. return null;
  504. }
  505. /**
  506. * 获得文件列表-阿里云
  507. * @return
  508. */
  509. public List<String> listKeysFromAli(String sourcePath) {
  510. List<String> keyList = new ArrayList<>();
  511. OSSClient ossClient = new OSSClient(point, key, secrey);
  512. boolean flag = true;
  513. String nextMaker = null;
  514. ListObjectsRequest listObjectsRequest = new ListObjectsRequest(this.bucket);
  515. //指定下一级文件
  516. listObjectsRequest.setPrefix(sourcePath);
  517. //设置分页的页容量
  518. listObjectsRequest.setMaxKeys(200);
  519. do
  520. {
  521. //获取下一页的起始点,它的下一项
  522. listObjectsRequest.setMarker(nextMaker);
  523. ObjectListing objectListing = ossClient.listObjects(listObjectsRequest);
  524. List<OSSObjectSummary> objectSummaries = objectListing.getObjectSummaries();
  525. List<String> collect = objectSummaries.stream().map(summary -> {
  526. return summary.getKey();
  527. }).collect(Collectors.toList());
  528. if(CollUtil.isNotEmpty(collect)){
  529. keyList.addAll(collect);
  530. }
  531. nextMaker = objectListing.getNextMarker();
  532. //全部执行完后,为false
  533. flag = objectListing.isTruncated();
  534. } while (flag);
  535. ossClient.shutdown();
  536. return keyList;
  537. }
  538. /**
  539. * 获得文件列表-亚马逊
  540. * @return
  541. */
  542. public List<String> listKeysFromAws(String sourcePath) {
  543. List<String> keyList = new ArrayList<>();
  544. BasicAWSCredentials awsCreds = new BasicAWSCredentials(s3key, s3secrey);
  545. AmazonS3 s3 = AmazonS3ClientBuilder.standard()
  546. .withCredentials(new AWSStaticCredentialsProvider(awsCreds))
  547. .withRegion(Regions.EU_WEST_2)
  548. .build();
  549. boolean flag = true;
  550. String nextMaker = null;
  551. com.amazonaws.services.s3.model.ListObjectsRequest listObjectsRequest = new com.amazonaws.services.s3.model.ListObjectsRequest();
  552. listObjectsRequest.setBucketName(this.bucket);
  553. listObjectsRequest.setPrefix(sourcePath);
  554. listObjectsRequest.setMaxKeys(200);
  555. do{
  556. listObjectsRequest.setMarker(nextMaker);
  557. com.amazonaws.services.s3.model.ObjectListing objectListing = s3.listObjects(listObjectsRequest);
  558. List<S3ObjectSummary> objectSummaries = objectListing.getObjectSummaries();
  559. List<String> collect =objectSummaries.stream().map(summary->{
  560. return summary.getKey();
  561. }).collect(Collectors.toList());
  562. if(CollUtil.isNotEmpty(collect)){
  563. keyList.addAll(collect);
  564. }
  565. nextMaker = objectListing.getNextMarker();
  566. flag = objectListing.isTruncated();
  567. }while (flag);
  568. s3.shutdown();
  569. return keyList;
  570. }
  571. /**
  572. * 获得文件列表-阿里云
  573. * @return
  574. */
  575. public List<String> listKeysFromLocal(String sourcePath) {
  576. List<String> keyList = new ArrayList<>();
  577. return keyList;
  578. }
  579. /**
  580. * <p>
  581. 拷贝目录
  582. * </p>
  583. * @author dengsixing
  584. * @date 2022/1/18
  585. * @param sourcePath
  586. * @param targetPath
  587. **/
  588. public void copyFiles(String sourcePath, String targetPath) throws IOException {
  589. StorageType storageType = StorageType.get(type);
  590. switch (storageType){
  591. case OSS:
  592. this.copyFilesFromAli(sourcePath, targetPath);
  593. break;
  594. case AWS:
  595. this.copyFilesFromAws(sourcePath, targetPath);
  596. break;
  597. case LOCAL: this.copyFilesFromLocal(sourcePath, targetPath);
  598. }
  599. }
  600. /**
  601. * <p>
  602. 拷贝文件
  603. * </p>
  604. * @author dengsixing
  605. * @date 2022/1/18
  606. * @param sourceKey
  607. * @param targetKey
  608. **/
  609. public void copyObject(String sourceKey, String targetKey) throws IOException {
  610. StorageType storageType = StorageType.get(type);
  611. switch (storageType){
  612. case OSS:
  613. this.copyObjectFromAli(sourceKey, targetKey);
  614. break;
  615. case AWS:
  616. this.copyObjectFromAws(sourceKey, targetKey);
  617. break;
  618. }
  619. }
  620. /**
  621. * <p>
  622. 拷贝-阿里云
  623. * </p>
  624. * @author dengsixing
  625. * @date 2022/1/18
  626. * @param sourcePath
  627. * @param targetPath
  628. **/
  629. public void copyObjectFromAli(String sourcePath, String targetPath) throws IOException {
  630. // 创建OSSClient实例。
  631. OSSClient ossClient = new OSSClient(point, key, secrey);
  632. // 复制文件
  633. log.info("开始复制:" + sourcePath);
  634. ossClient.copyObject(this.bucket, sourcePath, this.bucket, targetPath);
  635. log.info("复制成功:" + sourcePath);
  636. ossClient.shutdown();
  637. }
  638. /**
  639. * <p>
  640. 拷贝-阿里云
  641. * </p>
  642. * @author dengsixing
  643. * @date 2022/1/18
  644. * @param sourcePath
  645. * @param targetPath
  646. **/
  647. public void copyFilesFromAli(String sourcePath, String targetPath) throws IOException {
  648. //获取源文件列表
  649. List<String> sourceKeyList = this.listKeysFromAli(sourcePath);
  650. if(CollUtil.isEmpty(sourceKeyList)){
  651. return;
  652. }
  653. // 创建OSSClient实例。
  654. OSSClient ossClient = new OSSClient(point, key, secrey);
  655. // 复制文件
  656. sourceKeyList.parallelStream().forEach(key -> {
  657. log.info("开始复制:" + key);
  658. ossClient.copyObject(this.bucket, key, this.bucket, key.replace(sourcePath, targetPath));
  659. log.info("复制成功:" + key);
  660. });
  661. ossClient.shutdown();
  662. }
  663. /**
  664. * <p>
  665. 拷贝-亚马逊
  666. * </p>
  667. * @author dengsixing
  668. * @date 2022/1/18
  669. * @param sourcePath
  670. * @param targetPath
  671. **/
  672. public void copyFilesFromAws(String sourcePath, String targetPath){
  673. try {
  674. List<String> sourceKeyList = this.listKeysFromAws(sourcePath);
  675. /**
  676. * 创建s3对象
  677. */
  678. BasicAWSCredentials awsCreds = new BasicAWSCredentials(s3key, s3secrey);
  679. AmazonS3 s3 = AmazonS3ClientBuilder.standard()
  680. .withCredentials(new AWSStaticCredentialsProvider(awsCreds))
  681. .withRegion(Regions.EU_WEST_2)//s3 地区位置
  682. .build();
  683. // 复制文件
  684. sourceKeyList.parallelStream().forEach(key -> {
  685. log.info("开始复制:" + key);
  686. s3.copyObject(this.bucket, key, this.bucket, key.replace(sourcePath, targetPath));
  687. log.info("复制成功:" + key);
  688. });
  689. s3.shutdown();
  690. } catch (Exception ase) {
  691. log.error("amazonS拷贝异常 " + ase.getMessage(), ase);
  692. }
  693. }
  694. /**
  695. * <p>
  696. 拷贝-亚马逊
  697. * </p>
  698. * @author dengsixing
  699. * @date 2022/1/18
  700. * @param sourceKey
  701. * @param targetKey
  702. **/
  703. public void copyObjectFromAws(String sourceKey, String targetKey){
  704. BasicAWSCredentials awsCreds = new BasicAWSCredentials(s3key, s3secrey);
  705. AmazonS3 s3 = AmazonS3ClientBuilder.standard()
  706. .withCredentials(new AWSStaticCredentialsProvider(awsCreds))
  707. .withRegion(Regions.EU_WEST_2)//s3 地区位置
  708. .build();
  709. // 复制文件
  710. log.info("开始复制:" + sourceKey);
  711. s3.copyObject(this.bucket, sourceKey, this.bucket, targetKey);
  712. log.info("复制成功:" + sourceKey);
  713. s3.shutdown();
  714. }
  715. /**
  716. * <p>
  717. 拷贝-本地
  718. * </p>
  719. * @author dengsixing
  720. * @date 2022/1/18
  721. * @param sourcePath
  722. * @param targetPath
  723. **/
  724. public void copyFilesFromLocal(String sourcePath, String targetPath) throws IOException {
  725. // TODO: 2022/1/21
  726. }
  727. /**
  728. * 获取文件内容
  729. * @param bucketName
  730. * @param objectName
  731. * @return
  732. */
  733. public String getObjectContent(String bucketName, String objectName){
  734. StorageType storageType = StorageType.get(type);
  735. switch (storageType){
  736. case OSS:
  737. return this.getObjectContentFromAli(bucketName, objectName);
  738. case AWS:
  739. return this.getObjectContentFromAws(bucketName, objectName);
  740. case LOCAL:
  741. return this.getObjectContentFromLocal(objectName);
  742. }
  743. return null;
  744. }
  745. /**
  746. * 获取文件内容-阿里云
  747. * @param bucketName
  748. * @param objectName
  749. * @return
  750. */
  751. public String getObjectContentFromAli(String bucketName, String objectName){
  752. //创建oss客户端
  753. OSSClient ossClient = new OSSClient(point, key, secrey);
  754. InputStream objectContent = null;
  755. StringBuilder contentJson = new StringBuilder();
  756. try {
  757. // ossObject包含文件所在的存储空间名称、文件名称、文件元信息以及一个输入流。
  758. OSSObject ossObject = ossClient.getObject(bucketName, objectName);
  759. objectContent = ossObject.getObjectContent();
  760. try(BufferedReader reader = new BufferedReader(new InputStreamReader(objectContent))){
  761. while (true) {
  762. String line = reader.readLine();
  763. if (line == null) break;
  764. contentJson.append(line);
  765. }
  766. } catch (IOException e) {
  767. log.error("读取scene.json文件流失败", e);
  768. }
  769. ossClient.shutdown();
  770. }catch (Exception e){
  771. log.error("oos找不到文件,文件路径:{}", objectName);
  772. }
  773. return contentJson.toString();
  774. }
  775. /**
  776. * 获取文件内容-阿里云
  777. * @param objectName
  778. * @return
  779. */
  780. public boolean existOnAli(String objectName){
  781. //创建oss客户端
  782. OSSClient ossClient = new OSSClient(point, key, secrey);
  783. // ossObject包含文件所在的存储空间名称、文件名称、文件元信息以及一个输入流。
  784. OSSObject ossObject = ossClient.getObject(bucket, objectName);
  785. String key = ossObject.getKey();
  786. if(StrUtil.isNotEmpty(key))
  787. return true;
  788. return false;
  789. }
  790. /**
  791. * 获取文件内容-亚马逊
  792. * @param bucketName
  793. * @param objectName
  794. * @return
  795. */
  796. public String getObjectContentFromAws(String bucketName, String objectName){
  797. try {
  798. /**
  799. * 创建s3对象
  800. */
  801. BasicAWSCredentials awsCreds = new BasicAWSCredentials(s3key, s3secrey);
  802. AmazonS3 s3 = AmazonS3ClientBuilder.standard()
  803. .withCredentials(new AWSStaticCredentialsProvider(awsCreds))
  804. .withRegion(Regions.EU_WEST_2)
  805. .build();
  806. GetObjectRequest request = new GetObjectRequest(bucketName,objectName);
  807. S3Object object = s3.getObject(request);
  808. S3ObjectInputStream inputStream = object.getObjectContent();
  809. StringBuilder content = new StringBuilder();
  810. try(BufferedReader reader = new BufferedReader(new InputStreamReader(inputStream))){
  811. while (true) {
  812. String line = reader.readLine();
  813. if (line == null) break;
  814. content.append(line);
  815. }
  816. } catch (IOException e) {
  817. log.error("读取aws文件流失败", e);
  818. }
  819. return content.toString();
  820. } catch (Exception ase) {
  821. log.error("amazonS3下载文件异常 " + ase.getMessage(), ase);
  822. }
  823. return null;
  824. }
  825. /**
  826. * 获取文件内容-亚马逊
  827. * @param objectName
  828. * @return
  829. */
  830. public boolean existOnAws(String objectName){
  831. BasicAWSCredentials awsCreds = new BasicAWSCredentials(s3key, s3secrey);
  832. AmazonS3 s3 = AmazonS3ClientBuilder.standard()
  833. .withCredentials(new AWSStaticCredentialsProvider(awsCreds))
  834. .withRegion(Regions.EU_WEST_2)
  835. .build();
  836. GetObjectRequest request = new GetObjectRequest(bucket,objectName);
  837. S3Object s3Object = s3.getObject(request);
  838. String key = s3Object.getKey();
  839. if(StrUtil.isNotEmpty(key))
  840. return true;
  841. return false;
  842. }
  843. /**
  844. * 判断key是否存在
  845. * @param key
  846. * @return
  847. */
  848. public boolean existKey(String key){
  849. StorageType storageType = StorageType.get(type);
  850. switch (storageType){
  851. case OSS:
  852. return this.existOnAli(key);
  853. case AWS:
  854. return this.existOnAws(key);
  855. default:
  856. return false;
  857. }
  858. }
  859. /**
  860. * 获取文件内容-本地
  861. * @param objectName
  862. * @return
  863. */
  864. public String getObjectContentFromLocal(String objectName){
  865. // TODO: 2022/1/21
  866. return null;
  867. }
  868. /**
  869. * oss下载文件到本地
  870. * @param key
  871. * @param localPath
  872. */
  873. public boolean download(String objectName, String localPath){
  874. StorageType storageType = StorageType.get(this.type);
  875. switch (storageType){
  876. case OSS:
  877. return this.downFormAli(objectName, localPath);
  878. case AWS:
  879. return this.downFromS3(objectName, localPath);
  880. }
  881. return false;
  882. }
  883. /**
  884. * 从阿里云oss下载文件到本地
  885. * @param key 云端文件k地址
  886. * @param localPath 本地文件地址
  887. * @return
  888. */
  889. public boolean downFormAli(String objectName, String localPath){
  890. OSSClient ossClient = new OSSClient(point, key, secrey);
  891. try {
  892. com.aliyun.oss.model.GetObjectRequest request = new com.aliyun.oss.model.GetObjectRequest(bucket,objectName);
  893. ossClient.getObject(request, new File(localPath));
  894. return true;
  895. }catch (Exception e){
  896. log.error("阿里云oss文件下载失败,key=" + objectName, e);
  897. ossClient.shutdown();
  898. }
  899. return false;
  900. }
  901. /**
  902. * 从s3下载文件到本地
  903. * @param key 云端文件k地址
  904. * @param localPath 本地文件地址
  905. * @return
  906. */
  907. public boolean downFromS3(String objectName, String localPath) {
  908. BasicAWSCredentials awsCreds = new BasicAWSCredentials(s3key, s3secrey);
  909. AmazonS3 s3 = AmazonS3ClientBuilder.standard()
  910. .withCredentials(new AWSStaticCredentialsProvider(awsCreds))
  911. .withRegion(Regions.EU_WEST_2)
  912. .build();
  913. try {
  914. GetObjectRequest request = new GetObjectRequest(this.bucket,objectName);
  915. s3.getObject(request,new File(localPath));
  916. return true;
  917. } catch (Exception e) {
  918. log.error("amazonS3下载文件失败,key=" + objectName, e);
  919. s3.shutdown();
  920. }
  921. return false;
  922. }
  923. }