UploadToOssUtil.java 28 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034
  1. package com.fdkankan.fyun.oss;
  2. import cn.hutool.core.collection.CollUtil;
  3. import cn.hutool.core.io.FileUtil;
  4. import cn.hutool.core.util.StrUtil;
  5. import com.alibaba.fastjson.JSON;
  6. import com.aliyun.oss.OSS;
  7. import com.aliyun.oss.OSSClient;
  8. import com.aliyun.oss.model.*;
  9. import com.amazonaws.HttpMethod;
  10. import com.amazonaws.auth.AWSStaticCredentialsProvider;
  11. import com.amazonaws.auth.BasicAWSCredentials;
  12. import com.amazonaws.regions.Regions;
  13. import com.amazonaws.services.s3.AmazonS3;
  14. import com.amazonaws.services.s3.AmazonS3ClientBuilder;
  15. import com.amazonaws.services.s3.model.CannedAccessControlList;
  16. import com.amazonaws.services.s3.model.GeneratePresignedUrlRequest;
  17. import com.amazonaws.services.s3.model.GetObjectRequest;
  18. import com.amazonaws.services.s3.model.PutObjectRequest;
  19. import com.amazonaws.services.s3.model.S3Object;
  20. import com.amazonaws.services.s3.model.S3ObjectInputStream;
  21. import com.amazonaws.services.s3.model.S3ObjectSummary;
  22. import com.fdkankan.fyun.constant.StorageType;
  23. import com.qiniu.common.Zone;
  24. import com.qiniu.storage.Configuration;
  25. import com.qiniu.storage.UploadManager;
  26. import java.util.stream.Collectors;
  27. import lombok.extern.slf4j.Slf4j;
  28. import org.apache.commons.fileupload.FileItem;
  29. import org.apache.commons.fileupload.FileItemFactory;
  30. import org.apache.commons.fileupload.disk.DiskFileItemFactory;
  31. import org.apache.commons.io.FileUtils;
  32. import org.apache.commons.lang3.StringUtils;
  33. import org.springframework.beans.factory.annotation.Value;
  34. import org.springframework.stereotype.Component;
  35. import org.springframework.web.multipart.MultipartFile;
  36. import org.springframework.web.multipart.commons.CommonsMultipartFile;
  37. import java.io.*;
  38. import java.net.FileNameMap;
  39. import java.net.URL;
  40. import java.net.URLConnection;
  41. import java.util.ArrayList;
  42. import java.util.HashMap;
  43. import java.util.List;
  44. import java.util.Map;
  45. @Slf4j
  46. @Component
  47. public class UploadToOssUtil {
  48. Zone zone = Zone.autoZone();
  49. Configuration config = new Configuration(zone);
  50. UploadManager uploadManager = new UploadManager(config);
  51. @Value("${oss.point:http://oss-cn-shenzhen-internal.aliyuncs.com}")
  52. private String point;
  53. @Value("${oss.key:LTAIUrvuHqj8pvry}")
  54. private String key;
  55. @Value("${oss.secrey:JLOVl0k8Ke0aaM8nLMMiUAZ3EiiqI4}")
  56. private String secrey;
  57. @Value("${oss.bucket:4dkankan}")
  58. private String bucket;
  59. @Value("${oss.sdk:4dscene}")
  60. private String bucketSdk;
  61. @Value("${upload.type:oss}")
  62. private String type;
  63. @Value("${aws.s3key:AKIAWCV5QFZ3ZNELKYUY}")
  64. private String s3key;
  65. @Value("${aws.s3secrey:epS5ghyR4LJ7rxk/qJO9ZYh6m9Oz6g5haKDu4yws}")
  66. private String s3secrey;
  67. @Value("${aws.s3bucket:4dkankan}")
  68. private String s3bucket;
  69. @Value("${local.path:/home/4dkankan}")
  70. private String localPath;
  71. //上传的数据是byte[],key是上传后的文件名
  72. public void upload(byte[] data,String key1) throws IOException{
  73. log.info("开始上传文件 源路径:{},目标路径:{},type:{}" , new String(data, "UTF-8"),key1,type);
  74. StorageType storageType = StorageType.get(type);
  75. switch (storageType){
  76. case OSS:
  77. uploadOss(data,key1);
  78. break;
  79. case AWS:
  80. uploadAws(data,key1);
  81. break;
  82. case LOCAL:
  83. uploadLocal(data,key1);
  84. break;
  85. }
  86. }
  87. public void upload(String filePath, String key1) {
  88. log.info("开始上传文件 源路径:{},目标路径:{},type:{}" , filePath,key1,type);
  89. StorageType storageType = StorageType.get(type);
  90. switch (storageType){
  91. case OSS:
  92. uploadOss(filePath,key1);
  93. break;
  94. case AWS:
  95. uploadAws(filePath,key1);
  96. break;
  97. case LOCAL:
  98. uploadLocal(filePath,key1);
  99. break;
  100. }
  101. }
  102. public void uploadSdk(String filePath, String key1) {
  103. log.info("开始上传文件 源路径:{},目标路径:{},type:{}" , filePath,key1,type);
  104. switch (type){
  105. case "oss":uploadSdkOss(filePath,key1); break;
  106. case "aws": uploadAws(filePath,key1); break;
  107. case "local":uploadLocal(filePath,key1); break;
  108. }
  109. }
  110. public void upload2(String filePath, String key1) {
  111. log.info("开始上传文件 源路径:{},目标路径:{},type:{}" , filePath,key1,type);
  112. switch (type){
  113. case "oss":upload2Oss(filePath,key1); break;
  114. case "aws": uploadAws(filePath,key1); break;
  115. case "local":uploadLocal(filePath,key1); break;
  116. }
  117. }
  118. public void delete(String key1) throws IOException{
  119. switch (type){
  120. case "oss":deleteOss(key1); break;
  121. case "aws": deleteS3Object(key1); break;
  122. case "local":FileUtil.del(key1); break;
  123. }
  124. }
  125. public int deleteFile(String prefix){
  126. switch (type){
  127. case "oss":deleteOssFile(prefix); break;
  128. case "aws": deleteS3Object(prefix); break;
  129. case "local":FileUtil.del(prefix); break;
  130. }
  131. return 1;
  132. }
  133. public void deleteOss(String key1){
  134. OSSClient ossClient = new OSSClient(point, key, secrey);
  135. try {
  136. ossClient.deleteObject(bucket, key1);
  137. } catch (Exception e) {
  138. e.printStackTrace();
  139. }
  140. }
  141. public void deleteOssFile(String prefix){
  142. OSSClient ossClient = new OSSClient(point, key, secrey);
  143. ObjectListing objectListing = ossClient.listObjects(bucket, prefix);
  144. List<OSSObjectSummary> sums = objectListing.getObjectSummaries();
  145. try {
  146. for (OSSObjectSummary s : sums) {
  147. delete(s.getKey());
  148. }
  149. } catch (IOException e) {
  150. e.printStackTrace();
  151. }
  152. }
  153. public void uploadOss(byte[] data,String key1){
  154. OSSClient ossClient = new OSSClient(point, key, secrey);
  155. try {
  156. ossClient.putObject(bucket, key1, new ByteArrayInputStream(data));
  157. } catch (Exception e) {
  158. log.error(e.toString()+key1);
  159. }
  160. }
  161. public void uploadAws(byte[] data,String key1){
  162. }
  163. public void uploadLocal(byte[] data,String key1){
  164. InputStream in = new ByteArrayInputStream(data);
  165. File file = new File(key1);
  166. String path = key1.substring(0, key1.lastIndexOf("/"));
  167. if (!file.exists()) {
  168. new File(path).mkdir();
  169. }
  170. FileOutputStream fos = null;
  171. try {
  172. fos = new FileOutputStream(file);
  173. int len = 0;
  174. byte[] buf = new byte[1024];
  175. while ((len = in.read(buf)) != -1) {
  176. fos.write(buf, 0, len);
  177. }
  178. fos.flush();
  179. } catch (Exception e) {
  180. e.printStackTrace();
  181. } finally {
  182. if (null != fos) {
  183. try {
  184. fos.close();
  185. } catch (IOException e) {
  186. e.printStackTrace();
  187. }
  188. }
  189. }
  190. }
  191. public void uploadOss(String filePath, String key1){
  192. OSSClient ossClient = new OSSClient(point, key, secrey);
  193. try {
  194. File file = new File(filePath);
  195. if (!file.exists()) {
  196. log.error("要上传的文件不存在:" + filePath);
  197. return;
  198. }
  199. ObjectMetadata metadata = new ObjectMetadata();
  200. if(filePath.contains(".jpg")){
  201. metadata.setContentType("image/jpeg");
  202. }
  203. ossClient.putObject(bucket, key1, new File(filePath), metadata);
  204. } catch (Exception e) {
  205. log.error(e.toString() + filePath);
  206. } finally {
  207. ossClient.shutdown();
  208. }
  209. }
  210. public void uploadAws(String filePath, String key1){
  211. try{
  212. uploadS3File(filePath, key1);
  213. }catch (Exception e){
  214. e.printStackTrace();
  215. }
  216. }
  217. public void uploadLocal(String filePath, String key1){
  218. try {
  219. File srcFile = new File(filePath);
  220. File file = new File(localPath + key1);
  221. FileUtils.copyFile(srcFile,file);
  222. }catch (Exception e){
  223. e.printStackTrace();
  224. }
  225. }
  226. public void uploadSdkOss(String filePath, String key1){
  227. OSSClient ossClient = new OSSClient(point, key, secrey);
  228. try {
  229. File file = new File(filePath);
  230. if (!file.exists()) {
  231. log.error("要上传的文件不存在:" + filePath);
  232. return;
  233. }
  234. ObjectMetadata metadata = new ObjectMetadata();
  235. if(filePath.contains(".jpg")){
  236. metadata.setContentType("image/jpeg");
  237. }
  238. ossClient.putObject(bucketSdk, key1, new File(filePath), metadata);
  239. } catch (Exception e) {
  240. log.error(e.toString() + filePath);
  241. }
  242. }
  243. public void upload2Oss(String filePath, String key1){
  244. OSSClient ossClient = new OSSClient(point, key, secrey);
  245. try {
  246. ObjectMetadata metadata = new ObjectMetadata();
  247. if(filePath.contains(".jpg")){
  248. metadata.setContentType("image/jpeg");
  249. }
  250. if(filePath.contains(".mp4")){
  251. metadata.setContentType("video/mp4");
  252. }
  253. if(filePath.contains(".mp3")){
  254. metadata.setContentType("audio/mp3");
  255. }
  256. ossClient.putObject(bucket, key1, new File(filePath), metadata);
  257. } catch (Exception e) {
  258. log.error(e.toString() + filePath);
  259. }
  260. }
  261. //上传的数据是文件夹,参数是文件夹路径,key是上传后的文件名
  262. public void uploadMulFiles(Map<String, String> filepaths) {
  263. if (filepaths == null) {
  264. return;
  265. }
  266. Long start = System.currentTimeMillis();
  267. log.info("开始批量上传文件:");
  268. if (filepaths.size() > 50) {
  269. filepaths.entrySet().parallelStream().forEach(entry->{
  270. upload2(entry.getKey(), entry.getValue());
  271. });
  272. } else {
  273. filepaths.entrySet().parallelStream().forEach(entry->{
  274. upload(entry.getKey(), entry.getValue());
  275. });
  276. }
  277. log.info("批量上传文件结束,用时:{}" ,(System.currentTimeMillis() - start));
  278. }
  279. public Map<String, String> getUploadS3Url(List<String> urls){
  280. if(urls == null || urls.size() <= 0){
  281. return null;
  282. }
  283. BasicAWSCredentials awsCred = new BasicAWSCredentials(s3key, s3secrey);
  284. AmazonS3 s3Client = AmazonS3ClientBuilder.standard()
  285. .withCredentials(new AWSStaticCredentialsProvider(awsCred))
  286. .withRegion(Regions.EU_WEST_2)
  287. .build();
  288. // Set the pre-signed URL to expire after one hour.
  289. java.util.Date expiration = new java.util.Date();
  290. long expTimeMillis = expiration.getTime();
  291. expTimeMillis += 1000 * 60 * 60 * 8;
  292. expiration.setTime(expTimeMillis);
  293. //生成预签名URL
  294. log.info("生成预签名URL");
  295. GeneratePresignedUrlRequest generatePresignedUrlRequest = null;
  296. URL url = null;
  297. Map<String, String> map = new HashMap();
  298. for(String path : urls){
  299. // if(path.contains(".jpg") || path.contains("png")){
  300. // generatePresignedUrlRequest = new GeneratePresignedUrlRequest(s3bucket, path)
  301. // .withMethod(HttpMethod.PUT)
  302. // .withExpiration(expiration)
  303. // .withContentType("image/jpeg");
  304. // }else {
  305. generatePresignedUrlRequest = new GeneratePresignedUrlRequest(s3bucket, path)
  306. .withMethod(HttpMethod.PUT)
  307. .withExpiration(expiration);
  308. // }
  309. url = s3Client.generatePresignedUrl(generatePresignedUrlRequest);
  310. map.put(path, url.toString());
  311. }
  312. return map;
  313. }
  314. public String upload5(String filePath, String key1) {
  315. OSSClient ossClient = new OSSClient(point, key, secrey);
  316. PutObjectResult result = null;
  317. try {
  318. File file = new File(filePath);
  319. if (!file.exists()) {
  320. log.error("要上传的文件不存在:" + filePath);
  321. }
  322. result = ossClient.putObject(bucket, key1, new File(filePath));
  323. } catch (Exception e) {
  324. log.error(e.toString() + filePath);
  325. }
  326. log.info(" getETag : " + result.getETag());
  327. log.info("1 : " + result.toString());
  328. log.info("2 : " + result.getRequestId());
  329. log.info("3 : " + result.getClientCRC());
  330. log.info("4 : " + result.getResponse());
  331. log.info("5 : " + result.getServerCRC());
  332. return result.getETag();
  333. }
  334. //海外亚马逊s3
  335. /**
  336. * s3上传文件流
  337. *
  338. * @param file 文件
  339. * @param updatePath 上传路径[ eg: xxx/xxx ]
  340. */
  341. public String updateS3LoadFile(MultipartFile file, String updatePath) {
  342. if (isEmpty(file)) {
  343. return null;
  344. }
  345. /**
  346. * 创建s3对象
  347. */
  348. BasicAWSCredentials awsCreds = new BasicAWSCredentials(s3key, s3secrey);
  349. AmazonS3 s3 = AmazonS3ClientBuilder.standard()
  350. .withCredentials(new AWSStaticCredentialsProvider(awsCreds))
  351. .withRegion(Regions.EU_WEST_2)
  352. .build();
  353. try {
  354. // 创建临时文件,程序运行结束,会自动删除
  355. File localFile = File.createTempFile("temp", null);
  356. // 把文件写入内存中
  357. file.transferTo(localFile);
  358. // 指定要上传到服务器上的路径
  359. String key = updatePath;
  360. // 设置文件并设置公读
  361. PutObjectRequest request = new PutObjectRequest(s3bucket, key, localFile);
  362. request.withCannedAcl(CannedAccessControlList.PublicRead);
  363. // 上传文件
  364. com.amazonaws.services.s3.model.PutObjectResult putObjectResult = s3.putObject(request);
  365. if (StringUtils.isNotEmpty(putObjectResult.getETag())) {
  366. System.out.println("success");
  367. return key;
  368. }
  369. return null;
  370. } catch (IOException e) {
  371. }
  372. return null;
  373. }
  374. /**
  375. * s3上传文件
  376. * @param filePath
  377. * @param key1
  378. * @throws IOException
  379. */
  380. private void uploadS3File(String filePath, String key1) throws Exception {
  381. /**
  382. * 创建s3对象
  383. */
  384. BasicAWSCredentials awsCreds = new BasicAWSCredentials(s3key, s3secrey);
  385. AmazonS3 s3 = AmazonS3ClientBuilder.standard()
  386. .withCredentials(new AWSStaticCredentialsProvider(awsCreds))
  387. .withRegion(Regions.EU_WEST_2)
  388. .build();
  389. try{
  390. File file = new File(filePath);
  391. if(!file.exists()){
  392. log.info("要上传s3的文件不存在");
  393. return;
  394. }
  395. // 设置文件并设置公读
  396. com.amazonaws.services.s3.model.ObjectMetadata metadata = new com.amazonaws.services.s3.model.ObjectMetadata();
  397. if(filePath.contains(".jpg")){
  398. metadata.setContentType("image/jpeg");
  399. }
  400. if(filePath.contains(".png")){
  401. metadata.setContentType("image/png");
  402. }
  403. PutObjectRequest request = new PutObjectRequest(s3bucket, key1, file);
  404. request.withCannedAcl(CannedAccessControlList.PublicRead);
  405. request.withMetadata(metadata);
  406. // 上传文件
  407. com.amazonaws.services.s3.model.PutObjectResult putObjectResult = s3.putObject(request);
  408. if (StringUtils.isNotEmpty(putObjectResult.getETag())) {
  409. log.info("s3上传文件成功:" + key1);
  410. }
  411. }catch (Exception e){
  412. throw e;
  413. }finally {
  414. s3.shutdown();
  415. }
  416. }
  417. /**
  418. * 删除单个文件
  419. *
  420. * @param filePath 文件路径[ eg: /head/xxxx.jpg ]
  421. * @return
  422. */
  423. public void deleteS3Object(String filePath) {
  424. /**
  425. * 创建s3对象
  426. */
  427. BasicAWSCredentials awsCreds = new BasicAWSCredentials(s3key, s3secrey);
  428. AmazonS3 s3 = AmazonS3ClientBuilder.standard()
  429. .withCredentials(new AWSStaticCredentialsProvider(awsCreds))
  430. .withRegion(Regions.EU_WEST_2)
  431. .build();
  432. if (filePath.startsWith("/")) {
  433. filePath = filePath.substring(1);
  434. }
  435. try {
  436. s3.deleteObject(s3bucket, filePath);
  437. } catch (Exception e) {
  438. }
  439. }
  440. /**
  441. * 获取文件类型
  442. */
  443. public static String getContentType(String filePath){
  444. FileNameMap fileNameMap = URLConnection.getFileNameMap();
  445. String contentType = fileNameMap.getContentTypeFor(filePath);
  446. System.out.println(contentType);
  447. return contentType;
  448. }
  449. /**
  450. * 检查文件是否为空
  451. *
  452. * @param imageFile
  453. * @return
  454. */
  455. private static boolean isEmpty(MultipartFile imageFile) {
  456. if (imageFile == null || imageFile.getSize() <= 0) {
  457. return true;
  458. }
  459. return false;
  460. }
  461. private static MultipartFile getMulFileByPath(String picPath) {
  462. FileItem fileItem = createFileItem(picPath);
  463. MultipartFile mfile = new CommonsMultipartFile(fileItem);
  464. return mfile;
  465. }
  466. private static FileItem createFileItem(String filePath) {
  467. FileItemFactory factory = new DiskFileItemFactory(16, null);
  468. String textFieldName = "textField";
  469. int num = filePath.lastIndexOf(".");
  470. String extFile = filePath.substring(num);
  471. FileItem item = factory.createItem(textFieldName, "text/plain", true,
  472. "MyFileName" + extFile);
  473. File newfile = new File(filePath);
  474. int bytesRead = 0;
  475. byte[] buffer = new byte[8192];
  476. try
  477. {
  478. FileInputStream fis = new FileInputStream(newfile);
  479. OutputStream os = item.getOutputStream();
  480. while ((bytesRead = fis.read(buffer, 0, 8192))
  481. != -1)
  482. {
  483. os.write(buffer, 0, bytesRead);
  484. }
  485. os.close();
  486. fis.close();
  487. }
  488. catch (IOException e)
  489. {
  490. e.printStackTrace();
  491. }
  492. return item;
  493. }
  494. public List<String> listKeys(String sourcePath){
  495. StorageType storageType = StorageType.get(type);
  496. switch (storageType){
  497. case OSS:
  498. return this.listKeysFromAli(sourcePath);
  499. case AWS:
  500. return this.listKeysFromAws(sourcePath);
  501. case LOCAL:
  502. return this.listKeysFromLocal(sourcePath);
  503. }
  504. return null;
  505. }
  506. /**
  507. * 获得文件列表-阿里云
  508. * @return
  509. */
  510. public List<String> listKeysFromAli(String sourcePath) {
  511. List<String> keyList = new ArrayList<>();
  512. OSSClient ossClient = new OSSClient(point, key, secrey);
  513. boolean flag = true;
  514. String nextMaker = null;
  515. ListObjectsRequest listObjectsRequest = new ListObjectsRequest(this.bucket);
  516. //指定下一级文件
  517. listObjectsRequest.setPrefix(sourcePath);
  518. //设置分页的页容量
  519. listObjectsRequest.setMaxKeys(200);
  520. do
  521. {
  522. //获取下一页的起始点,它的下一项
  523. listObjectsRequest.setMarker(nextMaker);
  524. ObjectListing objectListing = ossClient.listObjects(listObjectsRequest);
  525. List<OSSObjectSummary> objectSummaries = objectListing.getObjectSummaries();
  526. List<String> collect = objectSummaries.stream().map(summary -> {
  527. return summary.getKey();
  528. }).collect(Collectors.toList());
  529. if(CollUtil.isNotEmpty(collect)){
  530. keyList.addAll(collect);
  531. }
  532. nextMaker = objectListing.getNextMarker();
  533. //全部执行完后,为false
  534. flag = objectListing.isTruncated();
  535. } while (flag);
  536. ossClient.shutdown();
  537. return keyList;
  538. }
  539. /**
  540. * 获得文件列表-亚马逊
  541. * @return
  542. */
  543. public List<String> listKeysFromAws(String sourcePath) {
  544. List<String> keyList = new ArrayList<>();
  545. BasicAWSCredentials awsCreds = new BasicAWSCredentials(s3key, s3secrey);
  546. AmazonS3 s3 = AmazonS3ClientBuilder.standard()
  547. .withCredentials(new AWSStaticCredentialsProvider(awsCreds))
  548. .withRegion(Regions.EU_WEST_2)
  549. .build();
  550. boolean flag = true;
  551. String nextMaker = null;
  552. com.amazonaws.services.s3.model.ListObjectsRequest listObjectsRequest = new com.amazonaws.services.s3.model.ListObjectsRequest();
  553. listObjectsRequest.setBucketName(this.bucket);
  554. listObjectsRequest.setPrefix(sourcePath);
  555. listObjectsRequest.setMaxKeys(200);
  556. do{
  557. listObjectsRequest.setMarker(nextMaker);
  558. com.amazonaws.services.s3.model.ObjectListing objectListing = s3.listObjects(listObjectsRequest);
  559. List<S3ObjectSummary> objectSummaries = objectListing.getObjectSummaries();
  560. List<String> collect =objectSummaries.stream().map(summary->{
  561. return summary.getKey();
  562. }).collect(Collectors.toList());
  563. if(CollUtil.isNotEmpty(collect)){
  564. keyList.addAll(collect);
  565. }
  566. nextMaker = objectListing.getNextMarker();
  567. flag = objectListing.isTruncated();
  568. }while (flag);
  569. s3.shutdown();
  570. return keyList;
  571. }
  572. /**
  573. * 获得文件列表-阿里云
  574. * @return
  575. */
  576. public List<String> listKeysFromLocal(String sourcePath) {
  577. List<String> keyList = new ArrayList<>();
  578. return keyList;
  579. }
  580. /**
  581. * <p>
  582. 拷贝目录
  583. * </p>
  584. * @author dengsixing
  585. * @date 2022/1/18
  586. * @param sourcePath
  587. * @param targetPath
  588. **/
  589. public void copyFiles(String sourcePath, String targetPath) throws IOException {
  590. StorageType storageType = StorageType.get(type);
  591. switch (storageType){
  592. case OSS:
  593. this.copyFilesFromAli(sourcePath, targetPath);
  594. break;
  595. case AWS:
  596. this.copyFilesFromAws(sourcePath, targetPath);
  597. break;
  598. case LOCAL: this.copyFilesFromLocal(sourcePath, targetPath);
  599. }
  600. }
  601. /**
  602. * <p>
  603. 拷贝文件
  604. * </p>
  605. * @author dengsixing
  606. * @date 2022/1/18
  607. * @param sourceKey
  608. * @param targetKey
  609. **/
  610. public void copyObject(String sourceKey, String targetKey) throws IOException {
  611. StorageType storageType = StorageType.get(type);
  612. switch (storageType){
  613. case OSS:
  614. this.copyObjectFromAli(sourceKey, targetKey);
  615. break;
  616. case AWS:
  617. this.copyObjectFromAws(sourceKey, targetKey);
  618. break;
  619. }
  620. }
  621. /**
  622. * <p>
  623. 拷贝-阿里云
  624. * </p>
  625. * @author dengsixing
  626. * @date 2022/1/18
  627. * @param sourcePath
  628. * @param targetPath
  629. **/
  630. public void copyObjectFromAli(String sourcePath, String targetPath) throws IOException {
  631. // 创建OSSClient实例。
  632. OSSClient ossClient = new OSSClient(point, key, secrey);
  633. // 复制文件
  634. log.info("开始复制:" + sourcePath);
  635. ossClient.copyObject(this.bucket, sourcePath, this.bucket, targetPath);
  636. log.info("复制成功:" + sourcePath);
  637. ossClient.shutdown();
  638. }
  639. /**
  640. * <p>
  641. 拷贝-阿里云
  642. * </p>
  643. * @author dengsixing
  644. * @date 2022/1/18
  645. * @param sourcePath
  646. * @param targetPath
  647. **/
  648. public void copyFilesFromAli(String sourcePath, String targetPath) throws IOException {
  649. //获取源文件列表
  650. List<String> sourceKeyList = this.listKeysFromAli(sourcePath);
  651. if(CollUtil.isEmpty(sourceKeyList)){
  652. return;
  653. }
  654. // 创建OSSClient实例。
  655. OSSClient ossClient = new OSSClient(point, key, secrey);
  656. // 复制文件
  657. sourceKeyList.parallelStream().forEach(key -> {
  658. log.info("开始复制:" + key);
  659. ossClient.copyObject(this.bucket, key, this.bucket, key.replace(sourcePath, targetPath));
  660. log.info("复制成功:" + key);
  661. });
  662. ossClient.shutdown();
  663. }
  664. /**
  665. * <p>
  666. 拷贝-亚马逊
  667. * </p>
  668. * @author dengsixing
  669. * @date 2022/1/18
  670. * @param sourcePath
  671. * @param targetPath
  672. **/
  673. public void copyFilesFromAws(String sourcePath, String targetPath){
  674. try {
  675. List<String> sourceKeyList = this.listKeysFromAws(sourcePath);
  676. /**
  677. * 创建s3对象
  678. */
  679. BasicAWSCredentials awsCreds = new BasicAWSCredentials(s3key, s3secrey);
  680. AmazonS3 s3 = AmazonS3ClientBuilder.standard()
  681. .withCredentials(new AWSStaticCredentialsProvider(awsCreds))
  682. .withRegion(Regions.EU_WEST_2)//s3 地区位置
  683. .build();
  684. // 复制文件
  685. sourceKeyList.parallelStream().forEach(key -> {
  686. log.info("开始复制:" + key);
  687. s3.copyObject(this.bucket, key, this.bucket, key.replace(sourcePath, targetPath));
  688. log.info("复制成功:" + key);
  689. });
  690. s3.shutdown();
  691. } catch (Exception ase) {
  692. log.error("amazonS拷贝异常 " + ase.getMessage(), ase);
  693. }
  694. }
  695. /**
  696. * <p>
  697. 拷贝-亚马逊
  698. * </p>
  699. * @author dengsixing
  700. * @date 2022/1/18
  701. * @param sourceKey
  702. * @param targetKey
  703. **/
  704. public void copyObjectFromAws(String sourceKey, String targetKey){
  705. BasicAWSCredentials awsCreds = new BasicAWSCredentials(s3key, s3secrey);
  706. AmazonS3 s3 = AmazonS3ClientBuilder.standard()
  707. .withCredentials(new AWSStaticCredentialsProvider(awsCreds))
  708. .withRegion(Regions.EU_WEST_2)//s3 地区位置
  709. .build();
  710. // 复制文件
  711. log.info("开始复制:" + sourceKey);
  712. s3.copyObject(this.bucket, sourceKey, this.bucket, targetKey);
  713. log.info("复制成功:" + sourceKey);
  714. s3.shutdown();
  715. }
  716. /**
  717. * <p>
  718. 拷贝-本地
  719. * </p>
  720. * @author dengsixing
  721. * @date 2022/1/18
  722. * @param sourcePath
  723. * @param targetPath
  724. **/
  725. public void copyFilesFromLocal(String sourcePath, String targetPath) throws IOException {
  726. // TODO: 2022/1/21
  727. }
  728. /**
  729. * 获取文件内容
  730. * @param bucketName
  731. * @param objectName
  732. * @return
  733. */
  734. public String getObjectContent(String bucketName, String objectName){
  735. StorageType storageType = StorageType.get(type);
  736. switch (storageType){
  737. case OSS:
  738. return this.getObjectContentFromAli(bucketName, objectName);
  739. case AWS:
  740. return this.getObjectContentFromAws(bucketName, objectName);
  741. case LOCAL:
  742. return this.getObjectContentFromLocal(objectName);
  743. }
  744. return null;
  745. }
  746. /**
  747. * 获取文件内容-阿里云
  748. * @param bucketName
  749. * @param objectName
  750. * @return
  751. */
  752. public String getObjectContentFromAli(String bucketName, String objectName){
  753. //创建oss客户端
  754. OSSClient ossClient = new OSSClient(point, key, secrey);
  755. InputStream objectContent = null;
  756. StringBuilder contentJson = new StringBuilder();
  757. try {
  758. // ossObject包含文件所在的存储空间名称、文件名称、文件元信息以及一个输入流。
  759. OSSObject ossObject = ossClient.getObject(bucketName, objectName);
  760. objectContent = ossObject.getObjectContent();
  761. try(BufferedReader reader = new BufferedReader(new InputStreamReader(objectContent))){
  762. while (true) {
  763. String line = reader.readLine();
  764. if (line == null) break;
  765. contentJson.append(line);
  766. }
  767. } catch (IOException e) {
  768. log.error("读取scene.json文件流失败", e);
  769. }
  770. ossClient.shutdown();
  771. }catch (Exception e){
  772. log.error("oos找不到文件,文件路径:{}", objectName);
  773. }
  774. return contentJson.toString();
  775. }
  776. /**
  777. * 获取文件内容-阿里云
  778. * @param objectName
  779. * @return
  780. */
  781. public boolean existOnAli(String objectName){
  782. //创建oss客户端
  783. OSSClient ossClient = new OSSClient(point, key, secrey);
  784. // ossObject包含文件所在的存储空间名称、文件名称、文件元信息以及一个输入流。
  785. OSSObject ossObject = ossClient.getObject(bucket, objectName);
  786. String key = ossObject.getKey();
  787. if(StrUtil.isNotEmpty(key))
  788. return true;
  789. return false;
  790. }
  791. /**
  792. * 获取文件内容-亚马逊
  793. * @param bucketName
  794. * @param objectName
  795. * @return
  796. */
  797. public String getObjectContentFromAws(String bucketName, String objectName){
  798. try {
  799. /**
  800. * 创建s3对象
  801. */
  802. BasicAWSCredentials awsCreds = new BasicAWSCredentials(s3key, s3secrey);
  803. AmazonS3 s3 = AmazonS3ClientBuilder.standard()
  804. .withCredentials(new AWSStaticCredentialsProvider(awsCreds))
  805. .withRegion(Regions.EU_WEST_2)
  806. .build();
  807. GetObjectRequest request = new GetObjectRequest(bucketName,objectName);
  808. S3Object object = s3.getObject(request);
  809. S3ObjectInputStream inputStream = object.getObjectContent();
  810. StringBuilder content = new StringBuilder();
  811. try(BufferedReader reader = new BufferedReader(new InputStreamReader(inputStream))){
  812. while (true) {
  813. String line = reader.readLine();
  814. if (line == null) break;
  815. content.append(line);
  816. }
  817. } catch (IOException e) {
  818. log.error("读取aws文件流失败", e);
  819. }
  820. return content.toString();
  821. } catch (Exception ase) {
  822. log.error("amazonS3下载文件异常 " + ase.getMessage(), ase);
  823. }
  824. return null;
  825. }
  826. /**
  827. * 获取文件内容-亚马逊
  828. * @param objectName
  829. * @return
  830. */
  831. public boolean existOnAws(String objectName){
  832. BasicAWSCredentials awsCreds = new BasicAWSCredentials(s3key, s3secrey);
  833. AmazonS3 s3 = AmazonS3ClientBuilder.standard()
  834. .withCredentials(new AWSStaticCredentialsProvider(awsCreds))
  835. .withRegion(Regions.EU_WEST_2)
  836. .build();
  837. GetObjectRequest request = new GetObjectRequest(bucket,objectName);
  838. S3Object s3Object = s3.getObject(request);
  839. String key = s3Object.getKey();
  840. if(StrUtil.isNotEmpty(key))
  841. return true;
  842. return false;
  843. }
  844. /**
  845. * 判断key是否存在
  846. * @param key
  847. * @return
  848. */
  849. public boolean existKey(String key){
  850. StorageType storageType = StorageType.get(type);
  851. switch (storageType){
  852. case OSS:
  853. return this.existOnAli(key);
  854. case AWS:
  855. return this.existOnAws(key);
  856. default:
  857. return false;
  858. }
  859. }
  860. /**
  861. * 获取文件内容-本地
  862. * @param objectName
  863. * @return
  864. */
  865. public String getObjectContentFromLocal(String objectName){
  866. // TODO: 2022/1/21
  867. return null;
  868. }
  869. /**
  870. * oss下载文件到本地
  871. * @param objectName
  872. * @param localPath
  873. */
  874. public boolean download(String objectName, String localPath){
  875. StorageType storageType = StorageType.get(this.type);
  876. switch (storageType){
  877. case OSS:
  878. return this.downFormAli(objectName, localPath);
  879. case AWS:
  880. return this.downFromS3(objectName, localPath);
  881. }
  882. return false;
  883. }
  884. /**
  885. * 从阿里云oss下载文件到本地
  886. * @param objectName 云端文件k地址
  887. * @param localPath 本地文件地址
  888. * @return
  889. */
  890. public boolean downFormAli(String objectName, String localPath){
  891. OSSClient ossClient = new OSSClient(point, key, secrey);
  892. try {
  893. com.aliyun.oss.model.GetObjectRequest request = new com.aliyun.oss.model.GetObjectRequest(bucket,objectName);
  894. ossClient.getObject(request, new File(localPath));
  895. return true;
  896. }catch (Exception e){
  897. log.error("阿里云oss文件下载失败,key=" + objectName, e);
  898. }finally {
  899. ossClient.shutdown();
  900. }
  901. return false;
  902. }
  903. /**
  904. * 从s3下载文件到本地
  905. * @param objectName 云端文件k地址
  906. * @param localPath 本地文件地址
  907. * @return
  908. */
  909. public boolean downFromS3(String objectName, String localPath) {
  910. BasicAWSCredentials awsCreds = new BasicAWSCredentials(s3key, s3secrey);
  911. AmazonS3 s3 = AmazonS3ClientBuilder.standard()
  912. .withCredentials(new AWSStaticCredentialsProvider(awsCreds))
  913. .withRegion(Regions.EU_WEST_2)
  914. .build();
  915. try {
  916. GetObjectRequest request = new GetObjectRequest(this.bucket,objectName);
  917. s3.getObject(request,new File(localPath));
  918. return true;
  919. } catch (Exception e) {
  920. log.error("amazonS3下载文件失败,key=" + objectName, e);
  921. }finally {
  922. s3.shutdown();
  923. }
  924. return false;
  925. }
  926. }