|
@@ -1,971 +0,0 @@
|
|
|
-package com.fdkankan.tools.utils;
|
|
|
-import cn.hutool.core.collection.CollUtil;
|
|
|
-import cn.hutool.core.io.FileUtil;
|
|
|
-import com.aliyun.oss.OSSClient;
|
|
|
-import com.aliyun.oss.model.ListObjectsRequest;
|
|
|
-import com.aliyun.oss.model.ObjectListing;
|
|
|
-import com.aliyun.oss.model.ObjectMetadata;
|
|
|
-import com.aliyun.oss.model.*;
|
|
|
-import com.amazonaws.auth.AWSStaticCredentialsProvider;
|
|
|
-import com.amazonaws.auth.BasicAWSCredentials;
|
|
|
-import com.amazonaws.regions.Regions;
|
|
|
-import com.amazonaws.services.s3.AmazonS3;
|
|
|
-import com.amazonaws.services.s3.AmazonS3ClientBuilder;
|
|
|
-import com.amazonaws.services.s3.model.CannedAccessControlList;
|
|
|
-import com.amazonaws.services.s3.model.GetObjectRequest;
|
|
|
-import com.amazonaws.services.s3.model.PutObjectRequest;
|
|
|
-import com.amazonaws.services.s3.model.*;
|
|
|
-import lombok.extern.slf4j.Slf4j;
|
|
|
-import org.apache.commons.io.FileUtils;
|
|
|
-import org.apache.commons.lang3.StringUtils;
|
|
|
-import org.springframework.beans.factory.annotation.Value;
|
|
|
-import org.springframework.stereotype.Component;
|
|
|
-import org.springframework.util.CollectionUtils;
|
|
|
-
|
|
|
-import java.io.*;
|
|
|
-import java.net.FileNameMap;
|
|
|
-import java.net.URLConnection;
|
|
|
-import java.util.ArrayList;
|
|
|
-import java.util.List;
|
|
|
-import java.util.stream.Collectors;
|
|
|
-
|
|
|
-@Slf4j
|
|
|
-@Component
|
|
|
-public class UploadToOss {
|
|
|
-
|
|
|
-
|
|
|
- @Value("${fyun.oss.point:http://oss-cn-shenzhen-internal.aliyuncs.com}")
|
|
|
- private String point;
|
|
|
-
|
|
|
- @Value("${fyun.oss.key:LTAIUrvuHqj8pvry}")
|
|
|
- private String key;
|
|
|
-
|
|
|
- @Value("${fyun.oss.secrey:JLOVl0k8Ke0aaM8nLMMiUAZ3EiiqI4}")
|
|
|
- private String secrey;
|
|
|
-
|
|
|
- @Value("${fyun.oss.bucket:4dkankan}")
|
|
|
- private String bucket;
|
|
|
-
|
|
|
- @Value("${upload.type:oss}")
|
|
|
- private String type;
|
|
|
-
|
|
|
- @Value("${fdkk.aws.s3key:AKIAWCV5QFZ3ZNELKYUY}")
|
|
|
- private String s3key;
|
|
|
-
|
|
|
- @Value("${fdkk.aws.s3secrey:epS5ghyR4LJ7rxk/qJO9ZYh6m9Oz6g5haKDu4yws}")
|
|
|
- private String s3secrey;
|
|
|
-
|
|
|
- @Value("${fdkk.aws.s3bucket:4dkankan}")
|
|
|
- private String s3bucket;
|
|
|
-
|
|
|
- @Value("${local.path:/home/4dkankan}")
|
|
|
- private String localPath;
|
|
|
-
|
|
|
- /**
|
|
|
- * oss文件上传命令
|
|
|
- * 第一个参数是oss路径,要包含bucket名称
|
|
|
- * 第二个参数是本地文件路径
|
|
|
- */
|
|
|
- private static final String UPLOAD_SH = "bash /opt/ossutil/upload.sh %s %s";
|
|
|
-
|
|
|
- //上传的数据是byte[],key是上传后的文件名
|
|
|
- public void upload(byte[] data,String key1) throws IOException{
|
|
|
- //log.info("开始上传文件 源路径:{},目标路径:{},type:{}" , new String(data, "UTF-8"),key1,type);
|
|
|
- StorageType storageType = StorageType.get(type);
|
|
|
- switch (storageType){
|
|
|
- case OSS:
|
|
|
- uploadOss(data,key1);
|
|
|
- break;
|
|
|
- case AWS:
|
|
|
- uploadAws(data,key1);
|
|
|
- break;
|
|
|
- case LOCAL:
|
|
|
- uploadLocal(data,key1);
|
|
|
- break;
|
|
|
- }
|
|
|
- }
|
|
|
- public void upload(String filePath, String key1) {
|
|
|
- //log.info("开始上传文件 源路径:{},目标路径:{},type:{}" , filePath,key1,type);
|
|
|
- StorageType storageType = StorageType.get(type);
|
|
|
- switch (storageType){
|
|
|
- case OSS:
|
|
|
- uploadOss(filePath,key1);
|
|
|
- break;
|
|
|
- case AWS:
|
|
|
- uploadAws(filePath,key1);
|
|
|
- break;
|
|
|
- case LOCAL:
|
|
|
- uploadLocal(filePath,key1);
|
|
|
- break;
|
|
|
- }
|
|
|
- }
|
|
|
-
|
|
|
- public void upload2(String filePath, String key1) {
|
|
|
- //log.info("开始上传文件 源路径:{},目标路径:{},type:{}" , filePath,key1,type);
|
|
|
- switch (type){
|
|
|
- case "oss":upload2Oss(filePath,key1); break;
|
|
|
- case "aws": uploadAws(filePath,key1); break;
|
|
|
- case "local":uploadLocal(filePath,key1); break;
|
|
|
- }
|
|
|
- }
|
|
|
- public void delete(String key1) throws IOException{
|
|
|
- switch (type){
|
|
|
- case "oss":deleteOss(key1); break;
|
|
|
- case "aws": deleteS3Object(key1); break;
|
|
|
- case "local":
|
|
|
- FileUtil.del(key1); break;
|
|
|
- }
|
|
|
- }
|
|
|
- public int deleteFile(String prefix){
|
|
|
- switch (type){
|
|
|
- case "oss":deleteOssFile(prefix); break;
|
|
|
- case "aws": deleteS3Object(prefix); break;
|
|
|
- case "local":FileUtil.del(prefix); break;
|
|
|
- }
|
|
|
- return 1;
|
|
|
- }
|
|
|
-
|
|
|
- public void deleteOss(String objectName){
|
|
|
- OSSClient ossClient = new OSSClient(point, key, secrey);
|
|
|
- try {
|
|
|
- ossClient.deleteObject(bucket, objectName);
|
|
|
- } catch (Exception e) {
|
|
|
- log.error("OSS删除文件失败,key=" + objectName);
|
|
|
- }finally {
|
|
|
- if(ossClient != null){
|
|
|
- ossClient.shutdown();
|
|
|
- }
|
|
|
- }
|
|
|
- }
|
|
|
-
|
|
|
- public void deleteOssFile(String prefix){
|
|
|
- OSSClient ossClient = new OSSClient(point, key, secrey);
|
|
|
- ObjectListing objectListing = ossClient.listObjects(bucket, prefix);
|
|
|
- List<OSSObjectSummary> sums = objectListing.getObjectSummaries();
|
|
|
- try {
|
|
|
- for (OSSObjectSummary s : sums) {
|
|
|
- delete(s.getKey());
|
|
|
- }
|
|
|
- } catch (IOException e) {
|
|
|
- e.printStackTrace();
|
|
|
- }
|
|
|
- }
|
|
|
-
|
|
|
- public void uploadOss(byte[] data,String objectName){
|
|
|
- OSSClient ossClient = new OSSClient(point, key, secrey);
|
|
|
- try {
|
|
|
- ossClient.putObject(bucket, objectName, new ByteArrayInputStream(data));
|
|
|
- } catch (Exception e) {
|
|
|
- log.error("oss上传文件失败", e);
|
|
|
- }finally {
|
|
|
- if(ossClient != null){
|
|
|
- ossClient.shutdown();
|
|
|
- }
|
|
|
- }
|
|
|
- }
|
|
|
- public void uploadAws(byte[] data,String objectName){
|
|
|
- BasicAWSCredentials awsCreds = new BasicAWSCredentials(s3key, s3secrey);
|
|
|
- AmazonS3 s3 = AmazonS3ClientBuilder.standard()
|
|
|
- .withCredentials(new AWSStaticCredentialsProvider(awsCreds))
|
|
|
- .withRegion(Regions.EU_WEST_2)
|
|
|
- .build();
|
|
|
- try {
|
|
|
- com.amazonaws.services.s3.model.ObjectMetadata metadata = new com.amazonaws.services.s3.model.ObjectMetadata();
|
|
|
- s3.putObject(this.bucket, objectName, new ByteArrayInputStream(data), metadata);
|
|
|
- }catch (Exception e){
|
|
|
- log.error("s3上传文件失败", e);
|
|
|
- }finally {
|
|
|
- if(s3 != null){
|
|
|
- s3.shutdown();
|
|
|
- }
|
|
|
- }
|
|
|
- }
|
|
|
- public void uploadLocal(byte[] data,String key1){
|
|
|
- InputStream in = new ByteArrayInputStream(data);
|
|
|
- File file = new File(key1);
|
|
|
- String path = key1.substring(0, key1.lastIndexOf("/"));
|
|
|
- if (!file.exists()) {
|
|
|
- new File(path).mkdir();
|
|
|
- }
|
|
|
- FileOutputStream fos = null;
|
|
|
- try {
|
|
|
- fos = new FileOutputStream(file);
|
|
|
- int len = 0;
|
|
|
- byte[] buf = new byte[1024];
|
|
|
- while ((len = in.read(buf)) != -1) {
|
|
|
- fos.write(buf, 0, len);
|
|
|
- }
|
|
|
- fos.flush();
|
|
|
- } catch (Exception e) {
|
|
|
- e.printStackTrace();
|
|
|
- } finally {
|
|
|
- if (null != fos) {
|
|
|
- try {
|
|
|
- fos.close();
|
|
|
- } catch (IOException e) {
|
|
|
- e.printStackTrace();
|
|
|
- }
|
|
|
- }
|
|
|
- }
|
|
|
- }
|
|
|
-
|
|
|
-
|
|
|
- public void uploadOss(String filePath, String key1){
|
|
|
- OSSClient ossClient = new OSSClient(point, key, secrey);
|
|
|
- try {
|
|
|
- File file = new File(filePath);
|
|
|
- if (!file.exists()) {
|
|
|
- log.error("要上传的文件不存在:" + filePath);
|
|
|
- return;
|
|
|
- }
|
|
|
- ObjectMetadata metadata = new ObjectMetadata();
|
|
|
- if(filePath.contains(".jpg")){
|
|
|
- metadata.setContentType("image/jpeg");
|
|
|
- }
|
|
|
- ossClient.putObject(bucket, key1, new File(filePath), metadata);
|
|
|
-
|
|
|
- } catch (Exception e) {
|
|
|
- log.error(e.toString() + filePath);
|
|
|
- } finally {
|
|
|
- ossClient.shutdown();
|
|
|
- }
|
|
|
- }
|
|
|
-
|
|
|
- public void uploadAws(String filePath, String key1){
|
|
|
- try{
|
|
|
- uploadS3File(filePath, key1);
|
|
|
- }catch (Exception e){
|
|
|
- e.printStackTrace();
|
|
|
- }
|
|
|
- }
|
|
|
- public void uploadLocal(String filePath, String key1){
|
|
|
- try {
|
|
|
- File srcFile = new File(filePath);
|
|
|
- File file = new File(localPath + key1);
|
|
|
- FileUtils.copyFile(srcFile,file);
|
|
|
- }catch (Exception e){
|
|
|
- e.printStackTrace();
|
|
|
- }
|
|
|
- }
|
|
|
-
|
|
|
-
|
|
|
-
|
|
|
- public void upload2Oss(String filePath, String key1){
|
|
|
- OSSClient ossClient = new OSSClient(point, key, secrey);
|
|
|
- try {
|
|
|
- ObjectMetadata metadata = new ObjectMetadata();
|
|
|
- if(filePath.contains(".jpg")){
|
|
|
- metadata.setContentType("image/jpeg");
|
|
|
- }
|
|
|
- if(filePath.contains(".mp4")){
|
|
|
- metadata.setContentType("video/mp4");
|
|
|
- }
|
|
|
- if(filePath.contains(".mp3")){
|
|
|
- metadata.setContentType("audio/mp3");
|
|
|
- }
|
|
|
- ossClient.putObject(bucket, key1, new File(filePath), metadata);
|
|
|
- } catch (Exception e) {
|
|
|
- log.error(e.toString() + filePath);
|
|
|
- }
|
|
|
- }
|
|
|
-
|
|
|
-
|
|
|
-
|
|
|
-
|
|
|
-
|
|
|
- //海外亚马逊s3
|
|
|
-
|
|
|
-
|
|
|
- /**
|
|
|
- * s3上传文件
|
|
|
- * @param filePath
|
|
|
- * @param key1
|
|
|
- * @throws IOException
|
|
|
- */
|
|
|
- private void uploadS3File(String filePath, String key1) throws Exception {
|
|
|
- /**
|
|
|
- * 创建s3对象
|
|
|
- */
|
|
|
- BasicAWSCredentials awsCreds = new BasicAWSCredentials(s3key, s3secrey);
|
|
|
- AmazonS3 s3 = AmazonS3ClientBuilder.standard()
|
|
|
- .withCredentials(new AWSStaticCredentialsProvider(awsCreds))
|
|
|
- .withRegion(Regions.EU_WEST_2)
|
|
|
- .build();
|
|
|
- try{
|
|
|
- File file = new File(filePath);
|
|
|
- if(!file.exists()){
|
|
|
- log.info("要上传s3的文件不存在");
|
|
|
- return;
|
|
|
- }
|
|
|
-
|
|
|
- // 设置文件并设置公读
|
|
|
- com.amazonaws.services.s3.model.ObjectMetadata metadata = new com.amazonaws.services.s3.model.ObjectMetadata();
|
|
|
- if(filePath.contains(".jpg")){
|
|
|
- metadata.setContentType("image/jpeg");
|
|
|
- }
|
|
|
- if(filePath.contains(".png")){
|
|
|
- metadata.setContentType("image/png");
|
|
|
- }
|
|
|
- PutObjectRequest request = new PutObjectRequest(s3bucket, key1, file);
|
|
|
- request.withCannedAcl(CannedAccessControlList.PublicRead);
|
|
|
- request.withMetadata(metadata);
|
|
|
-
|
|
|
- // 上传文件
|
|
|
- com.amazonaws.services.s3.model.PutObjectResult putObjectResult = s3.putObject(request);
|
|
|
- if (StringUtils.isNotEmpty(putObjectResult.getETag())) {
|
|
|
- log.info("s3上传文件成功:" + key1);
|
|
|
- }
|
|
|
- }catch (Exception e){
|
|
|
- throw e;
|
|
|
- }finally {
|
|
|
- s3.shutdown();
|
|
|
- }
|
|
|
-
|
|
|
- }
|
|
|
-
|
|
|
- /**
|
|
|
- * 删除单个文件
|
|
|
- *
|
|
|
- * @param objectName 文件路径[ eg: /head/xxxx.jpg ]
|
|
|
- * @return
|
|
|
- */
|
|
|
- public void deleteS3Object(String objectName) {
|
|
|
-
|
|
|
- /**
|
|
|
- * 创建s3对象
|
|
|
- */
|
|
|
- BasicAWSCredentials awsCreds = new BasicAWSCredentials(s3key, s3secrey);
|
|
|
- AmazonS3 s3 = AmazonS3ClientBuilder.standard()
|
|
|
- .withCredentials(new AWSStaticCredentialsProvider(awsCreds))
|
|
|
- .withRegion(Regions.EU_WEST_2)
|
|
|
- .build();
|
|
|
-
|
|
|
- if (objectName.startsWith("/")) {
|
|
|
- objectName = objectName.substring(1);
|
|
|
- }
|
|
|
-
|
|
|
- try {
|
|
|
- s3.deleteObject(s3bucket, objectName);
|
|
|
- } catch (Exception e) {
|
|
|
- log.error("s3删除文件失败,key="+objectName, e);
|
|
|
- }finally {
|
|
|
- if(s3 != null){
|
|
|
- s3.shutdown();
|
|
|
- }
|
|
|
- }
|
|
|
- }
|
|
|
-
|
|
|
-
|
|
|
-
|
|
|
- /**
|
|
|
- * 获取文件类型
|
|
|
- */
|
|
|
- public static String getContentType(String filePath){
|
|
|
- FileNameMap fileNameMap = URLConnection.getFileNameMap();
|
|
|
- String contentType = fileNameMap.getContentTypeFor(filePath);
|
|
|
- System.out.println(contentType);
|
|
|
- return contentType;
|
|
|
-
|
|
|
- }
|
|
|
-
|
|
|
-
|
|
|
-
|
|
|
- /**
|
|
|
- * 获得文件列表-阿里云
|
|
|
- * @return
|
|
|
- */
|
|
|
- public List<String> listKeysFromAli(String sourcePath) {
|
|
|
- List<String> keyList = new ArrayList<>();
|
|
|
- OSSClient ossClient = new OSSClient(point, key, secrey);
|
|
|
- try {
|
|
|
- boolean flag = true;
|
|
|
- String nextMaker = null;
|
|
|
- ListObjectsRequest listObjectsRequest = new ListObjectsRequest(this.bucket);
|
|
|
- //指定下一级文件
|
|
|
- listObjectsRequest.setPrefix(sourcePath);
|
|
|
- //设置分页的页容量
|
|
|
- listObjectsRequest.setMaxKeys(200);
|
|
|
- do
|
|
|
- {
|
|
|
- //获取下一页的起始点,它的下一项
|
|
|
- listObjectsRequest.setMarker(nextMaker);
|
|
|
- ObjectListing objectListing = ossClient.listObjects(listObjectsRequest);
|
|
|
- List<OSSObjectSummary> objectSummaries = objectListing.getObjectSummaries();
|
|
|
- List<String> collect = objectSummaries.stream().map(summary -> {
|
|
|
- return summary.getKey();
|
|
|
- }).collect(Collectors.toList());
|
|
|
- if(collect.size() >0){
|
|
|
- keyList.addAll(collect);
|
|
|
- }
|
|
|
- nextMaker = objectListing.getNextMarker();
|
|
|
- //全部执行完后,为false
|
|
|
- flag = objectListing.isTruncated();
|
|
|
- } while (flag);
|
|
|
- }catch (Exception e){
|
|
|
- log.error("获取文件列表失败,path="+sourcePath, e);
|
|
|
- }finally {
|
|
|
- if(ossClient != null){
|
|
|
- ossClient.shutdown();
|
|
|
- }
|
|
|
- }
|
|
|
-
|
|
|
-
|
|
|
- ossClient.shutdown();
|
|
|
-
|
|
|
- return keyList;
|
|
|
- }
|
|
|
-
|
|
|
- /**
|
|
|
- * 获得文件列表-亚马逊
|
|
|
- * @return
|
|
|
- */
|
|
|
- public List<String> listKeysFromAws(String sourcePath) {
|
|
|
- List<String> keyList = new ArrayList<>();
|
|
|
-
|
|
|
- BasicAWSCredentials awsCreds = new BasicAWSCredentials(s3key, s3secrey);
|
|
|
- AmazonS3 s3 = AmazonS3ClientBuilder.standard()
|
|
|
- .withCredentials(new AWSStaticCredentialsProvider(awsCreds))
|
|
|
- .withRegion(Regions.EU_WEST_2)
|
|
|
- .build();
|
|
|
-
|
|
|
- try {
|
|
|
- boolean flag = true;
|
|
|
- String nextMaker = null;
|
|
|
- com.amazonaws.services.s3.model.ListObjectsRequest listObjectsRequest = new com.amazonaws.services.s3.model.ListObjectsRequest();
|
|
|
- listObjectsRequest.setBucketName(this.s3bucket);
|
|
|
- listObjectsRequest.setPrefix(sourcePath);
|
|
|
- listObjectsRequest.setMaxKeys(200);
|
|
|
-
|
|
|
- do{
|
|
|
- listObjectsRequest.setMarker(nextMaker);
|
|
|
- com.amazonaws.services.s3.model.ObjectListing objectListing = s3.listObjects(listObjectsRequest);
|
|
|
- List<S3ObjectSummary> objectSummaries = objectListing.getObjectSummaries();
|
|
|
- List<String> collect =objectSummaries.stream().map(summary->{
|
|
|
- return summary.getKey();
|
|
|
- }).collect(Collectors.toList());
|
|
|
- if(collect.size() >0){
|
|
|
- keyList.addAll(collect);
|
|
|
- }
|
|
|
- nextMaker = objectListing.getNextMarker();
|
|
|
- flag = objectListing.isTruncated();
|
|
|
- }while (flag);
|
|
|
- }catch (Exception e){
|
|
|
- log.error("获取文件列表失败,path="+sourcePath, e);
|
|
|
- }finally {
|
|
|
- if(s3 != null){
|
|
|
- s3.shutdown();
|
|
|
- }
|
|
|
- }
|
|
|
- return keyList;
|
|
|
- }
|
|
|
-
|
|
|
- /**
|
|
|
- * 获得文件列表-阿里云
|
|
|
- * @return
|
|
|
- */
|
|
|
- public List<String> listKeysFromLocal(String sourcePath) {
|
|
|
- List<String> keyList = new ArrayList<>();
|
|
|
- return keyList;
|
|
|
- }
|
|
|
-
|
|
|
- /**
|
|
|
- * <p>
|
|
|
- 拷贝目录
|
|
|
- * </p>
|
|
|
- * @author dengsixing
|
|
|
- * @date 2022/1/18
|
|
|
- * @param sourcePath
|
|
|
- * @param targetPath
|
|
|
- **/
|
|
|
- public void copyFiles(String sourcePath, String targetPath) throws IOException {
|
|
|
- StorageType storageType = StorageType.get(type);
|
|
|
- switch (storageType){
|
|
|
- case OSS:
|
|
|
- this.copyFilesFromAli(sourcePath, targetPath);
|
|
|
- break;
|
|
|
- case AWS:
|
|
|
- this.copyFilesFromAws(sourcePath, targetPath);
|
|
|
- break;
|
|
|
- case LOCAL: this.copyFilesFromLocal(sourcePath, targetPath);
|
|
|
- }
|
|
|
- }
|
|
|
-
|
|
|
- /**
|
|
|
- * <p>
|
|
|
- 拷贝文件
|
|
|
- * </p>
|
|
|
- * @author dengsixing
|
|
|
- * @date 2022/1/18
|
|
|
- * @param sourceKey
|
|
|
- * @param targetKey
|
|
|
- **/
|
|
|
- public void copyObject(String sourceKey, String targetKey) throws IOException {
|
|
|
- StorageType storageType = StorageType.get(type);
|
|
|
- switch (storageType){
|
|
|
- case OSS:
|
|
|
- this.copyObjectFromAli(sourceKey, targetKey);
|
|
|
- break;
|
|
|
- case AWS:
|
|
|
- this.copyObjectFromAws(sourceKey, targetKey);
|
|
|
- break;
|
|
|
- }
|
|
|
- }
|
|
|
-
|
|
|
- /**
|
|
|
- * <p>
|
|
|
- 拷贝-阿里云
|
|
|
- * </p>
|
|
|
- * @author dengsixing
|
|
|
- * @date 2022/1/18
|
|
|
- * @param sourcePath
|
|
|
- * @param targetPath
|
|
|
- **/
|
|
|
- public void copyObjectFromAli(String sourcePath, String targetPath) throws IOException {
|
|
|
-
|
|
|
- // 创建OSSClient实例。
|
|
|
- OSSClient ossClient = new OSSClient(point, key, secrey);
|
|
|
- // 复制文件
|
|
|
- log.info("开始复制:" + sourcePath);
|
|
|
- ossClient.copyObject(this.bucket, sourcePath, this.bucket, targetPath);
|
|
|
- log.info("复制成功:" + sourcePath);
|
|
|
- ossClient.shutdown();
|
|
|
- }
|
|
|
-
|
|
|
-
|
|
|
-
|
|
|
- /**
|
|
|
- * <p>
|
|
|
- 拷贝-阿里云
|
|
|
- * </p>
|
|
|
- * @author dengsixing
|
|
|
- * @date 2022/1/18
|
|
|
- * @param sourcePath
|
|
|
- * @param targetPath
|
|
|
- **/
|
|
|
- public void copyFilesFromAli(String sourcePath, String targetPath) throws IOException {
|
|
|
-
|
|
|
- //获取源文件列表
|
|
|
- List<String> sourceKeyList = this.listKeysFromAli(sourcePath);
|
|
|
- if(CollUtil.isEmpty(sourceKeyList)){
|
|
|
- return;
|
|
|
- }
|
|
|
- // 创建OSSClient实例。
|
|
|
- OSSClient ossClient = new OSSClient(point, key, secrey);
|
|
|
- // 复制文件
|
|
|
- sourceKeyList.parallelStream().forEach(key -> {
|
|
|
- log.info("开始复制:" + key);
|
|
|
- ossClient.copyObject(this.bucket, key, this.bucket, key.replace(sourcePath, targetPath));
|
|
|
- log.info("复制成功:" + key);
|
|
|
- });
|
|
|
-
|
|
|
- ossClient.shutdown();
|
|
|
- }
|
|
|
-
|
|
|
-
|
|
|
- /**
|
|
|
- * <p>
|
|
|
- 拷贝-亚马逊
|
|
|
- * </p>
|
|
|
- * @author dengsixing
|
|
|
- * @date 2022/1/18
|
|
|
- * @param sourcePath
|
|
|
- * @param targetPath
|
|
|
- **/
|
|
|
- public void copyFilesFromAws(String sourcePath, String targetPath){
|
|
|
-
|
|
|
- try {
|
|
|
- List<String> sourceKeyList = this.listKeysFromAws(sourcePath);
|
|
|
- /**
|
|
|
- * 创建s3对象
|
|
|
- */
|
|
|
- BasicAWSCredentials awsCreds = new BasicAWSCredentials(s3key, s3secrey);
|
|
|
- AmazonS3 s3 = AmazonS3ClientBuilder.standard()
|
|
|
- .withCredentials(new AWSStaticCredentialsProvider(awsCreds))
|
|
|
- .withRegion(Regions.EU_WEST_2)//s3 地区位置
|
|
|
- .build();
|
|
|
-
|
|
|
- // 复制文件
|
|
|
- sourceKeyList.parallelStream().forEach(key -> {
|
|
|
- log.info("开始复制:" + key);
|
|
|
- s3.copyObject(this.s3bucket, key, this.s3bucket, key.replace(sourcePath, targetPath));
|
|
|
- log.info("复制成功:" + key);
|
|
|
- });
|
|
|
- s3.shutdown();
|
|
|
- } catch (Exception ase) {
|
|
|
- log.error("amazonS拷贝异常 " + ase.getMessage(), ase);
|
|
|
- }
|
|
|
- }
|
|
|
-
|
|
|
- /**
|
|
|
- * <p>
|
|
|
- 拷贝-亚马逊
|
|
|
- * </p>
|
|
|
- * @author dengsixing
|
|
|
- * @date 2022/1/18
|
|
|
- * @param sourceKey
|
|
|
- * @param targetKey
|
|
|
- **/
|
|
|
- public void copyObjectFromAws(String sourceKey, String targetKey){
|
|
|
- BasicAWSCredentials awsCreds = new BasicAWSCredentials(s3key, s3secrey);
|
|
|
- AmazonS3 s3 = AmazonS3ClientBuilder.standard()
|
|
|
- .withCredentials(new AWSStaticCredentialsProvider(awsCreds))
|
|
|
- .withRegion(Regions.EU_WEST_2)//s3 地区位置
|
|
|
- .build();
|
|
|
- // 复制文件
|
|
|
- log.info("开始复制:" + sourceKey);
|
|
|
- s3.copyObject(this.s3bucket, sourceKey, this.s3bucket, targetKey);
|
|
|
- log.info("复制成功:" + sourceKey);
|
|
|
- s3.shutdown();
|
|
|
- }
|
|
|
-
|
|
|
- /**
|
|
|
- * <p>
|
|
|
- 拷贝-本地
|
|
|
- * </p>
|
|
|
- * @author dengsixing
|
|
|
- * @date 2022/1/18
|
|
|
- * @param sourcePath
|
|
|
- * @param targetPath
|
|
|
- **/
|
|
|
- public void copyFilesFromLocal(String sourcePath, String targetPath) throws IOException {
|
|
|
- // TODO: 2022/1/21
|
|
|
-
|
|
|
- }
|
|
|
-
|
|
|
- /**
|
|
|
- * 获取文件内容
|
|
|
- * @param objectName
|
|
|
- * @return
|
|
|
- */
|
|
|
- public String getObjectContent( String objectName){
|
|
|
- log.info("下载文件:"+objectName);
|
|
|
- StorageType storageType = StorageType.get(type);
|
|
|
- switch (storageType){
|
|
|
- case OSS:
|
|
|
- return this.getObjectContentFromAli(bucket, objectName);
|
|
|
- case AWS:
|
|
|
- return this.getObjectContentFromAws(s3bucket, objectName);
|
|
|
- case LOCAL:
|
|
|
- return this.getObjectContentFromLocal(objectName);
|
|
|
- }
|
|
|
- return null;
|
|
|
- }
|
|
|
-
|
|
|
- /**
|
|
|
- * 获取文件内容-阿里云
|
|
|
- * @param bucketName
|
|
|
- * @param objectName
|
|
|
- * @return
|
|
|
- */
|
|
|
- public String getObjectContentFromAli(String bucketName, String objectName){
|
|
|
- //创建oss客戶端
|
|
|
- OSSClient ossClient = new OSSClient(point, key, secrey);
|
|
|
- InputStream objectContent = null;
|
|
|
- StringBuilder contentJson = new StringBuilder();
|
|
|
- try {
|
|
|
- // ossObject包含文件所在的存储空间名称、文件名称、文件元信息以及一个输入流。
|
|
|
- OSSObject ossObject = ossClient.getObject(bucketName, objectName);
|
|
|
- objectContent = ossObject.getObjectContent();
|
|
|
- try(BufferedReader reader = new BufferedReader(new InputStreamReader(objectContent))){
|
|
|
- while (true) {
|
|
|
- String line = reader.readLine();
|
|
|
- if (line == null) break;
|
|
|
- contentJson.append(line);
|
|
|
- }
|
|
|
- } catch (IOException e) {
|
|
|
- log.error("读取scene.json文件流失败", e);
|
|
|
- }
|
|
|
- }catch (Exception e){
|
|
|
- log.error("s3获取文件内容失败,key="+objectName, e);
|
|
|
- }finally {
|
|
|
- if(ossClient != null){
|
|
|
- ossClient.shutdown();
|
|
|
- }
|
|
|
- }
|
|
|
-
|
|
|
- return contentJson.toString();
|
|
|
- }
|
|
|
-
|
|
|
- /**
|
|
|
- * 获取文件内容-阿里云
|
|
|
- * @param objectName
|
|
|
- * @return
|
|
|
- */
|
|
|
- public boolean existOnAli(String objectName){
|
|
|
- //创建oss客戶端
|
|
|
- OSSClient ossClient = new OSSClient(point, key, secrey);
|
|
|
- // ossObject包含文件所在的存储空间名称、文件名称、文件元信息以及一个输入流。
|
|
|
- try{
|
|
|
- boolean exist = ossClient.doesObjectExist(bucket, objectName);
|
|
|
- return exist;
|
|
|
- }catch (Exception e){
|
|
|
- log.error("s4判断是否存在key异常,key=" + objectName, e);
|
|
|
- }finally {
|
|
|
- if(ossClient != null){
|
|
|
- ossClient.shutdown();
|
|
|
- }
|
|
|
- }
|
|
|
- return false;
|
|
|
- }
|
|
|
-
|
|
|
- /**
|
|
|
- * 获取文件内容-亚马逊
|
|
|
- * @param bucketName
|
|
|
- * @param objectName
|
|
|
- * @return
|
|
|
- */
|
|
|
- public String getObjectContentFromAws(String bucketName, String objectName){
|
|
|
-
|
|
|
- try {
|
|
|
- /**
|
|
|
- * 创建s3对象
|
|
|
- */
|
|
|
- BasicAWSCredentials awsCreds = new BasicAWSCredentials(s3key, s3secrey);
|
|
|
- AmazonS3 s3 = AmazonS3ClientBuilder.standard()
|
|
|
- .withCredentials(new AWSStaticCredentialsProvider(awsCreds))
|
|
|
- .withRegion(Regions.EU_WEST_2)
|
|
|
- .build();
|
|
|
-
|
|
|
- GetObjectRequest request = new GetObjectRequest(bucketName,objectName);
|
|
|
- S3Object object = s3.getObject(request);
|
|
|
- S3ObjectInputStream inputStream = object.getObjectContent();
|
|
|
- StringBuilder content = new StringBuilder();
|
|
|
- try(BufferedReader reader = new BufferedReader(new InputStreamReader(inputStream))){
|
|
|
- while (true) {
|
|
|
- String line = reader.readLine();
|
|
|
- if (line == null) break;
|
|
|
- content.append(line);
|
|
|
- }
|
|
|
- } catch (IOException e) {
|
|
|
- log.error("读取aws文件流失败", e);
|
|
|
- }
|
|
|
- return content.toString();
|
|
|
- } catch (Exception ase) {
|
|
|
- log.error("amazonS3下载文件异常 " + ase.getMessage(), ase);
|
|
|
- }
|
|
|
- return null;
|
|
|
-
|
|
|
- }
|
|
|
-
|
|
|
- /**
|
|
|
- * 获取文件内容-亚马逊
|
|
|
- * @param objectName
|
|
|
- * @return
|
|
|
- */
|
|
|
- public boolean existOnAws(String objectName){
|
|
|
-
|
|
|
- BasicAWSCredentials awsCreds = new BasicAWSCredentials(s3key, s3secrey);
|
|
|
- AmazonS3 s3 = AmazonS3ClientBuilder.standard()
|
|
|
- .withCredentials(new AWSStaticCredentialsProvider(awsCreds))
|
|
|
- .withRegion(Regions.EU_WEST_2)
|
|
|
- .build();
|
|
|
- try {
|
|
|
- boolean exist = s3.doesObjectExist(s3bucket, objectName);
|
|
|
- return exist;
|
|
|
- }catch (Exception e){
|
|
|
- log.error("s4判断是否存在key异常,key=" + objectName, e);
|
|
|
- }finally {
|
|
|
- if(s3 != null){
|
|
|
- s3.shutdown();
|
|
|
- }
|
|
|
- }
|
|
|
- return false;
|
|
|
- }
|
|
|
-
|
|
|
- /**
|
|
|
- * 判断key是否存在
|
|
|
- * @param key
|
|
|
- * @return
|
|
|
- */
|
|
|
- public boolean existKey(String key){
|
|
|
- StorageType storageType = StorageType.get(type);
|
|
|
- switch (storageType){
|
|
|
- case OSS:
|
|
|
- return this.existOnAli(key);
|
|
|
- case AWS:
|
|
|
- return this.existOnAws(key);
|
|
|
- default:
|
|
|
- return false;
|
|
|
- }
|
|
|
- }
|
|
|
-
|
|
|
- /**
|
|
|
- * 获取文件内容-本地
|
|
|
- * @param objectName
|
|
|
- * @return
|
|
|
- */
|
|
|
- public String getObjectContentFromLocal(String objectName){
|
|
|
- // TODO: 2022/1/21
|
|
|
- return null;
|
|
|
- }
|
|
|
-
|
|
|
- /**
|
|
|
- * oss下载文件到本地
|
|
|
- * @param objectName
|
|
|
- * @param localPath
|
|
|
- */
|
|
|
- public boolean download(String objectName, String localPath){
|
|
|
- StorageType storageType = StorageType.get(this.type);
|
|
|
- switch (storageType){
|
|
|
- case OSS:
|
|
|
- return this.downFormAli(objectName, localPath);
|
|
|
- case AWS:
|
|
|
- return this.downFromS3(objectName, localPath);
|
|
|
- }
|
|
|
- return false;
|
|
|
- }
|
|
|
-
|
|
|
- /**
|
|
|
- * 从阿里云oss下载文件到本地
|
|
|
- * @param objectName 云端文件k地址
|
|
|
- * @param localPath 本地文件地址
|
|
|
- * @return
|
|
|
- */
|
|
|
- public boolean downFormAli(String objectName, String localPath){
|
|
|
- OSSClient ossClient = new OSSClient(point, key, secrey);
|
|
|
- try {
|
|
|
- com.aliyun.oss.model.GetObjectRequest request = new com.aliyun.oss.model.GetObjectRequest(bucket,objectName);
|
|
|
- ossClient.getObject(request, new File(localPath));
|
|
|
- return true;
|
|
|
- }catch (Exception e){
|
|
|
- log.error("阿里云oss文件下载失败,key=" + objectName, e);
|
|
|
- }finally {
|
|
|
- if(ossClient != null){
|
|
|
- ossClient.shutdown();
|
|
|
- }
|
|
|
- }
|
|
|
- return false;
|
|
|
- }
|
|
|
-
|
|
|
- /**
|
|
|
- * 从s3下载文件到本地
|
|
|
- * @param objectName 云端文件k地址
|
|
|
- * @param localPath 本地文件地址
|
|
|
- * @return
|
|
|
- */
|
|
|
- public boolean downFromS3(String objectName, String localPath) {
|
|
|
- BasicAWSCredentials awsCreds = new BasicAWSCredentials(s3key, s3secrey);
|
|
|
- AmazonS3 s3 = AmazonS3ClientBuilder.standard()
|
|
|
- .withCredentials(new AWSStaticCredentialsProvider(awsCreds))
|
|
|
- .withRegion(Regions.EU_WEST_2)
|
|
|
- .build();
|
|
|
- try {
|
|
|
- GetObjectRequest request = new GetObjectRequest(this.s3bucket,objectName);
|
|
|
- s3.getObject(request,new File(localPath));
|
|
|
- return true;
|
|
|
- } catch (Exception e) {
|
|
|
- log.error("amazonS3下载文件失败,key=" + objectName, e);
|
|
|
- }finally {
|
|
|
- if(s3 != null){
|
|
|
- s3.shutdown();
|
|
|
- }
|
|
|
- }
|
|
|
- return false;
|
|
|
- }
|
|
|
-
|
|
|
- public List<String> listKeys(String sourcePath) {
|
|
|
- StorageType storageType = StorageType.get(type);
|
|
|
- switch (storageType){
|
|
|
- case OSS:
|
|
|
- return this.listKeysFromAli(sourcePath);
|
|
|
- case AWS:
|
|
|
- return this.listKeysFromAws(sourcePath);
|
|
|
- case LOCAL:
|
|
|
- return this.listKeysFromLocal(sourcePath);
|
|
|
- }
|
|
|
- return null;
|
|
|
- }
|
|
|
-
|
|
|
- public List<String> listRemoteFiles(String sourcePath) {
|
|
|
- ArrayList keyList = new ArrayList();
|
|
|
- OSSClient ossClient = new OSSClient(point, key, secrey);
|
|
|
- try {
|
|
|
- boolean flag = true;
|
|
|
- String nextMaker = null;
|
|
|
- ListObjectsRequest listObjectsRequest = new ListObjectsRequest(bucket);
|
|
|
- listObjectsRequest.setPrefix(sourcePath);
|
|
|
- listObjectsRequest.setMaxKeys(200);
|
|
|
-
|
|
|
- do {
|
|
|
- listObjectsRequest.setMarker(nextMaker);
|
|
|
- ObjectListing objectListing = ossClient.listObjects(listObjectsRequest);
|
|
|
- List<String> collect = (List)objectListing.getObjectSummaries().parallelStream().map(OSSObjectSummary::getKey).collect(Collectors.toList());
|
|
|
- if (!CollectionUtils.isEmpty(collect)) {
|
|
|
- keyList.addAll(collect);
|
|
|
- }
|
|
|
-
|
|
|
- nextMaker = objectListing.getNextMarker();
|
|
|
- flag = objectListing.isTruncated();
|
|
|
- } while(flag);
|
|
|
- } catch (Exception var9) {
|
|
|
- log.error("获取文件列表失败,path:" + sourcePath, var9);
|
|
|
- }
|
|
|
-
|
|
|
- return keyList;
|
|
|
- }
|
|
|
-
|
|
|
-
|
|
|
- public void copy(String sourcePath ,String bucket1,String bucket2) {
|
|
|
- OSSClient ossClient = new OSSClient(point, key, secrey);
|
|
|
- log.info("ossCopy--sourcePath:{},bucket1:{},bucket2:{}",sourcePath,bucket1,bucket2);
|
|
|
- try {
|
|
|
- List<String> files = listRemoteFiles(sourcePath);
|
|
|
- for (String file : files) {
|
|
|
- ossClient.copyObject(bucket1, file, bucket2,file);
|
|
|
- }
|
|
|
-
|
|
|
- } catch (Exception e) {
|
|
|
- log.error("oss-copyError-sourcePath:{},error:{}",sourcePath,e);
|
|
|
- throw e;
|
|
|
- }finally {
|
|
|
- ossClient.shutdown();
|
|
|
- }
|
|
|
- }
|
|
|
-
|
|
|
- public static List<String> listRemoteFiles(String point,String key ,String secrey,String bucket,String sourcePath) {
|
|
|
- ArrayList keyList = new ArrayList();
|
|
|
- OSSClient ossClient = new OSSClient(point, key, secrey);
|
|
|
- try {
|
|
|
- boolean flag = true;
|
|
|
- String nextMaker = null;
|
|
|
- ListObjectsRequest listObjectsRequest = new ListObjectsRequest(bucket);
|
|
|
- listObjectsRequest.setPrefix(sourcePath);
|
|
|
- listObjectsRequest.setMaxKeys(200);
|
|
|
-
|
|
|
- do {
|
|
|
- listObjectsRequest.setMarker(nextMaker);
|
|
|
- ObjectListing objectListing = ossClient.listObjects(listObjectsRequest);
|
|
|
- List<String> collect = (List)objectListing.getObjectSummaries().parallelStream().map(OSSObjectSummary::getKey).collect(Collectors.toList());
|
|
|
- if (!CollectionUtils.isEmpty(collect)) {
|
|
|
- keyList.addAll(collect);
|
|
|
- }
|
|
|
-
|
|
|
- nextMaker = objectListing.getNextMarker();
|
|
|
- flag = objectListing.isTruncated();
|
|
|
- } while(flag);
|
|
|
- } catch (Exception var9) {
|
|
|
- log.error("获取文件列表失败,path:" + sourcePath, var9);
|
|
|
- }
|
|
|
-
|
|
|
- return keyList;
|
|
|
- }
|
|
|
-
|
|
|
-
|
|
|
- public static void copy(String point,String key ,String secrey,String sourcePath ,String bucket1,String bucket2) {
|
|
|
- OSSClient ossClient = new OSSClient(point, key, secrey);
|
|
|
- log.info("ossCopy--sourcePath:{},bucket1:{},bucket2:{}",sourcePath,bucket1,bucket2);
|
|
|
- try {
|
|
|
- List<String> files = listRemoteFiles(point,key,secrey,bucket1,sourcePath);
|
|
|
- for (String file : files) {
|
|
|
- ossClient.copyObject(bucket1, file, bucket2,file);
|
|
|
- }
|
|
|
-
|
|
|
- } catch (Exception e) {
|
|
|
- log.error("oss-copyError-sourcePath:{},error:{}",sourcePath,e);
|
|
|
- throw e;
|
|
|
- }finally {
|
|
|
- ossClient.shutdown();
|
|
|
- }
|
|
|
- }
|
|
|
-
|
|
|
-}
|