|
@@ -1,13 +1,30 @@
|
|
|
package com.fdkankan.tk.common.util;
|
|
|
|
|
|
import cn.hutool.core.collection.CollUtil;
|
|
|
+import cn.hutool.core.io.FileUtil;
|
|
|
import com.aliyun.oss.OSSClient;
|
|
|
+import com.aliyun.oss.model.ListObjectsRequest;
|
|
|
+import com.aliyun.oss.model.ObjectListing;
|
|
|
+import com.aliyun.oss.model.ObjectMetadata;
|
|
|
import com.aliyun.oss.model.*;
|
|
|
+import com.amazonaws.auth.AWSStaticCredentialsProvider;
|
|
|
+import com.amazonaws.auth.BasicAWSCredentials;
|
|
|
+import com.amazonaws.regions.Regions;
|
|
|
+import com.amazonaws.services.s3.AmazonS3;
|
|
|
+import com.amazonaws.services.s3.AmazonS3ClientBuilder;
|
|
|
+import com.amazonaws.services.s3.model.CannedAccessControlList;
|
|
|
+import com.amazonaws.services.s3.model.GetObjectRequest;
|
|
|
+import com.amazonaws.services.s3.model.PutObjectRequest;
|
|
|
+import com.amazonaws.services.s3.model.*;
|
|
|
import lombok.extern.slf4j.Slf4j;
|
|
|
+import org.apache.commons.io.FileUtils;
|
|
|
+import org.apache.commons.lang3.StringUtils;
|
|
|
import org.springframework.beans.factory.annotation.Value;
|
|
|
import org.springframework.stereotype.Component;
|
|
|
|
|
|
import java.io.*;
|
|
|
+import java.net.FileNameMap;
|
|
|
+import java.net.URLConnection;
|
|
|
import java.util.ArrayList;
|
|
|
import java.util.List;
|
|
|
import java.util.stream.Collectors;
|
|
@@ -29,121 +46,358 @@ public class UploadToOssUtil {
|
|
|
@Value("${oss.bucket:4dkankan}")
|
|
|
private String bucket;
|
|
|
|
|
|
- @Value("${upload.query-path}")
|
|
|
- private String query;
|
|
|
+ @Value("${oss.sdk:4dscene}")
|
|
|
+ private String bucketSdk;
|
|
|
+
|
|
|
+ @Value("${upload.type:oss}")
|
|
|
+ private String type;
|
|
|
+
|
|
|
+ @Value("${fdkk.aws.s3key:AKIAWCV5QFZ3ZNELKYUY}")
|
|
|
+ private String s3key;
|
|
|
|
|
|
+ @Value("${fdkk.aws.s3secrey:epS5ghyR4LJ7rxk/qJO9ZYh6m9Oz6g5haKDu4yws}")
|
|
|
+ private String s3secrey;
|
|
|
+
|
|
|
+ @Value("${fdkk.aws.s3bucket:4dkankan}")
|
|
|
+ private String s3bucket;
|
|
|
+
|
|
|
+ @Value("${local.path:/home/4dkankan}")
|
|
|
+ private String localPath;
|
|
|
|
|
|
/**
|
|
|
- * 获取文件内容-阿里云
|
|
|
- * @param objectName
|
|
|
- * @return
|
|
|
+ * oss文件上传命令
|
|
|
+ * 第一个参数是oss路径,要包含bucket名称
|
|
|
+ * 第二个参数是本地文件路径
|
|
|
*/
|
|
|
- public boolean existKey(String objectName){
|
|
|
- //创建oss客户端
|
|
|
- if(objectName.contains(query)){
|
|
|
- objectName = objectName.replace(query,"");
|
|
|
+ private static final String UPLOAD_SH = "bash /opt/ossutil/upload.sh %s %s";
|
|
|
+
|
|
|
+ //上传的数据是byte[],key是上传后的文件名
|
|
|
+ public void upload(byte[] data,String key1) throws IOException{
|
|
|
+ log.info("开始上传文件 源路径:{},目标路径:{},type:{}" , new String(data, "UTF-8"),key1,type);
|
|
|
+ StorageType storageType = StorageType.get(type);
|
|
|
+ switch (storageType){
|
|
|
+ case OSS:
|
|
|
+ uploadOss(data,key1);
|
|
|
+ break;
|
|
|
+ case AWS:
|
|
|
+ uploadAws(data,key1);
|
|
|
+ break;
|
|
|
+ case LOCAL:
|
|
|
+ uploadLocal(data,key1);
|
|
|
+ break;
|
|
|
}
|
|
|
+ }
|
|
|
+ public void upload(String filePath, String key1) {
|
|
|
+ log.info("开始上传文件 源路径:{},目标路径:{},type:{}" , filePath,key1,type);
|
|
|
+ StorageType storageType = StorageType.get(type);
|
|
|
+ switch (storageType){
|
|
|
+ case OSS:
|
|
|
+ uploadOss(filePath,key1);
|
|
|
+ break;
|
|
|
+ case AWS:
|
|
|
+ uploadAws(filePath,key1);
|
|
|
+ break;
|
|
|
+ case LOCAL:
|
|
|
+ uploadLocal(filePath,key1);
|
|
|
+ break;
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+ public void uploadSdk(String filePath, String key1) {
|
|
|
+ log.info("开始上传文件 源路径:{},目标路径:{},type:{}" , filePath,key1,type);
|
|
|
+ switch (type){
|
|
|
+ case "oss":uploadSdkOss(filePath,key1); break;
|
|
|
+ case "aws": uploadAws(filePath,key1); break;
|
|
|
+ case "local":uploadLocal(filePath,key1); break;
|
|
|
+ }
|
|
|
+ }
|
|
|
+ public void upload2(String filePath, String key1) {
|
|
|
+ log.info("开始上传文件 源路径:{},目标路径:{},type:{}" , filePath,key1,type);
|
|
|
+ switch (type){
|
|
|
+ case "oss":upload2Oss(filePath,key1); break;
|
|
|
+ case "aws": uploadAws(filePath,key1); break;
|
|
|
+ case "local":uploadLocal(filePath,key1); break;
|
|
|
+ }
|
|
|
+ }
|
|
|
+ public void delete(String key1) throws IOException{
|
|
|
+ switch (type){
|
|
|
+ case "oss":deleteOss(key1); break;
|
|
|
+ case "aws": deleteS3Object(key1); break;
|
|
|
+ case "local":FileUtil.del(key1); break;
|
|
|
+ }
|
|
|
+ }
|
|
|
+ public int deleteFile(String prefix){
|
|
|
+ switch (type){
|
|
|
+ case "oss":deleteOssFile(prefix); break;
|
|
|
+ case "aws": deleteS3Object(prefix); break;
|
|
|
+ case "local":FileUtil.del(prefix); break;
|
|
|
+ }
|
|
|
+ return 1;
|
|
|
+ }
|
|
|
+
|
|
|
+ public void deleteOss(String objectName){
|
|
|
OSSClient ossClient = new OSSClient(point, key, secrey);
|
|
|
- // ossObject包含文件所在的存储空间名称、文件名称、文件元信息以及一个输入流。
|
|
|
- try{
|
|
|
- boolean exist = ossClient.doesObjectExist(bucket, objectName);
|
|
|
- return exist;
|
|
|
- }catch (Exception e){
|
|
|
- log.error("s4判断是否存在key异常,key=" + objectName, e);
|
|
|
+ try {
|
|
|
+ ossClient.deleteObject(bucket, objectName);
|
|
|
+ } catch (Exception e) {
|
|
|
+ log.error("OSS删除文件失败,key=" + objectName);
|
|
|
}finally {
|
|
|
if(ossClient != null){
|
|
|
ossClient.shutdown();
|
|
|
}
|
|
|
}
|
|
|
- return false;
|
|
|
}
|
|
|
- /**
|
|
|
- * 获取文件内容-阿里云
|
|
|
- * @param objectName
|
|
|
- * @return
|
|
|
- */
|
|
|
- public Long getSize(String objectName){
|
|
|
- //创建oss客户端
|
|
|
- if(objectName.contains(query)){
|
|
|
- objectName = objectName.replace(query,"");
|
|
|
+
|
|
|
+ public void deleteOssFile(String prefix){
|
|
|
+ OSSClient ossClient = new OSSClient(point, key, secrey);
|
|
|
+ ObjectListing objectListing = ossClient.listObjects(bucket, prefix);
|
|
|
+ List<OSSObjectSummary> sums = objectListing.getObjectSummaries();
|
|
|
+ try {
|
|
|
+ for (OSSObjectSummary s : sums) {
|
|
|
+ delete(s.getKey());
|
|
|
+ }
|
|
|
+ } catch (IOException e) {
|
|
|
+ e.printStackTrace();
|
|
|
}
|
|
|
+ }
|
|
|
+
|
|
|
+ public void uploadOss(byte[] data,String objectName){
|
|
|
OSSClient ossClient = new OSSClient(point, key, secrey);
|
|
|
- // ossObject包含文件所在的存储空间名称、文件名称、文件元信息以及一个输入流。
|
|
|
- try{
|
|
|
- OSSObject object = ossClient.getObject(bucket, objectName);
|
|
|
- ObjectMetadata objectMetadata = object.getObjectMetadata();
|
|
|
- return objectMetadata.getContentLength();
|
|
|
- }catch (Exception e){
|
|
|
- log.error("s4判断是否存在key异常,key=" + objectName, e);
|
|
|
+ try {
|
|
|
+ ossClient.putObject(bucket, objectName, new ByteArrayInputStream(data));
|
|
|
+ } catch (Exception e) {
|
|
|
+ log.error("oss上传文件失败", e);
|
|
|
}finally {
|
|
|
if(ossClient != null){
|
|
|
ossClient.shutdown();
|
|
|
}
|
|
|
}
|
|
|
- return 0L;
|
|
|
}
|
|
|
-
|
|
|
- /**
|
|
|
- * 从阿里云oss下载文件到本地
|
|
|
- * @param objectName 云端文件k地址
|
|
|
- * @param localPath 本地文件地址
|
|
|
- * @return
|
|
|
- */
|
|
|
- public boolean downFormAli(String objectName, String localPath){
|
|
|
- OSSClient ossClient = new OSSClient(point, key, secrey);
|
|
|
+ public void uploadAws(byte[] data,String objectName){
|
|
|
+ BasicAWSCredentials awsCreds = new BasicAWSCredentials(s3key, s3secrey);
|
|
|
+ AmazonS3 s3 = AmazonS3ClientBuilder.standard()
|
|
|
+ .withCredentials(new AWSStaticCredentialsProvider(awsCreds))
|
|
|
+ .withRegion(Regions.EU_WEST_2)
|
|
|
+ .build();
|
|
|
try {
|
|
|
- com.aliyun.oss.model.GetObjectRequest request = new com.aliyun.oss.model.GetObjectRequest(bucket,objectName);
|
|
|
- File file = new File(localPath);
|
|
|
- if(!file.getParentFile().exists()){
|
|
|
- file.getParentFile().mkdirs();
|
|
|
- file = new File(localPath);
|
|
|
- }
|
|
|
- ossClient.getObject(request, file);
|
|
|
- return true;
|
|
|
+ com.amazonaws.services.s3.model.ObjectMetadata metadata = new com.amazonaws.services.s3.model.ObjectMetadata();
|
|
|
+ s3.putObject(this.bucket, objectName, new ByteArrayInputStream(data), metadata);
|
|
|
}catch (Exception e){
|
|
|
- log.error("阿里云oss文件下载失败,key=" + objectName, e);
|
|
|
+ log.error("s3上传文件失败", e);
|
|
|
}finally {
|
|
|
- if(ossClient != null){
|
|
|
- ossClient.shutdown();
|
|
|
+ if(s3 != null){
|
|
|
+ s3.shutdown();
|
|
|
+ }
|
|
|
+ }
|
|
|
+ }
|
|
|
+ public void uploadLocal(byte[] data,String key1){
|
|
|
+ InputStream in = new ByteArrayInputStream(data);
|
|
|
+ File file = new File(key1);
|
|
|
+ String path = key1.substring(0, key1.lastIndexOf("/"));
|
|
|
+ if (!file.exists()) {
|
|
|
+ new File(path).mkdir();
|
|
|
+ }
|
|
|
+ FileOutputStream fos = null;
|
|
|
+ try {
|
|
|
+ fos = new FileOutputStream(file);
|
|
|
+ int len = 0;
|
|
|
+ byte[] buf = new byte[1024];
|
|
|
+ while ((len = in.read(buf)) != -1) {
|
|
|
+ fos.write(buf, 0, len);
|
|
|
+ }
|
|
|
+ fos.flush();
|
|
|
+ } catch (Exception e) {
|
|
|
+ e.printStackTrace();
|
|
|
+ } finally {
|
|
|
+ if (null != fos) {
|
|
|
+ try {
|
|
|
+ fos.close();
|
|
|
+ } catch (IOException e) {
|
|
|
+ e.printStackTrace();
|
|
|
+ }
|
|
|
}
|
|
|
}
|
|
|
- return false;
|
|
|
}
|
|
|
|
|
|
|
|
|
- public void uploadOss(String filePath, String key1){
|
|
|
+ public void uploadOss(String filePath, String key1){
|
|
|
OSSClient ossClient = new OSSClient(point, key, secrey);
|
|
|
try {
|
|
|
- log.info("upload-to-oss:file-path:{},oss-path:{}",filePath,key1);
|
|
|
File file = new File(filePath);
|
|
|
if (!file.exists()) {
|
|
|
- log.info("upload-to-oss:file-path:{},oss-path:{},filePath不存在!",filePath,key1);
|
|
|
+ log.error("要上传的文件不存在:" + filePath);
|
|
|
return;
|
|
|
}
|
|
|
ObjectMetadata metadata = new ObjectMetadata();
|
|
|
+ if(filePath.contains(".jpg")){
|
|
|
+ metadata.setContentType("image/jpeg");
|
|
|
+ }
|
|
|
ossClient.putObject(bucket, key1, new File(filePath), metadata);
|
|
|
+
|
|
|
} catch (Exception e) {
|
|
|
- e.printStackTrace();
|
|
|
+ log.error(e.toString() + filePath);
|
|
|
} finally {
|
|
|
ossClient.shutdown();
|
|
|
}
|
|
|
}
|
|
|
- public void delete(String objectName){
|
|
|
- if(objectName.contains(query)){
|
|
|
- objectName = objectName.replace(query,"");
|
|
|
+
|
|
|
+ public void uploadAws(String filePath, String key1){
|
|
|
+ try{
|
|
|
+ uploadS3File(filePath, key1);
|
|
|
+ }catch (Exception e){
|
|
|
+ e.printStackTrace();
|
|
|
}
|
|
|
+ }
|
|
|
+ public void uploadLocal(String filePath, String key1){
|
|
|
+ try {
|
|
|
+ File srcFile = new File(filePath);
|
|
|
+ File file = new File(localPath + key1);
|
|
|
+ FileUtils.copyFile(srcFile,file);
|
|
|
+ }catch (Exception e){
|
|
|
+ e.printStackTrace();
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+
|
|
|
+ public void uploadSdkOss(String filePath, String key1){
|
|
|
OSSClient ossClient = new OSSClient(point, key, secrey);
|
|
|
try {
|
|
|
- ossClient.deleteObject(bucket, objectName);
|
|
|
+ File file = new File(filePath);
|
|
|
+ if (!file.exists()) {
|
|
|
+ log.error("要上传的文件不存在:" + filePath);
|
|
|
+ return;
|
|
|
+ }
|
|
|
+ ObjectMetadata metadata = new ObjectMetadata();
|
|
|
+ if(filePath.contains(".jpg")){
|
|
|
+ metadata.setContentType("image/jpeg");
|
|
|
+ }
|
|
|
+ ossClient.putObject(bucketSdk, key1, new File(filePath), metadata);
|
|
|
+
|
|
|
} catch (Exception e) {
|
|
|
- log.error("OSS删除文件失败,key=" + objectName);
|
|
|
+ log.error(e.toString() + filePath);
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+ public void upload2Oss(String filePath, String key1){
|
|
|
+ OSSClient ossClient = new OSSClient(point, key, secrey);
|
|
|
+ try {
|
|
|
+ ObjectMetadata metadata = new ObjectMetadata();
|
|
|
+ if(filePath.contains(".jpg")){
|
|
|
+ metadata.setContentType("image/jpeg");
|
|
|
+ }
|
|
|
+ if(filePath.contains(".mp4")){
|
|
|
+ metadata.setContentType("video/mp4");
|
|
|
+ }
|
|
|
+ if(filePath.contains(".mp3")){
|
|
|
+ metadata.setContentType("audio/mp3");
|
|
|
+ }
|
|
|
+ ossClient.putObject(bucket, key1, new File(filePath), metadata);
|
|
|
+ } catch (Exception e) {
|
|
|
+ log.error(e.toString() + filePath);
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+
|
|
|
+
|
|
|
+
|
|
|
+
|
|
|
+ //海外亚马逊s3
|
|
|
+
|
|
|
+
|
|
|
+ /**
|
|
|
+ * s3上传文件
|
|
|
+ * @param filePath
|
|
|
+ * @param key1
|
|
|
+ * @throws IOException
|
|
|
+ */
|
|
|
+ private void uploadS3File(String filePath, String key1) throws Exception {
|
|
|
+ /**
|
|
|
+ * 创建s3对象
|
|
|
+ */
|
|
|
+ BasicAWSCredentials awsCreds = new BasicAWSCredentials(s3key, s3secrey);
|
|
|
+ AmazonS3 s3 = AmazonS3ClientBuilder.standard()
|
|
|
+ .withCredentials(new AWSStaticCredentialsProvider(awsCreds))
|
|
|
+ .withRegion(Regions.EU_WEST_2)
|
|
|
+ .build();
|
|
|
+ try{
|
|
|
+ File file = new File(filePath);
|
|
|
+ if(!file.exists()){
|
|
|
+ log.info("要上传s3的文件不存在");
|
|
|
+ return;
|
|
|
+ }
|
|
|
+
|
|
|
+ // 设置文件并设置公读
|
|
|
+ com.amazonaws.services.s3.model.ObjectMetadata metadata = new com.amazonaws.services.s3.model.ObjectMetadata();
|
|
|
+ if(filePath.contains(".jpg")){
|
|
|
+ metadata.setContentType("image/jpeg");
|
|
|
+ }
|
|
|
+ if(filePath.contains(".png")){
|
|
|
+ metadata.setContentType("image/png");
|
|
|
+ }
|
|
|
+ PutObjectRequest request = new PutObjectRequest(s3bucket, key1, file);
|
|
|
+ request.withCannedAcl(CannedAccessControlList.PublicRead);
|
|
|
+ request.withMetadata(metadata);
|
|
|
+
|
|
|
+ // 上传文件
|
|
|
+ com.amazonaws.services.s3.model.PutObjectResult putObjectResult = s3.putObject(request);
|
|
|
+ if (StringUtils.isNotEmpty(putObjectResult.getETag())) {
|
|
|
+ log.info("s3上传文件成功:" + key1);
|
|
|
+ }
|
|
|
+ }catch (Exception e){
|
|
|
+ throw e;
|
|
|
}finally {
|
|
|
- if(ossClient != null){
|
|
|
- ossClient.shutdown();
|
|
|
+ s3.shutdown();
|
|
|
+ }
|
|
|
+
|
|
|
+ }
|
|
|
+
|
|
|
+ /**
|
|
|
+ * 删除单个文件
|
|
|
+ *
|
|
|
+ * @param objectName 文件路径[ eg: /head/xxxx.jpg ]
|
|
|
+ * @return
|
|
|
+ */
|
|
|
+ public void deleteS3Object(String objectName) {
|
|
|
+
|
|
|
+ /**
|
|
|
+ * 创建s3对象
|
|
|
+ */
|
|
|
+ BasicAWSCredentials awsCreds = new BasicAWSCredentials(s3key, s3secrey);
|
|
|
+ AmazonS3 s3 = AmazonS3ClientBuilder.standard()
|
|
|
+ .withCredentials(new AWSStaticCredentialsProvider(awsCreds))
|
|
|
+ .withRegion(Regions.EU_WEST_2)
|
|
|
+ .build();
|
|
|
+
|
|
|
+ if (objectName.startsWith("/")) {
|
|
|
+ objectName = objectName.substring(1);
|
|
|
+ }
|
|
|
+
|
|
|
+ try {
|
|
|
+ s3.deleteObject(s3bucket, objectName);
|
|
|
+ } catch (Exception e) {
|
|
|
+ log.error("s3删除文件失败,key="+objectName, e);
|
|
|
+ }finally {
|
|
|
+ if(s3 != null){
|
|
|
+ s3.shutdown();
|
|
|
}
|
|
|
}
|
|
|
}
|
|
|
|
|
|
+
|
|
|
+
|
|
|
+ /**
|
|
|
+ * 获取文件类型
|
|
|
+ */
|
|
|
+ public static String getContentType(String filePath){
|
|
|
+ FileNameMap fileNameMap = URLConnection.getFileNameMap();
|
|
|
+ String contentType = fileNameMap.getContentTypeFor(filePath);
|
|
|
+ System.out.println(contentType);
|
|
|
+ return contentType;
|
|
|
+
|
|
|
+ }
|
|
|
+
|
|
|
+
|
|
|
+
|
|
|
/**
|
|
|
* 获得文件列表-阿里云
|
|
|
* @return
|
|
@@ -190,12 +444,248 @@ public class UploadToOssUtil {
|
|
|
}
|
|
|
|
|
|
/**
|
|
|
+ * 获得文件列表-亚马逊
|
|
|
+ * @return
|
|
|
+ */
|
|
|
+ public List<String> listKeysFromAws(String sourcePath) {
|
|
|
+ List<String> keyList = new ArrayList<>();
|
|
|
+
|
|
|
+ BasicAWSCredentials awsCreds = new BasicAWSCredentials(s3key, s3secrey);
|
|
|
+ AmazonS3 s3 = AmazonS3ClientBuilder.standard()
|
|
|
+ .withCredentials(new AWSStaticCredentialsProvider(awsCreds))
|
|
|
+ .withRegion(Regions.EU_WEST_2)
|
|
|
+ .build();
|
|
|
+
|
|
|
+ try {
|
|
|
+ boolean flag = true;
|
|
|
+ String nextMaker = null;
|
|
|
+ com.amazonaws.services.s3.model.ListObjectsRequest listObjectsRequest = new com.amazonaws.services.s3.model.ListObjectsRequest();
|
|
|
+ listObjectsRequest.setBucketName(this.s3bucket);
|
|
|
+ listObjectsRequest.setPrefix(sourcePath);
|
|
|
+ listObjectsRequest.setMaxKeys(200);
|
|
|
+
|
|
|
+ do{
|
|
|
+ listObjectsRequest.setMarker(nextMaker);
|
|
|
+ com.amazonaws.services.s3.model.ObjectListing objectListing = s3.listObjects(listObjectsRequest);
|
|
|
+ List<S3ObjectSummary> objectSummaries = objectListing.getObjectSummaries();
|
|
|
+ List<String> collect =objectSummaries.stream().map(summary->{
|
|
|
+ return summary.getKey();
|
|
|
+ }).collect(Collectors.toList());
|
|
|
+ if(CollUtil.isNotEmpty(collect)){
|
|
|
+ keyList.addAll(collect);
|
|
|
+ }
|
|
|
+ nextMaker = objectListing.getNextMarker();
|
|
|
+ flag = objectListing.isTruncated();
|
|
|
+ }while (flag);
|
|
|
+ }catch (Exception e){
|
|
|
+ log.error("获取文件列表失败,path="+sourcePath, e);
|
|
|
+ }finally {
|
|
|
+ if(s3 != null){
|
|
|
+ s3.shutdown();
|
|
|
+ }
|
|
|
+ }
|
|
|
+ return keyList;
|
|
|
+ }
|
|
|
+
|
|
|
+ /**
|
|
|
+ * 获得文件列表-阿里云
|
|
|
+ * @return
|
|
|
+ */
|
|
|
+ public List<String> listKeysFromLocal(String sourcePath) {
|
|
|
+ List<String> keyList = new ArrayList<>();
|
|
|
+ return keyList;
|
|
|
+ }
|
|
|
+
|
|
|
+ /**
|
|
|
+ * <p>
|
|
|
+ 拷贝目录
|
|
|
+ * </p>
|
|
|
+ * @author dengsixing
|
|
|
+ * @date 2022/1/18
|
|
|
+ * @param sourcePath
|
|
|
+ * @param targetPath
|
|
|
+ **/
|
|
|
+ public void copyFiles(String sourcePath, String targetPath) throws IOException {
|
|
|
+ StorageType storageType = StorageType.get(type);
|
|
|
+ switch (storageType){
|
|
|
+ case OSS:
|
|
|
+ this.copyFilesFromAli(sourcePath, targetPath);
|
|
|
+ break;
|
|
|
+ case AWS:
|
|
|
+ this.copyFilesFromAws(sourcePath, targetPath);
|
|
|
+ break;
|
|
|
+ case LOCAL: this.copyFilesFromLocal(sourcePath, targetPath);
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+ /**
|
|
|
+ * <p>
|
|
|
+ 拷贝文件
|
|
|
+ * </p>
|
|
|
+ * @author dengsixing
|
|
|
+ * @date 2022/1/18
|
|
|
+ * @param sourceKey
|
|
|
+ * @param targetKey
|
|
|
+ **/
|
|
|
+ public void copyObject(String sourceKey, String targetKey) throws IOException {
|
|
|
+ StorageType storageType = StorageType.get(type);
|
|
|
+ switch (storageType){
|
|
|
+ case OSS:
|
|
|
+ this.copyObjectFromAli(sourceKey, targetKey);
|
|
|
+ break;
|
|
|
+ case AWS:
|
|
|
+ this.copyObjectFromAws(sourceKey, targetKey);
|
|
|
+ break;
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+ /**
|
|
|
+ * <p>
|
|
|
+ 拷贝-阿里云
|
|
|
+ * </p>
|
|
|
+ * @author dengsixing
|
|
|
+ * @date 2022/1/18
|
|
|
+ * @param sourcePath
|
|
|
+ * @param targetPath
|
|
|
+ **/
|
|
|
+ public void copyObjectFromAli(String sourcePath, String targetPath) throws IOException {
|
|
|
+
|
|
|
+ // 创建OSSClient实例。
|
|
|
+ OSSClient ossClient = new OSSClient(point, key, secrey);
|
|
|
+ // 复制文件
|
|
|
+ log.info("开始复制:" + sourcePath);
|
|
|
+ ossClient.copyObject(this.bucket, sourcePath, this.bucket, targetPath);
|
|
|
+ log.info("复制成功:" + sourcePath);
|
|
|
+ ossClient.shutdown();
|
|
|
+ }
|
|
|
+
|
|
|
+
|
|
|
+
|
|
|
+ /**
|
|
|
+ * <p>
|
|
|
+ 拷贝-阿里云
|
|
|
+ * </p>
|
|
|
+ * @author dengsixing
|
|
|
+ * @date 2022/1/18
|
|
|
+ * @param sourcePath
|
|
|
+ * @param targetPath
|
|
|
+ **/
|
|
|
+ public void copyFilesFromAli(String sourcePath, String targetPath) throws IOException {
|
|
|
+
|
|
|
+ //获取源文件列表
|
|
|
+ List<String> sourceKeyList = this.listKeysFromAli(sourcePath);
|
|
|
+ if(CollUtil.isEmpty(sourceKeyList)){
|
|
|
+ return;
|
|
|
+ }
|
|
|
+ // 创建OSSClient实例。
|
|
|
+ OSSClient ossClient = new OSSClient(point, key, secrey);
|
|
|
+ // 复制文件
|
|
|
+ sourceKeyList.parallelStream().forEach(key -> {
|
|
|
+ log.info("开始复制:" + key);
|
|
|
+ ossClient.copyObject(this.bucket, key, this.bucket, key.replace(sourcePath, targetPath));
|
|
|
+ log.info("复制成功:" + key);
|
|
|
+ });
|
|
|
+
|
|
|
+ ossClient.shutdown();
|
|
|
+ }
|
|
|
+
|
|
|
+
|
|
|
+ /**
|
|
|
+ * <p>
|
|
|
+ 拷贝-亚马逊
|
|
|
+ * </p>
|
|
|
+ * @author dengsixing
|
|
|
+ * @date 2022/1/18
|
|
|
+ * @param sourcePath
|
|
|
+ * @param targetPath
|
|
|
+ **/
|
|
|
+ public void copyFilesFromAws(String sourcePath, String targetPath){
|
|
|
+
|
|
|
+ try {
|
|
|
+ List<String> sourceKeyList = this.listKeysFromAws(sourcePath);
|
|
|
+ /**
|
|
|
+ * 创建s3对象
|
|
|
+ */
|
|
|
+ BasicAWSCredentials awsCreds = new BasicAWSCredentials(s3key, s3secrey);
|
|
|
+ AmazonS3 s3 = AmazonS3ClientBuilder.standard()
|
|
|
+ .withCredentials(new AWSStaticCredentialsProvider(awsCreds))
|
|
|
+ .withRegion(Regions.EU_WEST_2)//s3 地区位置
|
|
|
+ .build();
|
|
|
+
|
|
|
+ // 复制文件
|
|
|
+ sourceKeyList.parallelStream().forEach(key -> {
|
|
|
+ log.info("开始复制:" + key);
|
|
|
+ s3.copyObject(this.s3bucket, key, this.s3bucket, key.replace(sourcePath, targetPath));
|
|
|
+ log.info("复制成功:" + key);
|
|
|
+ });
|
|
|
+ s3.shutdown();
|
|
|
+ } catch (Exception ase) {
|
|
|
+ log.error("amazonS拷贝异常 " + ase.getMessage(), ase);
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+ /**
|
|
|
+ * <p>
|
|
|
+ 拷贝-亚马逊
|
|
|
+ * </p>
|
|
|
+ * @author dengsixing
|
|
|
+ * @date 2022/1/18
|
|
|
+ * @param sourceKey
|
|
|
+ * @param targetKey
|
|
|
+ **/
|
|
|
+ public void copyObjectFromAws(String sourceKey, String targetKey){
|
|
|
+ BasicAWSCredentials awsCreds = new BasicAWSCredentials(s3key, s3secrey);
|
|
|
+ AmazonS3 s3 = AmazonS3ClientBuilder.standard()
|
|
|
+ .withCredentials(new AWSStaticCredentialsProvider(awsCreds))
|
|
|
+ .withRegion(Regions.EU_WEST_2)//s3 地区位置
|
|
|
+ .build();
|
|
|
+ // 复制文件
|
|
|
+ log.info("开始复制:" + sourceKey);
|
|
|
+ s3.copyObject(this.s3bucket, sourceKey, this.s3bucket, targetKey);
|
|
|
+ log.info("复制成功:" + sourceKey);
|
|
|
+ s3.shutdown();
|
|
|
+ }
|
|
|
+
|
|
|
+ /**
|
|
|
+ * <p>
|
|
|
+ 拷贝-本地
|
|
|
+ * </p>
|
|
|
+ * @author dengsixing
|
|
|
+ * @date 2022/1/18
|
|
|
+ * @param sourcePath
|
|
|
+ * @param targetPath
|
|
|
+ **/
|
|
|
+ public void copyFilesFromLocal(String sourcePath, String targetPath) throws IOException {
|
|
|
+ // TODO: 2022/1/21
|
|
|
+
|
|
|
+ }
|
|
|
+
|
|
|
+ /**
|
|
|
+ * 获取文件内容
|
|
|
+ * @param objectName
|
|
|
+ * @return
|
|
|
+ */
|
|
|
+ public String getObjectContent( String objectName){
|
|
|
+ log.info("下载文件:"+objectName);
|
|
|
+ StorageType storageType = StorageType.get(type);
|
|
|
+ switch (storageType){
|
|
|
+ case OSS:
|
|
|
+ return this.getObjectContentFromAli(bucket, objectName);
|
|
|
+ case AWS:
|
|
|
+ return this.getObjectContentFromAws(s3bucket, objectName);
|
|
|
+ case LOCAL:
|
|
|
+ return this.getObjectContentFromLocal(objectName);
|
|
|
+ }
|
|
|
+ return null;
|
|
|
+ }
|
|
|
+
|
|
|
+ /**
|
|
|
* 获取文件内容-阿里云
|
|
|
* @param bucketName
|
|
|
* @param objectName
|
|
|
* @return
|
|
|
*/
|
|
|
- public String getObjectContent(String bucketName, String objectName){
|
|
|
+ public String getObjectContentFromAli(String bucketName, String objectName){
|
|
|
//创建oss客户端
|
|
|
OSSClient ossClient = new OSSClient(point, key, secrey);
|
|
|
InputStream objectContent = null;
|
|
@@ -224,17 +714,247 @@ public class UploadToOssUtil {
|
|
|
return contentJson.toString();
|
|
|
}
|
|
|
|
|
|
- public void uploadFileOss(File file) {
|
|
|
- if(file.isFile()){
|
|
|
- String ossPath = file.getPath();
|
|
|
- ossPath = ossPath.replace("/mnt/","");
|
|
|
- ossPath = ossPath.replace("\\","/");
|
|
|
- this.uploadOss(file.getPath(),ossPath);
|
|
|
- }else {
|
|
|
- File[] files = file.listFiles();
|
|
|
- for (File file1 : files) {
|
|
|
- uploadFileOss(file1);
|
|
|
+ /**
|
|
|
+ * 获取文件内容-阿里云
|
|
|
+ * @param objectName
|
|
|
+ * @return
|
|
|
+ */
|
|
|
+ public boolean existOnAli(String objectName){
|
|
|
+ //创建oss客户端
|
|
|
+ OSSClient ossClient = new OSSClient(point, key, secrey);
|
|
|
+ // ossObject包含文件所在的存储空间名称、文件名称、文件元信息以及一个输入流。
|
|
|
+ try{
|
|
|
+ boolean exist = ossClient.doesObjectExist(bucket, objectName);
|
|
|
+ return exist;
|
|
|
+ }catch (Exception e){
|
|
|
+ log.error("s4判断是否存在key异常,key=" + objectName, e);
|
|
|
+ }finally {
|
|
|
+ if(ossClient != null){
|
|
|
+ ossClient.shutdown();
|
|
|
+ }
|
|
|
+ }
|
|
|
+ return false;
|
|
|
+ }
|
|
|
+
|
|
|
+ /**
|
|
|
+ * 获取文件内容-亚马逊
|
|
|
+ * @param bucketName
|
|
|
+ * @param objectName
|
|
|
+ * @return
|
|
|
+ */
|
|
|
+ public String getObjectContentFromAws(String bucketName, String objectName){
|
|
|
+
|
|
|
+ try {
|
|
|
+ /**
|
|
|
+ * 创建s3对象
|
|
|
+ */
|
|
|
+ BasicAWSCredentials awsCreds = new BasicAWSCredentials(s3key, s3secrey);
|
|
|
+ AmazonS3 s3 = AmazonS3ClientBuilder.standard()
|
|
|
+ .withCredentials(new AWSStaticCredentialsProvider(awsCreds))
|
|
|
+ .withRegion(Regions.EU_WEST_2)
|
|
|
+ .build();
|
|
|
+
|
|
|
+ GetObjectRequest request = new GetObjectRequest(bucketName,objectName);
|
|
|
+ S3Object object = s3.getObject(request);
|
|
|
+ S3ObjectInputStream inputStream = object.getObjectContent();
|
|
|
+ StringBuilder content = new StringBuilder();
|
|
|
+ try(BufferedReader reader = new BufferedReader(new InputStreamReader(inputStream))){
|
|
|
+ while (true) {
|
|
|
+ String line = reader.readLine();
|
|
|
+ if (line == null) break;
|
|
|
+ content.append(line);
|
|
|
+ }
|
|
|
+ } catch (IOException e) {
|
|
|
+ log.error("读取aws文件流失败", e);
|
|
|
+ }
|
|
|
+ return content.toString();
|
|
|
+ } catch (Exception ase) {
|
|
|
+ log.error("amazonS3下载文件异常 " + ase.getMessage(), ase);
|
|
|
+ }
|
|
|
+ return null;
|
|
|
+
|
|
|
+ }
|
|
|
+
|
|
|
+ /**
|
|
|
+ * 获取文件内容-亚马逊
|
|
|
+ * @param objectName
|
|
|
+ * @return
|
|
|
+ */
|
|
|
+ public boolean existOnAws(String objectName){
|
|
|
+
|
|
|
+ BasicAWSCredentials awsCreds = new BasicAWSCredentials(s3key, s3secrey);
|
|
|
+ AmazonS3 s3 = AmazonS3ClientBuilder.standard()
|
|
|
+ .withCredentials(new AWSStaticCredentialsProvider(awsCreds))
|
|
|
+ .withRegion(Regions.EU_WEST_2)
|
|
|
+ .build();
|
|
|
+ try {
|
|
|
+ boolean exist = s3.doesObjectExist(s3bucket, objectName);
|
|
|
+ return exist;
|
|
|
+ }catch (Exception e){
|
|
|
+ log.error("s4判断是否存在key异常,key=" + objectName, e);
|
|
|
+ }finally {
|
|
|
+ if(s3 != null){
|
|
|
+ s3.shutdown();
|
|
|
+ }
|
|
|
+ }
|
|
|
+ return false;
|
|
|
+ }
|
|
|
+
|
|
|
+ /**
|
|
|
+ * 判断key是否存在
|
|
|
+ * @param key
|
|
|
+ * @return
|
|
|
+ */
|
|
|
+ public boolean existKey(String key){
|
|
|
+ StorageType storageType = StorageType.get(type);
|
|
|
+ switch (storageType){
|
|
|
+ case OSS:
|
|
|
+ return this.existOnAli(key);
|
|
|
+ case AWS:
|
|
|
+ return this.existOnAws(key);
|
|
|
+ default:
|
|
|
+ return false;
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+ /**
|
|
|
+ * 获取文件内容-本地
|
|
|
+ * @param objectName
|
|
|
+ * @return
|
|
|
+ */
|
|
|
+ public String getObjectContentFromLocal(String objectName){
|
|
|
+ // TODO: 2022/1/21
|
|
|
+ return null;
|
|
|
+ }
|
|
|
+
|
|
|
+ /**
|
|
|
+ * oss下载文件到本地
|
|
|
+ * @param objectName
|
|
|
+ * @param localPath
|
|
|
+ */
|
|
|
+ public boolean download(String objectName, String localPath){
|
|
|
+ StorageType storageType = StorageType.get(this.type);
|
|
|
+ switch (storageType){
|
|
|
+ case OSS:
|
|
|
+ return this.downFormAli(objectName, localPath);
|
|
|
+ case AWS:
|
|
|
+ return this.downFromS3(objectName, localPath);
|
|
|
+ }
|
|
|
+ return false;
|
|
|
+ }
|
|
|
+
|
|
|
+ /**
|
|
|
+ * 从阿里云oss下载文件到本地
|
|
|
+ * @param objectName 云端文件k地址
|
|
|
+ * @param localPath 本地文件地址
|
|
|
+ * @return
|
|
|
+ */
|
|
|
+ public boolean downFormAli(String objectName, String localPath){
|
|
|
+ OSSClient ossClient = new OSSClient(point, key, secrey);
|
|
|
+ try {
|
|
|
+ com.aliyun.oss.model.GetObjectRequest request = new com.aliyun.oss.model.GetObjectRequest(bucket,objectName);
|
|
|
+ ossClient.getObject(request, new File(localPath));
|
|
|
+ return true;
|
|
|
+ }catch (Exception e){
|
|
|
+ log.error("阿里云oss文件下载失败,key=" + objectName, e);
|
|
|
+ }finally {
|
|
|
+ if(ossClient != null){
|
|
|
+ ossClient.shutdown();
|
|
|
}
|
|
|
}
|
|
|
+ return false;
|
|
|
}
|
|
|
+
|
|
|
+ /**
|
|
|
+ * 从s3下载文件到本地
|
|
|
+ * @param objectName 云端文件k地址
|
|
|
+ * @param localPath 本地文件地址
|
|
|
+ * @return
|
|
|
+ */
|
|
|
+ public boolean downFromS3(String objectName, String localPath) {
|
|
|
+ BasicAWSCredentials awsCreds = new BasicAWSCredentials(s3key, s3secrey);
|
|
|
+ AmazonS3 s3 = AmazonS3ClientBuilder.standard()
|
|
|
+ .withCredentials(new AWSStaticCredentialsProvider(awsCreds))
|
|
|
+ .withRegion(Regions.EU_WEST_2)
|
|
|
+ .build();
|
|
|
+ try {
|
|
|
+ GetObjectRequest request = new GetObjectRequest(this.s3bucket,objectName);
|
|
|
+ s3.getObject(request,new File(localPath));
|
|
|
+ return true;
|
|
|
+ } catch (Exception e) {
|
|
|
+ log.error("amazonS3下载文件失败,key=" + objectName, e);
|
|
|
+ }finally {
|
|
|
+ if(s3 != null){
|
|
|
+ s3.shutdown();
|
|
|
+ }
|
|
|
+ }
|
|
|
+ return false;
|
|
|
+ }
|
|
|
+
|
|
|
+ @Value("${upload.query-path}")
|
|
|
+ private String query;
|
|
|
+
|
|
|
+ public Long getSize(String objectName){
|
|
|
+ StorageType storageType = StorageType.get(this.type);
|
|
|
+ switch (storageType){
|
|
|
+ case OSS:
|
|
|
+ return this.getSizeOss(objectName);
|
|
|
+ case AWS:
|
|
|
+ return this.getSizeS3(objectName);
|
|
|
+ }
|
|
|
+ return 0L;
|
|
|
+ }
|
|
|
+
|
|
|
+
|
|
|
+ /**
|
|
|
+ * 获取文件内容-阿里云
|
|
|
+ * @param objectName
|
|
|
+ * @return
|
|
|
+ */
|
|
|
+ public Long getSizeOss(String objectName){
|
|
|
+ //创建oss客户端
|
|
|
+ if(objectName.contains(query)){
|
|
|
+ objectName = objectName.replace(query,"");
|
|
|
+ }
|
|
|
+ OSSClient ossClient = new OSSClient(point, key, secrey);
|
|
|
+ // ossObject包含文件所在的存储空间名称、文件名称、文件元信息以及一个输入流。
|
|
|
+ try{
|
|
|
+ OSSObject object = ossClient.getObject(bucket, objectName);
|
|
|
+ ObjectMetadata objectMetadata = object.getObjectMetadata();
|
|
|
+ return objectMetadata.getContentLength();
|
|
|
+ }catch (Exception e){
|
|
|
+ log.error("s4判断是否存在key异常,key=" + objectName, e);
|
|
|
+ }finally {
|
|
|
+ if(ossClient != null){
|
|
|
+ ossClient.shutdown();
|
|
|
+ }
|
|
|
+ }
|
|
|
+ return 0L;
|
|
|
+ }
|
|
|
+ /**
|
|
|
+ * 获取文件内容-阿里云
|
|
|
+ * @param objectName
|
|
|
+ * @return
|
|
|
+ */
|
|
|
+ public Long getSizeS3(String objectName){
|
|
|
+ BasicAWSCredentials awsCreds = new BasicAWSCredentials(s3key, s3secrey);
|
|
|
+ AmazonS3 s3 = AmazonS3ClientBuilder.standard()
|
|
|
+ .withCredentials(new AWSStaticCredentialsProvider(awsCreds))
|
|
|
+ .withRegion(Regions.EU_WEST_2)
|
|
|
+ .build();
|
|
|
+ try {
|
|
|
+ S3Object object = s3.getObject(new GetObjectRequest(s3bucket, objectName));
|
|
|
+ com.amazonaws.services.s3.model.ObjectMetadata objectMetadata = object.getObjectMetadata();
|
|
|
+ return objectMetadata.getInstanceLength();
|
|
|
+
|
|
|
+ } catch (Exception e) {
|
|
|
+ log.error("amazonS3下载文件失败,key=" + objectName, e);
|
|
|
+ }finally {
|
|
|
+ if(s3 != null){
|
|
|
+ s3.shutdown();
|
|
|
+ }
|
|
|
+ }
|
|
|
+ return 0L;
|
|
|
+ }
|
|
|
+
|
|
|
}
|