场景批量下载

pom.xml

@@ -207,6 +207,12 @@
             <version>3.0.0-SNAPSHOT</version>
         </dependency>
 
+        <dependency>
+            <groupId>com.fdkankan</groupId>
+            <artifactId>4dkankan-utils-rabbitmq</artifactId>
+            <version>3.0.0-SNAPSHOT</version>
+        </dependency>
+
     </dependencies>
 
     <dependencyManagement>

src/main/java/com/fdkankan/download/service/IDownloadService.java

@@ -3,5 +3,15 @@ package com.fdkankan.download.service;
 public interface IDownloadService {
     void downloadHandler(String num) throws Exception;
 
+    /**
+     *
+     * @param num
+     * @param path
+     * @param resolution 分辨路 2k 4k
+     * @param subPath 需要切图的子目录 tiles-场景本身全景图  panorama/d9rHSlO544803/tiles-场景关联全景图
+     * @throws Exception
+     */
+    void cutImg(String num, String path, String resolution, String subPath) throws Exception;
+
 }
 

src/main/java/com/fdkankan/download/service/impl/DownloadServiceImpl.java

@@ -90,6 +90,8 @@ public class DownloadServiceImpl implements IDownloadService {
     private String exeName;
     @Value("${path.zip-oss}")
     private String zipOssFormat;
+//    @Value("${cutImgType}")
+//    private String cutImgType;
 
     @Autowired
     private IScenePlusService scenePlusService;
@@ -101,6 +103,7 @@ public class DownloadServiceImpl implements IDownloadService {
     private RedisUtil redisUtil;
 
 
+
     @Override
     public void downloadHandler(String num) throws Exception {
 
@@ -120,13 +123,13 @@ public class DownloadServiceImpl implements IDownloadService {
 
             Set<String> cacheKeys = new ConcurrentHashSet<>();
 
-            Map<String, List<String>> allFiles = this.getAllFiles(num, v4localPath, bucket);
-            List<String> ossFilePaths = allFiles.get("ossFilePaths");
-            List<String> v4localFilePaths = allFiles.get("localFilePaths");
+//            Map<String, List<String>> allFiles = this.getAllFiles(num, v4localPath, bucket);
+//            List<String> ossFilePaths = allFiles.get("ossFilePaths");
+//            List<String> v4localFilePaths = allFiles.get("localFilePaths");
 
             //key总个数
-            int total = ossFilePaths.size() + v4localFilePaths.size();
-            AtomicInteger count = new AtomicInteger(0);
+//            int total = ossFilePaths.size() + v4localFilePaths.size();
+//            AtomicInteger count = new AtomicInteger(0);
 
             //定义压缩包
             zipPath = String.format(this.zipLocalFormat, num);
@@ -149,11 +152,11 @@ public class DownloadServiceImpl implements IDownloadService {
             }
 
             //固定文件写入
-            this.zipLocalFiles(v4localFilePaths, v4localPath, num, count, total, "v4");
+            this.zipLocalFiles(num, "v4");
             log.info("打包固定文件耗时, num:{}, time:{}", num, timer.intervalRestart());
 
             //oss文件写入
-            this.zipOssFiles(ossFilePaths, num, count, total, resolution, imagesVersion, cacheKeys, "v4");
+            this.zipOssFiles(num, resolution, imagesVersion, cacheKeys, "v4");
             log.info("打包oss文件耗时, num:{}, time:{}", num, timer.intervalRestart());
 
             //重新写入scene.json（去掉密码访问设置）
@@ -196,70 +199,74 @@ public class DownloadServiceImpl implements IDownloadService {
         FileUtil.writeUtf8String(JSON.toJSONString(sceneViewInfo, SerializerFeature.WriteNullStringAsEmpty, SerializerFeature.WriteNullNumberAsZero), String.format(this.sourceLocal, num, this.wwwroot + sceneJsonPath));
     }
 
-    private void zipOssFiles(List<String> ossFilePaths, String num, AtomicInteger count,
-                             int total, String resolution, int imagesVersion, Set<String> cacheKeys, String version) throws Exception{
-        if(CollUtil.isEmpty(ossFilePaths)){
-            return;
-        }
+    private void zipOssFiles(String num, String resolution, int imagesVersion, Set<String> cacheKeys, String version) throws Exception{
+
+        fYunFileService.downloadFileByCommand(String.format(sourceLocal, num, this.wwwroot), String.format(UploadFilePath.VIEW_PATH, num));
+
+        //特殊文件处理
+        this.reWriteFile(num);
+
+        //切图
+        String filePath = String.format(sourceLocal, num, this.wwwroot + String.format(UploadFilePath.IMG_VIEW_PATH, num) + "tiles/");
+        this.cutImg(num, filePath, resolution, "tiles");
+
+        //切图-场景关联
+
+    }
+    @Override
+    public void cutImg(String num, String path, String resolution, String subPath) throws Exception {
+
         String imageNumPath = String.format(UploadFilePath.IMG_VIEW_PATH, num);
-        if("v3".equals(version)){
-            imageNumPath = String.format("images/images%s/", num);
-        }
+        String tilesOssPath = imageNumPath.concat(subPath + "/" + resolution);
+        List<String> cubeList = fYunFileService.listRemoteFiles(tilesOssPath);
         ExecutorService executorService = Executors.newFixedThreadPool(this.zipNthreads);
         List<Future> futureList = new ArrayList<>();
-        for (String filePath : ossFilePaths) {
-            String finalImageNumPath = imageNumPath;
+        for (String key : cubeList) {
             Callable<Boolean> call = new Callable() {
                 @Override
                 public Boolean call() throws Exception {
-                    zipOssFilesHandler(num, count, total, resolution,
-                            imagesVersion, cacheKeys,filePath, finalImageNumPath, version);
+                    processImage(key, resolution, path);
                     return true;
                 }
             };
             futureList.add(executorService.submit(call));
         }
-        //这里一定要加阻塞，不然会导致oss文件还没打包好，主程序已经结束返回了
-        Boolean zipSuccess = true;
         for (Future future : futureList) {
             try {
                 future.get();
             }catch (Exception e){
-                log.error("打包oss文件失败", e);
-                zipSuccess = false;
+                log.error("切图失败", e);
+                executorService.shutdownNow();
+                throw new Exception("切图失败");
             }
         }
-        if(!zipSuccess){
-            throw new Exception("打包oss文件失败");
-        }
     }
 
-    private void zipOssFilesHandler(String num,
-                                    AtomicInteger count, int total, String resolution,
-                                    int imagesVersion, Set<String> cacheKeys,
-                                    String filePath, String imageNumPath, String version) throws Exception{
-
-        if(filePath.endsWith("/")){
-            return;
-        }
-
-        //某个目录不需要打包
-        if(filePath.contains(imageNumPath + "panorama/panorama_edit/"))
-            return;
-
-        //切图
-        if(!"notNeadCut".equals(resolution)){
-            if((filePath.contains(imageNumPath + "panorama/") && filePath.contains("tiles/" + resolution))
-                    || filePath.contains(imageNumPath + "tiles/" + resolution + "/")) {
-                this.processImage(num, filePath, resolution, imagesVersion, cacheKeys);
-                return;
-            }
-        }
-
-        //其他文件打包
-        this.ProcessFiles(num, filePath, this.wwwroot, cacheKeys);
-
-    }
+//    private void zipOssFilesHandler(String num, String resolution,
+//                                    int imagesVersion, Set<String> cacheKeys,
+//                                    String filePath, String imageNumPath, String version) throws Exception{
+//
+//        if(filePath.endsWith("/")){
+//            return;
+//        }
+//
+//        //某个目录不需要打包
+//        if(filePath.contains(imageNumPath + "panorama/panorama_edit/"))
+//            return;
+//
+//        //切图
+//        if(!"notNeadCut".equals(resolution)){
+//            if((filePath.contains(imageNumPath + "panorama/") && filePath.contains("tiles/" + resolution))
+//                    || filePath.contains(imageNumPath + "tiles/" + resolution + "/")) {
+//                this.processImage(num, filePath, resolution, imagesVersion, cacheKeys);
+//                return;
+//            }
+//        }
+//
+//        //其他文件打包
+//        this.ProcessFiles(num, filePath, this.wwwroot, cacheKeys);
+//
+//    }
 
     public void ProcessFiles(String num, String key, String prefix, Set<String> cacheKeys) throws Exception{
         if(cacheKeys.contains(key)){
@@ -291,7 +298,18 @@ public class DownloadServiceImpl implements IDownloadService {
         }
     }
 
-    private void processImage(String sceneNum, String key, String resolution, int imagesVersion, Set<String> imgKeys) throws Exception{
+    private void reWriteFile(String num){
+        String filePath = String.format(sourceLocal, num, this.wwwroot + String.format(UploadFilePath.USER_VIEW_PATH, num) + "hot.json");
+        if(FileUtil.exist(filePath)){
+            String content = FileUtil.readUtf8String(filePath);
+            content = content.replace(publicUrl, "")
+                    .replace("https://spc.html","spc.html")
+                    .replace("https://smobile.html", "smobile.html");
+            FileUtil.writeUtf8String(content, filePath);
+        }
+    }
+
+    private void processImage(String key, String resolution, String path) throws Exception{
 
         if(key.contains("x-oss-process") || key.endsWith("/")){
             return;
@@ -315,11 +333,6 @@ public class DownloadServiceImpl implements IDownloadService {
                 continue;
             }
 
-//            imageTypes.add(ImageType.builder().name("4k_face").size("4096").ranges(new String[]{"0", "511", "1023", "1535", "2047","2559","3071","3583"}).build());
-//            imageTypes.add(ImageType.builder().name("2k_face").size("2048").ranges(new String[]{"0", "511", "1023", "1535"}).build());
-//            imageTypes.add(ImageType.builder().name("1k_face").size("1024").ranges(new String[]{"0", "511"}).build());
-//            imageTypes.add(ImageType.builder().name("512_face").size("512").ranges(new String[]{"0"}).build());
-
             List<ImageTypeDetail> items = Lists.newArrayList();
             String[] ranges = imageType.getRanges();
             for(int i = 0; i < ranges.length; i++){
@@ -338,9 +351,9 @@ public class DownloadServiceImpl implements IDownloadService {
             }
             for (ImageTypeDetail item : items) {
                 String par = "?x-oss-process=image/resize,m_lfit,w_" + imageType.getSize() + "/crop,w_512,h_512,x_" + item.getX() + ",y_" + item.getY();
-                if(FYunTypeEnum.AWS.code().equals(uploadType)){
-                    par += "&imagesVersion="+ imagesVersion;
-                }
+//                if(FYunTypeEnum.AWS.code().equals(uploadType)){
+//                    par += "&imagesVersion="+ imagesVersion;
+//                }
 
                 var url = this.resourceUrl + key;
                 FYunTypeEnum storageType = FYunTypeEnum.get(uploadType);
@@ -352,13 +365,10 @@ public class DownloadServiceImpl implements IDownloadService {
                         url += URLEncoder.encode(par.replace("/", "@"), "UTF-8");
                         break;
                 }
-                //scene_view_data/t-jp-WXWxmOuj4Kf/images/tiles/0/4k_face_0_0_0.jpg
-                var fky = key.split("/" + resolution + "/")[0] + "/" + dir + "/" + imageType.getName() +  num + "_" + item.getI()  + "_" + item.getJ() + ext;
-                if(imgKeys.contains(fky)){
-                    continue;
-                }
-                imgKeys.add(fky);
-                this.downloadFile(url, String.format(sourceLocal, sceneNum, this.wwwroot + fky));
+                //key.split("/" + resolution + "/")[0] + "/" +
+                var fky = dir + "/" + imageType.getName() +  num + "_" + item.getI()  + "_" + item.getJ() + ext;
+                this.downloadFile(url, path + fky);
+//                this.downloadFile(url, path);
             }
 
         }
@@ -373,16 +383,10 @@ public class DownloadServiceImpl implements IDownloadService {
         HttpUtil.downloadFile(url, path);
     }
 
-    private void zipLocalFiles(List<String> localFilePaths, String v3localPath, String num, AtomicInteger count, int total, String version) throws Exception{
+    private void zipLocalFiles(String num, String version) throws Exception{
         String sourcePath = String.format(this.sourceLocal, num, "");
         String localPath = "v4".equals(version) ? this.v4localPath : this.v3localPath;
-        for (String localFilePath : localFilePaths) {
-            try (FileInputStream in = new FileInputStream(new File(localFilePath));){
-                FileUtil.copy(localFilePath, localFilePath.replace(localPath, sourcePath), true);
-            }catch (Exception e){
-                throw e;
-            }
-        }
+        FileUtil.copyContent(FileUtil.newFile(localPath), FileUtil.newFile(sourcePath), true);
         //写入code.txt
         FileUtil.writeUtf8String(num, String.format(sourceLocal, num, "code.txt"));
     }

src/main/java/com/fdkankan/download/service/impl/GenSceneRunnerImpl.java

@@ -13,6 +13,7 @@ import com.fdkankan.download.entity.*;
 import com.fdkankan.download.service.*;
 import com.fdkankan.redis.util.RedisLockUtil;
 import com.fdkankan.redis.util.RedisUtil;
+import com.fdkankan.rabbitmq.util.RabbitMqProducer;
 import lombok.extern.slf4j.Slf4j;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.boot.CommandLineRunner;
@@ -44,6 +45,8 @@ public class GenSceneRunnerImpl implements CommandLineRunner {
     private RedisLockUtil redisLockUtil;
     @Autowired
     private IDownloadService downloadService;
+    @Autowired
+    private RabbitMqProducer mqProducer;
 
     private final static ThreadPoolExecutor threadPoolExecutor = ExecutorBuilder.create().setCorePoolSize(5).setMaxPoolSize(5).build();
 
@@ -119,4 +122,9 @@ public class GenSceneRunnerImpl implements CommandLineRunner {
             }
         }
     }
+
+//    private void send(String zipPath){
+//        mqProducer.sendByWorkQueue("rsync-scene", );
+//    }
+
 }

src/main/java/com/fdkankan/download/util/ImgUtil.java

@@ -1,7 +1,5 @@
 package com.fdkankan.download.util;
 
-import cn.hutool.system.HostInfo;
-import cn.hutool.system.SystemUtil;
 import cn.hutool.system.oshi.OshiUtil;
 import com.fdkankan.common.util.CmdUtils;
 import oshi.software.os.OperatingSystem;

src/main/resources/application.yml

@@ -16,6 +16,20 @@ spring:
         max-wait: -1 # 连接池最大阻塞等待时间（使用负值表示没有限制）
     lettuce:
       shutdown-timeout: 0ms
+  rabbitmq:
+    host: 120.24.144.164
+    port: 5672
+    username: admin
+    password: adminv41234
+    virtual-host: 4dkankan
+    connection-timeout: 0
+    listener:
+      simple:
+        prefetch: 5
+        max-concurrency: 10
+        acknowledge-mode: manual #开启消费者手动确认
+    #开启消息投递确认机制
+    publisher-confirm-type: correlated
 
 mybatis-flex:
   datasource:
@@ -90,6 +104,8 @@ download:
       start http://127.0.0.1:9000/spg.html?m=%s
       http.exe -nc -p 9000 -r wwwroot
 
+#cutImgType: oss   #oss-阿里云接口切图  local-本地切图
+