package com.fdkankan.download.service.impl; import cn.hutool.core.collection.CollUtil; import cn.hutool.core.collection.ConcurrentHashSet; import cn.hutool.core.date.DateUtil; import cn.hutool.core.date.TimeInterval; import cn.hutool.core.io.FileUtil; import cn.hutool.core.util.StrUtil; import cn.hutool.core.util.ZipUtil; import cn.hutool.http.HttpUtil; import com.alibaba.fastjson.JSON; import com.alibaba.fastjson.serializer.SerializerFeature; import com.fdkankan.common.constant.CommonStatus; import com.fdkankan.common.constant.ErrorCode; import com.fdkankan.common.constant.SceneDownloadProgressStatus; import com.fdkankan.common.constant.SceneKind; import com.fdkankan.common.exception.BusinessException; import com.fdkankan.common.util.FileUtils; import com.fdkankan.download.CommonConstant; import com.fdkankan.download.bean.ImageType; import com.fdkankan.download.bean.ImageTypeDetail; import com.fdkankan.download.bean.SceneEditControlsBean; import com.fdkankan.download.bean.SceneViewInfoBean; import com.fdkankan.download.entity.ScenePlus; import com.fdkankan.download.entity.ScenePlusExt; import com.fdkankan.download.service.IDownloadService; import com.fdkankan.download.service.IScenePlusExtService; import com.fdkankan.download.service.IScenePlusService; import com.fdkankan.redis.constant.RedisKey; import com.fdkankan.redis.util.RedisUtil; import com.google.common.collect.Lists; import lombok.extern.slf4j.Slf4j; import lombok.var; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; import java.io.File; import java.io.FileInputStream; import java.math.BigDecimal; import java.net.URLEncoder; import java.util.*; import java.util.concurrent.Callable; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.Future; import java.util.concurrent.atomic.AtomicInteger; import java.util.stream.Collectors; import com.fdkankan.model.constants.UploadFilePath; import com.fdkankan.fyun.face.FYunFileServiceInterface; import com.fdkankan.fyun.constant.FYunTypeEnum; import javax.annotation.Resource; @Slf4j(topic = "IDownloadService") @Service public class DownloadServiceImpl implements IDownloadService { // private static final List imageTypes = Lists.newArrayList(); // static{ // imageTypes.add(ImageType.builder().name("4k_face").size("4096").ranges(new String[]{"0", "511", "1023", "1535", "2047","2559","3071","3583"}).build()); // imageTypes.add(ImageType.builder().name("2k_face").size("2048").ranges(new String[]{"0", "511", "1023", "1535"}).build()); // imageTypes.add(ImageType.builder().name("1k_face").size("1024").ranges(new String[]{"0", "511"}).build()); // imageTypes.add(ImageType.builder().name("512_face").size("512").ranges(new String[]{"0"}).build()); // } @Value("${path.v4school}") private String v4localPath; @Value("${fyun.type:oss}") private String uploadType; @Value("${path.zip-local}") private String zipLocalFormat; @Value("${path.source-local}") private String sourceLocal; @Value("${fyun.bucket:4dkankan}") private String bucket; @Value("${download.config.public-url}") private String publicUrl; @Value("${path.v3school:#{null}}") private String v3localPath; @Value("${zip.nThreads}") private int zipNthreads; @Value("${download.config.resource-url}") private String resourceUrl; @Value("${path.zip-root}") private String wwwroot; @Value("${download.config.exe-content}") private String exeContent; @Value("${download.config.exe-content-v3:#{null}}") private String exeContentV3; @Value("${download.config.exe-name}") private String exeName; @Value("${path.zip-oss}") private String zipOssFormat; @Autowired private IScenePlusService scenePlusService; @Autowired private IScenePlusExtService scenePlusExtService; @Resource private FYunFileServiceInterface fYunFileService; @Autowired private RedisUtil redisUtil; @Override public void downloadHandler(String num) throws Exception { //zip包路径 String zipPath = null; try { TimeInterval timer = DateUtil.timer(); //删除资源目录 FileUtil.del(String.format(this.sourceLocal, num, "")); ScenePlus scenePlus = scenePlusService.getByNum(num); if(Objects.isNull(scenePlus)) throw new BusinessException(ErrorCode.FAILURE_CODE_5005); ScenePlusExt scenePlusExt = scenePlusExtService.getByPlusId(scenePlus.getId()); String bucket = scenePlusExt.getYunFileBucket(); Set cacheKeys = new ConcurrentHashSet<>(); Map> allFiles = this.getAllFiles(num, v4localPath, bucket); List ossFilePaths = allFiles.get("ossFilePaths"); List v4localFilePaths = allFiles.get("localFilePaths"); //key总个数 int total = ossFilePaths.size() + v4localFilePaths.size(); AtomicInteger count = new AtomicInteger(0); //定义压缩包 zipPath = String.format(this.zipLocalFormat, num); File zipFile = new File(zipPath); if(!zipFile.getParentFile().exists()){ zipFile.getParentFile().mkdirs(); } SceneViewInfoBean sceneViewInfo = this.getSceneJson(num); String resolution = sceneViewInfo.getSceneResolution(); //国际版存在已经切好图的情况,下载时不需要再切图,只需要把文件直接下载下来打包就可以了 if(SceneKind.FACE.code().equals(sceneViewInfo.getSceneKind())){ resolution = "notNeadCut"; } int imagesVersion = -1; Integer version = sceneViewInfo.getVersion(); if(Objects.nonNull(version)){ imagesVersion = version; } //固定文件写入 this.zipLocalFiles(v4localFilePaths, v4localPath, num, count, total, "v4"); log.info("打包固定文件耗时, num:{}, time:{}", num, timer.intervalRestart()); //oss文件写入 this.zipOssFiles(ossFilePaths, num, count, total, resolution, imagesVersion, cacheKeys, "v4"); log.info("打包oss文件耗时, num:{}, time:{}", num, timer.intervalRestart()); //重新写入scene.json(去掉密码访问设置) this.zipSceneJson(num, sceneViewInfo); //写入启动命令 this.zipBat(num, "v4"); //打压缩包 ZipUtil.zip(String.format(this.sourceLocal, num, ""), zipPath); // TODO: 2024/1/4 生成的压缩包放哪里待定 // String uploadPath = String.format(this.zipOssFormat, num); // fYunFileService.uploadFileByCommand(bucket, zipPath, uploadPath); }catch (Exception e){ //更新进度为下载失败 throw e; }finally { FileUtil.del(zipPath); FileUtil.del(String.format(this.sourceLocal, num, "")); } } private void zipBat(String num, String version) throws Exception{ String batContent = String.format(this.exeContent, num); if("v3".equals(version)){ batContent = String.format(this.exeContentV3, num); } // this.zipBytes(out, exeName, batContent.getBytes()); FileUtil.writeUtf8String(batContent, String.format(this.sourceLocal, num, exeName)); } private void zipSceneJson(String num, SceneViewInfoBean sceneViewInfo) throws Exception{ //访问密码置0 SceneEditControlsBean controls = sceneViewInfo.getControls(); controls.setShowLock(CommonStatus.NO.code().intValue()); String sceneJsonPath = String.format(UploadFilePath.DATA_VIEW_PATH, num) + "scene.json"; FileUtil.writeUtf8String(JSON.toJSONString(sceneViewInfo, SerializerFeature.WriteNullStringAsEmpty, SerializerFeature.WriteNullNumberAsZero), String.format(this.sourceLocal, num, this.wwwroot + sceneJsonPath)); } private void zipOssFiles(List ossFilePaths, String num, AtomicInteger count, int total, String resolution, int imagesVersion, Set cacheKeys, String version) throws Exception{ if(CollUtil.isEmpty(ossFilePaths)){ return; } String imageNumPath = String.format(UploadFilePath.IMG_VIEW_PATH, num); if("v3".equals(version)){ imageNumPath = String.format("images/images%s/", num); } ExecutorService executorService = Executors.newFixedThreadPool(this.zipNthreads); List futureList = new ArrayList<>(); for (String filePath : ossFilePaths) { String finalImageNumPath = imageNumPath; Callable call = new Callable() { @Override public Boolean call() throws Exception { zipOssFilesHandler(num, count, total, resolution, imagesVersion, cacheKeys,filePath, finalImageNumPath, version); return true; } }; futureList.add(executorService.submit(call)); } //这里一定要加阻塞,不然会导致oss文件还没打包好,主程序已经结束返回了 Boolean zipSuccess = true; for (Future future : futureList) { try { future.get(); }catch (Exception e){ log.error("打包oss文件失败", e); zipSuccess = false; } } if(!zipSuccess){ throw new Exception("打包oss文件失败"); } } private void zipOssFilesHandler(String num, AtomicInteger count, int total, String resolution, int imagesVersion, Set cacheKeys, String filePath, String imageNumPath, String version) throws Exception{ if(filePath.endsWith("/")){ return; } //某个目录不需要打包 if(filePath.contains(imageNumPath + "panorama/panorama_edit/")) return; //切图 if(!"notNeadCut".equals(resolution)){ if((filePath.contains(imageNumPath + "panorama/") && filePath.contains("tiles/" + resolution)) || filePath.contains(imageNumPath + "tiles/" + resolution + "/")) { this.processImage(num, filePath, resolution, imagesVersion, cacheKeys); return; } } //其他文件打包 this.ProcessFiles(num, filePath, this.wwwroot, cacheKeys); } public void ProcessFiles(String num, String key, String prefix, Set cacheKeys) throws Exception{ if(cacheKeys.contains(key)){ return; } if(key.equals(String.format(UploadFilePath.DATA_VIEW_PATH, num) + "scene.json")){ return; } cacheKeys.add(key); String fileName = key.substring(key.lastIndexOf("/") + 1); String url = this.resourceUrl + key.replace(fileName, URLEncoder.encode(fileName, "UTF-8")) + "?t=" + Calendar.getInstance().getTimeInMillis(); if(key.contains("hot.json") || key.contains("link-scene.json")){ String content = fYunFileService.getFileContent(key); if(StrUtil.isEmpty(content)){ return; } content = content.replace(publicUrl, "") // .replace(publicUrl+"v3/", "") .replace("https://spc.html","spc.html") .replace("https://smobile.html", "smobile.html"); FileUtil.writeUtf8String(content, String.format(sourceLocal, num, prefix + key)); }else{ try { this.downloadFile(url, String.format(sourceLocal, num, prefix + key)); }catch (Exception e){ log.info("下载文件报错,path:{}", String.format(sourceLocal, num, prefix + key)); } } } private void processImage(String sceneNum, String key, String resolution, int imagesVersion, Set imgKeys) throws Exception{ if(key.contains("x-oss-process") || key.endsWith("/")){ return; } String fileName = key.substring(key.lastIndexOf("/")+1, key.indexOf("."));//0_skybox0.jpg String ext = key.substring(key.lastIndexOf(".")); String[] arr = fileName.split("_skybox"); String dir = arr[0]; //0 String num = arr[1]; //0 if(StrUtil.isEmpty(fileName) || StrUtil.isEmpty(ext) || (".jpg".equals(ext) && ".png".equals(ext)) || StrUtil.isEmpty(dir) || StrUtil.isEmpty(num)){ throw new Exception("本地下载图片资源不符合规则,key:" + key); } for (ImageType imageType : CommonConstant.imageTypes) { if(imageType.getName().equals("4k_face") && !"4k".equals(resolution)){ continue; } // imageTypes.add(ImageType.builder().name("4k_face").size("4096").ranges(new String[]{"0", "511", "1023", "1535", "2047","2559","3071","3583"}).build()); // imageTypes.add(ImageType.builder().name("2k_face").size("2048").ranges(new String[]{"0", "511", "1023", "1535"}).build()); // imageTypes.add(ImageType.builder().name("1k_face").size("1024").ranges(new String[]{"0", "511"}).build()); // imageTypes.add(ImageType.builder().name("512_face").size("512").ranges(new String[]{"0"}).build()); List items = Lists.newArrayList(); String[] ranges = imageType.getRanges(); for(int i = 0; i < ranges.length; i++){ String x = ranges[i]; for(int j = 0; j < ranges.length; j++){ String y = ranges[j]; items.add( ImageTypeDetail.builder() .i(String.valueOf(i)) .j(String.valueOf(j)) .x(x) .y(y) .build() ); } } for (ImageTypeDetail item : items) { String par = "?x-oss-process=image/resize,m_lfit,w_" + imageType.getSize() + "/crop,w_512,h_512,x_" + item.getX() + ",y_" + item.getY(); if(FYunTypeEnum.AWS.code().equals(uploadType)){ par += "&imagesVersion="+ imagesVersion; } var url = this.resourceUrl + key; FYunTypeEnum storageType = FYunTypeEnum.get(uploadType); switch (storageType){ case OSS: url += par; break; case AWS: url += URLEncoder.encode(par.replace("/", "@"), "UTF-8"); break; } //scene_view_data/t-jp-WXWxmOuj4Kf/images/tiles/0/4k_face_0_0_0.jpg var fky = key.split("/" + resolution + "/")[0] + "/" + dir + "/" + imageType.getName() + num + "_" + item.getI() + "_" + item.getJ() + ext; if(imgKeys.contains(fky)){ continue; } imgKeys.add(fky); this.downloadFile(url, String.format(sourceLocal, sceneNum, this.wwwroot + fky)); } } } public void downloadFile(String url, String path){ File file = new File(path); if(!file.getParentFile().exists()){ file.getParentFile().mkdirs(); } HttpUtil.downloadFile(url, path); } private void zipLocalFiles(List localFilePaths, String v3localPath, String num, AtomicInteger count, int total, String version) throws Exception{ String sourcePath = String.format(this.sourceLocal, num, ""); String localPath = "v4".equals(version) ? this.v4localPath : this.v3localPath; for (String localFilePath : localFilePaths) { try (FileInputStream in = new FileInputStream(new File(localFilePath));){ FileUtil.copy(localFilePath, localFilePath.replace(localPath, sourcePath), true); }catch (Exception e){ throw e; } } //写入code.txt FileUtil.writeUtf8String(num, String.format(sourceLocal, num, "code.txt")); } private SceneViewInfoBean getSceneJson(String num){ String sceneJsonData = redisUtil.get(String.format(RedisKey.SCENE_JSON, num)); if(StrUtil.isEmpty(sceneJsonData)){ sceneJsonData = fYunFileService.getFileContent(bucket, String.format(UploadFilePath.DATA_VIEW_PATH, num) + "scene.json"); } sceneJsonData = sceneJsonData.replace(this.publicUrl, ""); SceneViewInfoBean sceneInfoVO = JSON.parseObject(sceneJsonData, SceneViewInfoBean.class); sceneInfoVO.setScenePassword(null); if(Objects.isNull(sceneInfoVO.getFloorPlanAngle())){ sceneInfoVO.setFloorPlanAngle(0f); } if(Objects.isNull(sceneInfoVO.getFloorPlanCompass())){ sceneInfoVO.setFloorPlanCompass(0f); } SceneEditControlsBean controls = sceneInfoVO.getControls(); if(Objects.isNull(controls.getShowShare())){ controls.setShowShare(CommonStatus.YES.code().intValue()); } if(Objects.isNull(controls.getShowCapture())){ controls.setShowCapture(CommonStatus.YES.code().intValue()); } if(Objects.isNull(controls.getShowBillboardTitle())){ controls.setShowBillboardTitle(CommonStatus.YES.code().intValue()); } return sceneInfoVO; } private Map> getAllFiles(String num, String v4localPath, String bucket) throws Exception{ //列出oss所有文件路径 List ossFilePaths = new ArrayList<>(); for (String prefix : CommonConstant.prefixArr) { prefix = String.format(prefix, num); List keys = fYunFileService.listRemoteFiles(bucket, prefix); if(CollUtil.isEmpty(keys)){ continue; } if(FYunTypeEnum.AWS.code().equals(this.uploadType)){ keys = keys.stream().filter(key->{ if(key.contains("x-oss-process")){ return false; } return true; }).collect(Collectors.toList()); } ossFilePaths.addAll(keys); } //列出v3local所有文件路径 File file = new File(v4localPath); List localFilePaths = FileUtils.list(file); HashMap> map = new HashMap<>(); map.put("ossFilePaths", ossFilePaths); map.put("localFilePaths", localFilePaths); return map; } }