package com.fdkankan.download.service.impl; import cn.hutool.core.collection.CollUtil; import cn.hutool.core.collection.ConcurrentHashSet; import cn.hutool.core.exceptions.ExceptionUtil; import cn.hutool.core.util.StrUtil; import cn.hutool.json.JSONObject; import cn.hutool.json.JSONUtil; import com.alibaba.fastjson.JSON; import com.fdkankan.common.bean.DownLoadProgressBean; import com.fdkankan.common.bean.DownLoadTaskBean; import com.fdkankan.common.constant.SceneDownloadProgressStatus; import com.fdkankan.common.constant.SceneFrom; import com.fdkankan.common.constant.SceneResolution; import com.fdkankan.common.constant.ServerCode; import com.fdkankan.common.constant.UploadFilePath; import com.fdkankan.common.response.ResultData; import com.fdkankan.common.util.FileUtils; import com.fdkankan.download.bean.CurrentDownloadNumUtil; import com.fdkankan.download.bean.ImageType; import com.fdkankan.download.bean.ImageTypeDetail; import com.fdkankan.fyun.constant.StorageType; import com.fdkankan.fyun.oss.UploadToOssUtil; import com.fdkankan.redis.constant.RedisKey; import com.fdkankan.redis.util.RedisUtil; import com.fdkankan.scene.api.dto.SceneInfoDTO; import com.fdkankan.scene.api.feign.SceneUserSceneClient; import com.google.common.collect.Lists; import java.io.File; import java.io.FileInputStream; import java.math.BigDecimal; import java.net.URLEncoder; import java.util.ArrayList; import java.util.Calendar; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Objects; import java.util.Set; import java.util.concurrent.Callable; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.Future; import java.util.concurrent.atomic.AtomicInteger; import java.util.stream.Collectors; import lombok.extern.slf4j.Slf4j; import lombok.var; import org.apache.tools.zip.ZipOutputStream; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.cloud.context.config.annotation.RefreshScope; import org.springframework.scheduling.annotation.Async; import org.springframework.stereotype.Service; import org.springframework.web.client.RestTemplate; /** *

* TODO *

* * @author dengsixing * @since 2022/2/22 **/ @RefreshScope @Slf4j @Service public class SceneDownloadHandlerServiceImpl { private static final String[] prefixArr = new String[]{ UploadFilePath.DATA_VIEW_PATH, UploadFilePath.VOICE_VIEW_PATH, UploadFilePath.VIDEOS_VIEW_PATH, UploadFilePath.IMG_VIEW_PATH, UploadFilePath.USER_VIEW_PATH, }; private static final List imageTypes = Lists.newArrayList(); static{ imageTypes.add(ImageType.builder().name("2k_face").size("2048").ranges(new String[]{"0", "511", "1023", "1535"}).build()); imageTypes.add(ImageType.builder().name("1k_face").size("1024").ranges(new String[]{"0", "511"}).build()); imageTypes.add(ImageType.builder().name("512_face").size("512").ranges(new String[]{"0"}).build()); } @Autowired private SceneUserSceneClient sceneUserSceneClient; @Value("${path.v4school}") private String v4localPath; @Value("${path.zip-local}") private String zipLocalFormat; @Value("${path.zip-oss}") private String zipOssFormat; @Value("${path.zip-root}") private String wwwroot; @Value("${zip.nThreads}") private int zipNthreads; @Value("${oss.bucket:4dkankan}") private String bucket; @Value("${upload.type:oss}") private String uploadType; @Value("${download.config.resource-url}") private String resourceUrl; @Value("${download.config.public-url}") private String publicUrl; @Value("${download.config.exe-name}") private String exeName; @Value("${download.config.exe-content}") private String exeContent; @Autowired RestTemplate restTemplate; @Autowired RedisUtil redisUtil; @Autowired UploadToOssUtil uploadToOssUtil; @Async("sceneDownLoadExecutror") public void download(DownLoadTaskBean downLoadTaskBean){ //场景码 String num = null; try { num = downLoadTaskBean.getNum(); log.info("场景下载开始 - num[{}] - threadName[{}]", num, Thread.currentThread().getName()); long startTime = Calendar.getInstance().getTimeInMillis(); //执行场景下载逻辑 this.downloadHandler(downLoadTaskBean); //耗时 long consumeTime = Calendar.getInstance().getTimeInMillis() - startTime; log.info("场景下载结束 - num[{}] - threadName[{}] - consumeTime[{}]", num, Thread.currentThread().getName(), consumeTime); }catch (Exception e){ log.error(ExceptionUtil.stacktraceToString(e)); }finally { if(StrUtil.isNotEmpty(num)){ //本地正在下载任务出队 CurrentDownloadNumUtil.removeSceneNum(num); //删除正在下载任务 redisUtil.lRemove(RedisKey.SCENE_DOWNLOAD_ING, 1, num); } } } public void downloadHandler(DownLoadTaskBean downLoadTaskBean) throws Exception{ String num = downLoadTaskBean.getNum(); //zip包路径 String zipPath = null; try { Set cacheKeys = new ConcurrentHashSet<>(); Map> allFiles = this.getAllFiles(num, v4localPath); List ossFilePaths = allFiles.get("ossFilePaths"); List v3localFilePaths = allFiles.get("v3localFilePaths"); //key总个数 int total = ossFilePaths.size() + v3localFilePaths.size(); AtomicInteger count = new AtomicInteger(0); //定义压缩包 zipPath = String.format(this.zipLocalFormat, num); File zipFile = new File(zipPath); if(!zipFile.getParentFile().exists()){ zipFile.getParentFile().mkdirs(); } ZipOutputStream out = new ZipOutputStream(zipFile); // JSONObject getInfoJson = this.zipGetInfoJson(out, this.wwwroot, num); String sceneJsonData = uploadToOssUtil.getObjectContent(bucket, String.format(UploadFilePath.DATA_VIEW_PATH, num) + "scene.json"); JSONObject sceneJson = JSONUtil.parseObj(sceneJsonData); String resolution = "4k"; String sceneForm = sceneJson.getStr("sceneFrom"); if(StrUtil.isNotEmpty(sceneForm) && SceneFrom.PRO.code().equals(sceneForm)){ resolution = "2k"; } //国际版存在已经切好图的情况,下载时不需要再切图,只需要把文件直接下载下来打包就可以了 String sceneResolution = sceneJson.getStr("sceneResolution"); if(SceneResolution.TILES.code().equals(sceneResolution)){ resolution = "notNeadCut"; } int imagesVersion = -1; // TODO: 2022/3/29 V4版本目前没有imagesVersion字段,暂时用version字段替代 // if(getInfoJson.getInt("imagesVersion") != null){ // imagesVersion = getInfoJson.getInt("imagesVersion"); // } Integer version = sceneJson.getInt("version"); if(Objects.nonNull(version)){ imagesVersion = version; } long start = Calendar.getInstance().getTimeInMillis(); //固定文件写入 this.zipLocalFiles(out, v3localFilePaths, v4localPath, num, count, total); long end1 = Calendar.getInstance().getTimeInMillis(); log.info("打包固定文件耗时, num:{}, time:{}", num, end1 - start); //oss文件写入 this.zipOssFiles(out, ossFilePaths, num, count, total, resolution, imagesVersion, cacheKeys); long end2 = Calendar.getInstance().getTimeInMillis(); log.info("打包oss文件耗时, num:{}, time:{}", num, end2 - end1); //重新写入scene.json(去掉密码访问设置) this.zipSceneJson(out, this.wwwroot, num, sceneJson); //写入启动命令 this.zipBat(out, num); out.close(); //上传压缩包 String uploadPath = String.format(this.zipOssFormat, num); uploadToOssUtil.uploadBySh(zipPath, uploadPath); //更新进度100 String url = this.publicUrl + uploadPath; this.updateProgress(null, num, SceneDownloadProgressStatus.DOWNLOAD_SUCCESS.code(), url); // TODO: 2022/5/24 v3 停止后要开启-----------------------start //更新用户场景已下载次数 // platformUserClient.updateDownloadNum(userId, 1); // // //更新下载log状态为成功 // sceneUserSceneClient.updateSceneDownloadLog(num, DownloadStatus.SUCCESS.code(), url, null); // TODO: 2022/5/24 v3 停止后要开启-----------------------end }catch (Exception e){ //更新进度为下载失败 this.updateProgress( null, num, SceneDownloadProgressStatus.DOWNLOAD_FAILED.code(), null); //更新下载log状态为成功 // TODO: 2022/5/24 v3 停止后要开启-----------------------start // sceneUserSceneClient.updateSceneDownloadLog(num, DownloadStatus.FAILD.code(), null, ExceptionUtil.stacktraceToString(e)); // TODO: 2022/5/24 v3 停止后要开启-----------------------send throw e; }finally { if(StrUtil.isNotBlank(zipPath)){ //删除本地zip包 FileUtils.deleteFile(zipPath); } } } private void zipOssFiles(ZipOutputStream out, List ossFilePaths, String num, AtomicInteger count, int total, String resolution, int imagesVersion, Set cacheKeys) throws Exception{ String imageNumPath = String.format(UploadFilePath.IMG_VIEW_PATH, num); ExecutorService executorService = Executors.newFixedThreadPool(this.zipNthreads); List futureList = new ArrayList<>(); for (String filePath : ossFilePaths) { Callable call = new Callable() { @Override public Boolean call() throws Exception { zipOssFilesHandler(out, num, count, total, resolution, imagesVersion, cacheKeys,filePath, imageNumPath); return true; } }; futureList.add(executorService.submit(call)); } //这里一定要加阻塞,不然会导致oss文件还没打包好,主程序已经结束返回了 for (Future future : futureList) { future.get(); } } private void zipOssFilesHandler(ZipOutputStream out, String num, AtomicInteger count, int total, String resolution, int imagesVersion, Set cacheKeys, String filePath, String imageNumPath) throws Exception{ //更新进度 this.updateProgress(new BigDecimal(count.incrementAndGet()).divide(new BigDecimal(total), 6, BigDecimal.ROUND_HALF_UP), num, SceneDownloadProgressStatus.DOWNLOADING.code(), null); //某个目录不需要打包 if(filePath.contains(imageNumPath + "panorama/panorama_edit/")) return; //切图 if(!"notNeadCut".equals(resolution)){ if((filePath.contains(imageNumPath + "panorama/") && filePath.contains("tiles/" + resolution)) || filePath.contains(imageNumPath + "tiles/" + resolution + "/")) { this.processImage(filePath, out, resolution, imagesVersion, cacheKeys); return; } } //其他文件打包 this.ProcessFiles(num, filePath, out, this.wwwroot, cacheKeys); } private void zipLocalFiles(ZipOutputStream out, List v3localFilePaths, String v3localPath, String num, AtomicInteger count, int total) throws Exception{ for (String v3localFilePath : v3localFilePaths) { try (FileInputStream in = new FileInputStream(new File(v3localFilePath));){ this.zipInputStream(out, v3localFilePath.replace(v3localPath, ""), in); }catch (Exception e){ throw e; } //更新进度 this.updateProgress( new BigDecimal(count.incrementAndGet()).divide(new BigDecimal(total), 6, BigDecimal.ROUND_HALF_UP), num, SceneDownloadProgressStatus.DOWNLOAD_COMPRESSING.code(), null); } //写入code.txt this.zipBytes(out, "code.txt", num.getBytes()); } private void zipBat(ZipOutputStream out, String num) throws Exception{ String batContent = String.format(this.exeContent, num); this.zipBytes(out, exeName, batContent.getBytes()); //更新进度为90% this.updateProgress(new BigDecimal("0.9").divide(new BigDecimal("0.8"), 6, BigDecimal.ROUND_HALF_UP), num, SceneDownloadProgressStatus.DOWNLOAD_COMPRESSING.code(), null); } private Map> getAllFiles(String num, String v3localPath) throws Exception{ //列出oss所有文件路径 List ossFilePaths = new ArrayList<>(); for (String prefix : prefixArr) { prefix = String.format(prefix, num); List keys = uploadToOssUtil.listKeys(prefix); if(CollUtil.isEmpty(keys)){ continue; } if(StorageType.AWS.code().equals(this.uploadType)){ keys = keys.stream().filter(key->{ if(key.contains("x-oss-process")){ return false; } return true; }).collect(Collectors.toList()); } ossFilePaths.addAll(keys); } //列出v3local所有文件路径 File file = new File(v3localPath); List v3localFilePaths = FileUtils.list(file); HashMap> map = new HashMap<>(); map.put("ossFilePaths", ossFilePaths); map.put("v3localFilePaths", v3localFilePaths); return map; } private JSONObject zipGetInfoJson(ZipOutputStream out, String root, String num) throws Exception{ ResultData sceneViewInfo = sceneUserSceneClient.getSceneViewInfo(num); if(!sceneViewInfo.getSuccess()){ throw new Exception(ServerCode.FEIGN_REQUEST_FAILD.message()); } SceneInfoDTO data = sceneViewInfo.getData(); JSONObject getInfoJson = null; if(Objects.isNull(data)){ getInfoJson = new JSONObject(); }else { getInfoJson = JSONUtil.parseObj(data); } getInfoJson.set("sceneScheme", 3); getInfoJson.set("needKey", 0); getInfoJson.set("sceneKey",""); //写入getInfo.json String getInfoJsonPath = root + String.format(UploadFilePath.DATA_VIEW_PATH, num) + "getInfo.json"; this.zipBytes(out, getInfoJsonPath, getInfoJson.toString().getBytes()); return getInfoJson; } private void zipSceneJson(ZipOutputStream out, String root, String num, JSONObject sceneJson) throws Exception{ //访问密码置0 JSONObject controls = sceneJson.getJSONObject("controls"); controls.set("showLock", 0); String sceneJsonPath = root + String.format(UploadFilePath.DATA_VIEW_PATH, num) + "scene.json"; this.zipBytes(out, sceneJsonPath, sceneJson.toString().getBytes()); } private void processImage(String key, ZipOutputStream out, String resolution, int imagesVersion, Set imgKeys) throws Exception{ if(key.contains("x-oss-process") || key.endsWith("/")){ return; } String fileName = key.substring(key.lastIndexOf("/")+1, key.indexOf(".")); String ext = key.substring(key.lastIndexOf(".")); String[] arr = fileName.split("_skybox"); String dir = arr[0]; String num = arr[1]; if(StrUtil.isEmpty(fileName) || StrUtil.isEmpty(ext) || (".jpg".equals(ext) && ".png".equals(ext)) || StrUtil.isEmpty(dir) || StrUtil.isEmpty(num)){ throw new Exception("本地下载图片资源不符合规则,key:" + key); } for (ImageType imageType : imageTypes) { List items = Lists.newArrayList(); String[] ranges = imageType.getRanges(); for(int i = 0; i < ranges.length; i++){ String x = ranges[i]; for(int j = 0; j < ranges.length; j++){ String y = ranges[j]; items.add( ImageTypeDetail.builder() .i(String.valueOf(i)) .j(String.valueOf(j)) .x(x) .y(y) .build() ); } } for (ImageTypeDetail item : items) { String par = "?x-oss-process=image/resize,m_lfit,w_" + imageType.getSize() + "/crop,w_512,h_512,x_" + item.getX() + ",y_" + item.getY(); if(StorageType.AWS.code().equals(uploadType)){ par += "&imagesVersion="+ imagesVersion; } var url = this. resourceUrl + key; StorageType storageType = StorageType.get(uploadType); switch (storageType){ case OSS: url += par; break; case AWS: url += URLEncoder.encode(par.replace("/", "@"), "UTF-8"); break; } var fky = key.split("/" + resolution + "/")[0] + "/" + dir + "/" + imageType.getName() + num + "_" + item.getI() + "_" + item.getJ() + ext; if(imgKeys.contains(fky)){ continue; } imgKeys.add(fky); this.zipBytes(out, wwwroot + fky, FileUtils.getBytesFromUrl(url)); } } } public void ProcessFiles(String num, String key, ZipOutputStream out, String prefix, Set cacheKeys) throws Exception{ if(cacheKeys.contains(key)){ return; } if(key.equals(String.format(UploadFilePath.DATA_VIEW_PATH, num) + "scene.json")){ return; } cacheKeys.add(key); String url = this.resourceUrl + key + "?t=" + Calendar.getInstance().getTimeInMillis(); if(key.contains("hot.json") || key.contains("link-scene.json")){ String content = FileUtils.getStringFromUrl(url); content.replace(publicUrl, "") // .replace(publicUrl+"v3/", "") .replace("https://spc.html","spc.html") .replace("https://smobile.html", "smobile.html"); zipBytes(out, prefix + key, content.getBytes()); }else{ zipBytes(out, prefix + key, FileUtils.getBytesFromUrl(url)); } } public void updateProgress(BigDecimal precent, String num, Integer status, String url){ SceneDownloadProgressStatus progressStatus = SceneDownloadProgressStatus.get(status); switch (progressStatus){ case DOWNLOAD_SUCCESS: precent = new BigDecimal("100"); break; case DOWNLOAD_FAILED: precent = new BigDecimal("0"); break; default: precent = precent.multiply(new BigDecimal("0.8")).multiply(new BigDecimal("100")); } DownLoadProgressBean progress = null; String key = String.format(RedisKey.PREFIX_DOWNLOAD_PROGRESS_V4, num); String progressStr = redisUtil.get(key); if(StrUtil.isEmpty(progressStr)){ progress = DownLoadProgressBean.builder().percent(precent.intValue()).status(status).url(url).build(); }else{ progress = JSONUtil.toBean(progressStr, DownLoadProgressBean.class); //如果下载失败,进度不变 if(status == SceneDownloadProgressStatus.DOWNLOAD_FAILED.code() && progress.getPercent() != null){ precent = new BigDecimal(progress.getPercent()); } progress.setPercent(precent.intValue()); progress.setStatus(status); progress.setUrl(url); } redisUtil.set(key, JSONUtil.toJsonStr(progress)); } public void zipInputStream(ZipOutputStream out, String key, FileInputStream in) throws Exception { out.putNextEntry(new org.apache.tools.zip.ZipEntry(key)); byte[] bytes = new byte[1024]; int b = 0; while ((b = in.read(bytes)) != -1) { out.write(bytes, 0, b); } } public synchronized void zipBytes(ZipOutputStream out, String key, byte[] bytes) throws Exception { out.putNextEntry(new org.apache.tools.zip.ZipEntry(key)); out.write(bytes); } }