package com.fdkankan.download.service.impl;
import cn.hutool.core.collection.CollUtil;
import cn.hutool.core.collection.ConcurrentHashSet;
import cn.hutool.core.exceptions.ExceptionUtil;
import cn.hutool.core.util.StrUtil;
import cn.hutool.json.JSONObject;
import cn.hutool.json.JSONUtil;
import com.alibaba.fastjson.JSON;
import com.fdkankan.common.constant.ErrorCode;
import com.fdkankan.common.constant.SceneDownloadProgressStatus;
import com.fdkankan.common.constant.SceneFrom;
import com.fdkankan.common.constant.SceneResolution;
import com.fdkankan.common.constant.ServerCode;
import com.fdkankan.common.constant.UploadFilePath;
import com.fdkankan.common.exception.BusinessException;
import com.fdkankan.common.response.ResultData;
import com.fdkankan.common.util.FileUtils;
import com.fdkankan.download.bean.CurrentDownloadNumUtil;
import com.fdkankan.download.bean.DownLoadProgressBean;
import com.fdkankan.download.bean.DownLoadTaskBean;
import com.fdkankan.download.bean.ImageType;
import com.fdkankan.download.bean.ImageTypeDetail;
import com.fdkankan.download.entity.ScenePlus;
import com.fdkankan.download.entity.ScenePlusExt;
import com.fdkankan.download.service.IScenePlusExtService;
import com.fdkankan.download.service.IScenePlusService;
import com.fdkankan.fyun.constant.FYunTypeEnum;
import com.fdkankan.fyun.face.FYunFileServiceInterface;
import com.fdkankan.redis.constant.RedisKey;
import com.fdkankan.redis.util.RedisUtil;
import com.google.common.collect.Lists;
import java.io.File;
import java.io.FileInputStream;
import java.math.BigDecimal;
import java.net.URLEncoder;
import java.util.ArrayList;
import java.util.Calendar;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.concurrent.Callable;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.stream.Collectors;
import lombok.extern.slf4j.Slf4j;
import lombok.var;
import org.apache.tools.zip.ZipOutputStream;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.cloud.context.config.annotation.RefreshScope;
import org.springframework.scheduling.annotation.Async;
import org.springframework.stereotype.Service;
import org.springframework.web.client.RestTemplate;
/**
*
* TODO
*
*
* @author dengsixing
* @since 2022/2/22
**/
@RefreshScope
@Slf4j
@Service
public class SceneDownloadHandlerServiceImpl {
private static final String[] prefixArr = new String[]{
UploadFilePath.DATA_VIEW_PATH,
UploadFilePath.VOICE_VIEW_PATH,
UploadFilePath.VIDEOS_VIEW_PATH,
UploadFilePath.IMG_VIEW_PATH,
UploadFilePath.USER_VIEW_PATH,
};
private static final List imageTypes = Lists.newArrayList();
static{
imageTypes.add(ImageType.builder().name("2k_face").size("2048").ranges(new String[]{"0", "511", "1023", "1535"}).build());
imageTypes.add(ImageType.builder().name("1k_face").size("1024").ranges(new String[]{"0", "511"}).build());
imageTypes.add(ImageType.builder().name("512_face").size("512").ranges(new String[]{"0"}).build());
}
@Value("${path.v4school}")
private String v4localPath;
@Value("${path.zip-local}")
private String zipLocalFormat;
@Value("${path.zip-oss}")
private String zipOssFormat;
@Value("${path.zip-root}")
private String wwwroot;
@Value("${zip.nThreads}")
private int zipNthreads;
@Value("${oss.bucket:4dkankan}")
private String bucket;
@Value("${upload.type:oss}")
private String uploadType;
@Value("${download.config.resource-url}")
private String resourceUrl;
@Value("${download.config.public-url}")
private String publicUrl;
@Value("${download.config.exe-name}")
private String exeName;
@Value("${download.config.exe-content}")
private String exeContent;
@Autowired
private RedisUtil redisUtil;
@Autowired
private FYunFileServiceInterface fYunFileService;
@Autowired
private IScenePlusService scenePlusService;
@Autowired
private IScenePlusExtService scenePlusExtService;
@Async("sceneDownLoadExecutror")
public void download(DownLoadTaskBean downLoadTaskBean){
//场景码
String num = null;
try {
num = downLoadTaskBean.getNum();
log.info("场景下载开始 - num[{}] - threadName[{}]", num, Thread.currentThread().getName());
long startTime = Calendar.getInstance().getTimeInMillis();
//执行场景下载逻辑
this.downloadHandler(downLoadTaskBean);
//耗时
long consumeTime = Calendar.getInstance().getTimeInMillis() - startTime;
log.info("场景下载结束 - num[{}] - threadName[{}] - consumeTime[{}]", num, Thread.currentThread().getName(), consumeTime);
}catch (Exception e){
log.error(ExceptionUtil.stacktraceToString(e));
}finally {
if(StrUtil.isNotEmpty(num)){
//本地正在下载任务出队
CurrentDownloadNumUtil.removeSceneNum(num);
//删除正在下载任务
redisUtil.lRemove(RedisKey.SCENE_DOWNLOAD_ING, 1, num);
}
}
}
public void downloadHandler(DownLoadTaskBean downLoadTaskBean) throws Exception{
String num = downLoadTaskBean.getNum();
//zip包路径
String zipPath = null;
try {
ScenePlus scenePlus = scenePlusService.getByNum(num);
if(Objects.isNull(scenePlus))
throw new BusinessException(ErrorCode.FAILURE_CODE_5005);
ScenePlusExt scenePlusExt = scenePlusExtService.getByScenePlusId(scenePlus.getId());
String bucket = scenePlusExt.getYunFileBucket();
Set cacheKeys = new ConcurrentHashSet<>();
Map> allFiles = this.getAllFiles(num, v4localPath, bucket);
List ossFilePaths = allFiles.get("ossFilePaths");
List v3localFilePaths = allFiles.get("v3localFilePaths");
//key总个数
int total = ossFilePaths.size() + v3localFilePaths.size();
AtomicInteger count = new AtomicInteger(0);
//定义压缩包
zipPath = String.format(this.zipLocalFormat, num);
File zipFile = new File(zipPath);
if(!zipFile.getParentFile().exists()){
zipFile.getParentFile().mkdirs();
}
ZipOutputStream out = new ZipOutputStream(zipFile);
String sceneJsonData = fYunFileService.getFileContent(bucket, String.format(UploadFilePath.DATA_VIEW_PATH, num) + "scene.json");
JSONObject sceneJson = JSONUtil.parseObj(sceneJsonData);
String resolution = "4k";
String sceneForm = sceneJson.getStr("sceneFrom");
if(StrUtil.isNotEmpty(sceneForm) && SceneFrom.PRO.code().equals(sceneForm)){
resolution = "2k";
}
//国际版存在已经切好图的情况,下载时不需要再切图,只需要把文件直接下载下来打包就可以了
String sceneResolution = sceneJson.getStr("sceneResolution");
if(SceneResolution.TILES.code().equals(sceneResolution)){
resolution = "notNeadCut";
}
int imagesVersion = -1;
Integer version = sceneJson.getInt("version");
if(Objects.nonNull(version)){
imagesVersion = version;
}
long start = Calendar.getInstance().getTimeInMillis();
//固定文件写入
this.zipLocalFiles(out, v3localFilePaths, v4localPath, num, count, total);
long end1 = Calendar.getInstance().getTimeInMillis();
log.info("打包固定文件耗时, num:{}, time:{}", num, end1 - start);
//oss文件写入
this.zipOssFiles(out, ossFilePaths, num, count, total, resolution, imagesVersion, cacheKeys);
long end2 = Calendar.getInstance().getTimeInMillis();
log.info("打包oss文件耗时, num:{}, time:{}", num, end2 - end1);
//重新写入scene.json(去掉密码访问设置)
this.zipSceneJson(out, this.wwwroot, num, sceneJson);
//写入启动命令
this.zipBat(out, num);
out.close();
//上传压缩包
String uploadPath = String.format(this.zipOssFormat, num);
fYunFileService.uploadFileByCommand(bucket, zipPath, uploadPath);
//更新进度100
String url = this.publicUrl + uploadPath + "?t=" + Calendar.getInstance().getTimeInMillis();
this.updateProgress(null, num, SceneDownloadProgressStatus.DOWNLOAD_SUCCESS.code(), url);
}catch (Exception e){
//更新进度为下载失败
this.updateProgress( null, num, SceneDownloadProgressStatus.DOWNLOAD_FAILED.code(), null);
throw e;
}finally {
if(StrUtil.isNotBlank(zipPath)){
//删除本地zip包
FileUtils.deleteFile(zipPath);
}
}
}
private void zipOssFiles(ZipOutputStream out, List ossFilePaths, String num, AtomicInteger count,
int total, String resolution, int imagesVersion, Set cacheKeys) throws Exception{
if(CollUtil.isEmpty(ossFilePaths)){
return;
}
String imageNumPath = String.format(UploadFilePath.IMG_VIEW_PATH, num);
ExecutorService executorService = Executors.newFixedThreadPool(this.zipNthreads);
List futureList = new ArrayList<>();
for (String filePath : ossFilePaths) {
Callable call = new Callable() {
@Override
public Boolean call() throws Exception {
zipOssFilesHandler(out, num, count, total, resolution,
imagesVersion, cacheKeys,filePath, imageNumPath);
return true;
}
};
futureList.add(executorService.submit(call));
}
//这里一定要加阻塞,不然会导致oss文件还没打包好,主程序已经结束返回了
Boolean zipSuccess = true;
for (Future future : futureList) {
try {
future.get();
}catch (Exception e){
log.error("打包oss文件失败", e);
zipSuccess = false;
}
}
if(!zipSuccess){
throw new Exception("打包oss文件失败");
}
}
private void zipOssFilesHandler(ZipOutputStream out, String num,
AtomicInteger count, int total, String resolution,
int imagesVersion, Set cacheKeys,
String filePath, String imageNumPath) throws Exception{
//更新进度
this.updateProgress(new BigDecimal(count.incrementAndGet()).divide(new BigDecimal(total), 6, BigDecimal.ROUND_HALF_UP),
num, SceneDownloadProgressStatus.DOWNLOADING.code(), null);
//某个目录不需要打包
if(filePath.contains(imageNumPath + "panorama/panorama_edit/"))
return;
//切图
if(!"notNeadCut".equals(resolution)){
if((filePath.contains(imageNumPath + "panorama/") && filePath.contains("tiles/" + resolution))
|| filePath.contains(imageNumPath + "tiles/" + resolution + "/")) {
this.processImage(filePath, out, resolution, imagesVersion, cacheKeys);
return;
}
}
//其他文件打包
this.ProcessFiles(num, filePath, out, this.wwwroot, cacheKeys);
}
private void zipLocalFiles(ZipOutputStream out, List v3localFilePaths, String v3localPath, String num, AtomicInteger count, int total) throws Exception{
for (String v3localFilePath : v3localFilePaths) {
try (FileInputStream in = new FileInputStream(new File(v3localFilePath));){
this.zipInputStream(out, v3localFilePath.replace(v3localPath, ""), in);
}catch (Exception e){
throw e;
}
//更新进度
this.updateProgress(
new BigDecimal(count.incrementAndGet()).divide(new BigDecimal(total), 6, BigDecimal.ROUND_HALF_UP),
num, SceneDownloadProgressStatus.DOWNLOAD_COMPRESSING.code(), null);
}
//写入code.txt
this.zipBytes(out, "code.txt", num.getBytes());
}
private void zipBat(ZipOutputStream out, String num) throws Exception{
String batContent = String.format(this.exeContent, num);
this.zipBytes(out, exeName, batContent.getBytes());
//更新进度为90%
this.updateProgress(new BigDecimal("0.9").divide(new BigDecimal("0.8"), 6, BigDecimal.ROUND_HALF_UP), num,
SceneDownloadProgressStatus.DOWNLOAD_COMPRESSING.code(), null);
}
private Map> getAllFiles(String num, String v3localPath, String bucket) throws Exception{
//列出oss所有文件路径
List ossFilePaths = new ArrayList<>();
for (String prefix : prefixArr) {
prefix = String.format(prefix, num);
List keys = fYunFileService.listRemoteFiles(bucket, prefix);
if(CollUtil.isEmpty(keys)){
continue;
}
if(FYunTypeEnum.AWS.code().equals(this.uploadType)){
keys = keys.stream().filter(key->{
if(key.contains("x-oss-process")){
return false;
}
return true;
}).collect(Collectors.toList());
}
ossFilePaths.addAll(keys);
}
//列出v3local所有文件路径
File file = new File(v3localPath);
List v3localFilePaths = FileUtils.list(file);
HashMap> map = new HashMap<>();
map.put("ossFilePaths", ossFilePaths);
map.put("v3localFilePaths", v3localFilePaths);
return map;
}
private void zipSceneJson(ZipOutputStream out, String root, String num, JSONObject sceneJson) throws Exception{
//访问密码置0
JSONObject controls = sceneJson.getJSONObject("controls");
controls.set("showLock", 0);
String sceneJsonPath = root + String.format(UploadFilePath.DATA_VIEW_PATH, num) + "scene.json";
this.zipBytes(out, sceneJsonPath, sceneJson.toString().getBytes());
}
private void processImage(String key, ZipOutputStream out, String resolution, int imagesVersion, Set imgKeys) throws Exception{
if(key.contains("x-oss-process") || key.endsWith("/")){
return;
}
String fileName = key.substring(key.lastIndexOf("/")+1, key.indexOf("."));
String ext = key.substring(key.lastIndexOf("."));
String[] arr = fileName.split("_skybox");
String dir = arr[0];
String num = arr[1];
if(StrUtil.isEmpty(fileName)
|| StrUtil.isEmpty(ext)
|| (".jpg".equals(ext) && ".png".equals(ext))
|| StrUtil.isEmpty(dir)
|| StrUtil.isEmpty(num)){
throw new Exception("本地下载图片资源不符合规则,key:" + key);
}
for (ImageType imageType : imageTypes) {
List items = Lists.newArrayList();
String[] ranges = imageType.getRanges();
for(int i = 0; i < ranges.length; i++){
String x = ranges[i];
for(int j = 0; j < ranges.length; j++){
String y = ranges[j];
items.add(
ImageTypeDetail.builder()
.i(String.valueOf(i))
.j(String.valueOf(j))
.x(x)
.y(y)
.build()
);
}
}
for (ImageTypeDetail item : items) {
String par = "?x-oss-process=image/resize,m_lfit,w_" + imageType.getSize() + "/crop,w_512,h_512,x_" + item.getX() + ",y_" + item.getY();
if(FYunTypeEnum.AWS.code().equals(uploadType)){
par += "&imagesVersion="+ imagesVersion;
}
var url = this.
resourceUrl + key;
FYunTypeEnum storageType = FYunTypeEnum.get(uploadType);
switch (storageType){
case OSS:
url += par;
break;
case AWS:
url += URLEncoder.encode(par.replace("/", "@"), "UTF-8");
break;
}
var fky = key.split("/" + resolution + "/")[0] + "/" + dir + "/" + imageType.getName() + num + "_" + item.getI() + "_" + item.getJ() + ext;
if(imgKeys.contains(fky)){
continue;
}
imgKeys.add(fky);
this.zipBytes(out, wwwroot + fky, FileUtils.getBytesFromUrl(url));
}
}
}
public void ProcessFiles(String num, String key, ZipOutputStream out, String prefix, Set cacheKeys) throws Exception{
if(cacheKeys.contains(key)){
return;
}
if(key.equals(String.format(UploadFilePath.DATA_VIEW_PATH, num) + "scene.json")){
return;
}
cacheKeys.add(key);
String fileName = key.substring(key.lastIndexOf("/") + 1);
String url = this.resourceUrl + key.replace(fileName, URLEncoder.encode(fileName, "UTF-8")) + "?t=" + Calendar.getInstance().getTimeInMillis();
if(key.contains("hot.json") || key.contains("link-scene.json")){
String content = FileUtils.getStringFromUrl(url);
content.replace(publicUrl, "")
// .replace(publicUrl+"v3/", "")
.replace("https://spc.html","spc.html")
.replace("https://smobile.html", "smobile.html");
zipBytes(out, prefix + key, content.getBytes());
}else{
zipBytes(out, prefix + key, FileUtils.getBytesFromUrl(url));
}
}
public void updateProgress(BigDecimal precent, String num, Integer status, String url){
SceneDownloadProgressStatus progressStatus = SceneDownloadProgressStatus.get(status);
switch (progressStatus){
case DOWNLOAD_SUCCESS:
precent = new BigDecimal("100");
break;
case DOWNLOAD_FAILED:
precent = new BigDecimal("0");
break;
default:
precent = precent.multiply(new BigDecimal("0.8")).multiply(new BigDecimal("100"));
}
DownLoadProgressBean progress = null;
String key = String.format(RedisKey.PREFIX_DOWNLOAD_PROGRESS_V4, num);
String progressStr = redisUtil.get(key);
if(StrUtil.isEmpty(progressStr)){
progress = DownLoadProgressBean.builder().percent(precent.intValue()).status(status).url(url).build();
}else{
progress = JSONUtil.toBean(progressStr, DownLoadProgressBean.class);
//如果下载失败,进度不变
if(status == SceneDownloadProgressStatus.DOWNLOAD_FAILED.code() && progress.getPercent() != null){
precent = new BigDecimal(progress.getPercent());
}
progress.setPercent(precent.intValue());
progress.setStatus(status);
progress.setUrl(url);
}
redisUtil.set(key, JSONUtil.toJsonStr(progress));
}
public void zipInputStream(ZipOutputStream out, String key, FileInputStream in) throws Exception {
out.putNextEntry(new org.apache.tools.zip.ZipEntry(key));
byte[] bytes = new byte[1024];
int b = 0;
while ((b = in.read(bytes)) != -1) {
out.write(bytes, 0, b);
}
}
public synchronized void zipBytes(ZipOutputStream out, String key, byte[] bytes) throws Exception {
out.putNextEntry(new org.apache.tools.zip.ZipEntry(key));
out.write(bytes);
}
}