4dkankan-parent/4dkankan-center-scene-download/src/main/java/com/fdkankan/download/service/impl/SceneDownloadHandlerServiceImpl.java

package com.fdkankan.download.service.impl;

import cn.hutool.core.collection.CollUtil;
import cn.hutool.core.collection.ConcurrentHashSet;
import cn.hutool.core.exceptions.ExceptionUtil;
import cn.hutool.core.util.StrUtil;
import cn.hutool.json.JSONObject;
import cn.hutool.json.JSONUtil;
import com.alibaba.fastjson.JSON;
import com.fdkankan.common.bean.DownLoadProgressBean;
import com.fdkankan.common.bean.DownLoadTaskBean;
import com.fdkankan.common.constant.SceneDownloadProgressStatus;
import com.fdkankan.common.constant.SceneFrom;
import com.fdkankan.common.constant.SceneKind;
import com.fdkankan.common.constant.SceneResolution;
import com.fdkankan.common.constant.ServerCode;
import com.fdkankan.common.constant.UploadFilePath;
import com.fdkankan.common.response.ResultData;
import com.fdkankan.common.util.FileUtils;
import com.fdkankan.download.bean.CurrentDownloadNumUtil;
import com.fdkankan.download.bean.ImageType;
import com.fdkankan.download.bean.ImageTypeDetail;
import com.fdkankan.fyun.constant.StorageType;
import com.fdkankan.fyun.oss.UploadToOssUtil;
import com.fdkankan.redis.constant.RedisKey;
import com.fdkankan.redis.util.RedisUtil;
import com.fdkankan.scene.api.dto.SceneInfoDTO;
import com.fdkankan.scene.api.feign.SceneUserSceneClient;
import com.google.common.collect.Lists;
import java.io.File;
import java.io.FileInputStream;
import java.math.BigDecimal;
import java.net.URLEncoder;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.Calendar;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.concurrent.Callable;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.stream.Collectors;
import lombok.extern.slf4j.Slf4j;
import lombok.var;
import org.apache.tools.zip.ZipOutputStream;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.cloud.context.config.annotation.RefreshScope;
import org.springframework.scheduling.annotation.Async;
import org.springframework.stereotype.Service;
import org.springframework.web.client.RestTemplate;

/**
 * <p>
 * TODO
 * </p>
 *
 * @author dengsixing
 * @since 2022/2/22
 **/
@RefreshScope
@Slf4j
@Service
public class SceneDownloadHandlerServiceImpl {

    private static final String[] prefixArr = new String[]{
        UploadFilePath.DATA_VIEW_PATH,
        UploadFilePath.VOICE_VIEW_PATH,
        UploadFilePath.VIDEOS_VIEW_PATH,
        UploadFilePath.IMG_VIEW_PATH,
        UploadFilePath.USER_VIEW_PATH,
    };

    private static final List<ImageType> imageTypes = Lists.newArrayList();
    static{
        imageTypes.add(ImageType.builder().name("2k_face").size("2048").ranges(new String[]{"0", "511", "1023", "1535"}).build());
        imageTypes.add(ImageType.builder().name("1k_face").size("1024").ranges(new String[]{"0", "511"}).build());
        imageTypes.add(ImageType.builder().name("512_face").size("512").ranges(new String[]{"0"}).build());
    }

    @Autowired
    private SceneUserSceneClient sceneUserSceneClient;

    @Value("${path.v4school}")
    private String v4localPath;

    @Value("${path.zip-local}")
    private String zipLocalFormat;

    @Value("${path.zip-oss}")
    private String zipOssFormat;

    @Value("${path.zip-root}")
    private String wwwroot;

    @Value("${zip.nThreads}")
    private int zipNthreads;

    @Value("${oss.bucket:4dkankan}")
    private String bucket;

    @Value("${upload.type:oss}")
    private String uploadType;
    @Value("${download.config.resource-url}")
    private String resourceUrl;
    @Value("${download.config.public-url}")
    private String publicUrl;
    @Value("${download.config.exe-name}")
    private String exeName;
    @Value("${download.config.exe-content}")
    private String exeContent;
    @Autowired
    RestTemplate restTemplate;

    @Autowired
    RedisUtil redisUtil;
    @Autowired
    UploadToOssUtil uploadToOssUtil;

    @Async("sceneDownLoadExecutror")
    public void download(DownLoadTaskBean downLoadTaskBean){
        //场景码
        String num = null;

        try {
            num = downLoadTaskBean.getNum();

            log.info("场景下载开始 - num[{}] - threadName[{}]", num, Thread.currentThread().getName());

            long startTime = Calendar.getInstance().getTimeInMillis();

            //执行场景下载逻辑
            this.downloadHandler(downLoadTaskBean);

            //耗时
            long consumeTime = Calendar.getInstance().getTimeInMillis() - startTime;

            log.info("场景下载结束 - num[{}] - threadName[{}] - consumeTime[{}]", num, Thread.currentThread().getName(), consumeTime);

        }catch (Exception e){
            log.error(ExceptionUtil.stacktraceToString(e));
        }finally {
            if(StrUtil.isNotEmpty(num)){
                //本地正在下载任务出队
                CurrentDownloadNumUtil.removeSceneNum(num);
                //删除正在下载任务
                redisUtil.lRemove(RedisKey.SCENE_DOWNLOAD_ING, 1, num);
            }
        }
    }

    public void downloadHandler(DownLoadTaskBean downLoadTaskBean) throws Exception{

        String num = downLoadTaskBean.getNum();
        //zip包路径
        String zipPath = null;

        try {
            Set<String> cacheKeys = new ConcurrentHashSet<>();

            Map<String, List<String>> allFiles = this.getAllFiles(num, v4localPath);
            List<String> ossFilePaths = allFiles.get("ossFilePaths");
            List<String> v3localFilePaths = allFiles.get("v3localFilePaths");

            //key总个数
            int total = ossFilePaths.size() + v3localFilePaths.size();
            AtomicInteger count = new AtomicInteger(0);
            //定义压缩包
            zipPath = String.format(this.zipLocalFormat, num);
            File zipFile = new File(zipPath);
            if(!zipFile.getParentFile().exists()){
                zipFile.getParentFile().mkdirs();
            }
            ZipOutputStream out = new ZipOutputStream(zipFile);

//            JSONObject getInfoJson = this.zipGetInfoJson(out, this.wwwroot, num);
            String sceneJsonData = uploadToOssUtil.getObjectContent(bucket, String.format(UploadFilePath.DATA_VIEW_PATH, num) + "scene.json");
            JSONObject sceneJson = JSONUtil.parseObj(sceneJsonData);
            String resolution = "4k";
            String sceneForm = sceneJson.getStr("sceneFrom");
            if(StrUtil.isNotEmpty(sceneForm) && SceneFrom.PRO.code().equals(sceneForm)){
                resolution = "2k";
            }
            //如果是已经切好图的场景，不需要再切图了
//            String sceneResolution = sceneJson.getStr("sceneResolution");
//            if(SceneResolution.TILES.code().equals(sceneResolution)){
//                resolution = "notNeadCut";
//            }
            String sceneKind = sceneJson.getStr("sceneKind");
            if(StrUtil.isNotEmpty(sceneKind) && SceneKind.FACE.code().equals(sceneKind)){
                resolution = "notNeadCut";
            }

            int imagesVersion = -1;
            // TODO: 2022/3/29  V4版本目前没有imagesVersion字段，暂时用version字段替代
//            if(getInfoJson.getInt("imagesVersion") != null){
//                imagesVersion = getInfoJson.getInt("imagesVersion");
//            }
            Integer version = sceneJson.getInt("version");
            if(Objects.nonNull(version)){
                imagesVersion = version;
            }


            long start = Calendar.getInstance().getTimeInMillis();

            //固定文件写入
            this.zipLocalFiles(out, v3localFilePaths, v4localPath, num, count, total);
            long end1 = Calendar.getInstance().getTimeInMillis();
            log.info("打包固定文件耗时, num:{}, time:{}", num, end1 - start);

            //oss文件写入
            this.zipOssFiles(out, ossFilePaths, num, count, total, resolution, imagesVersion, cacheKeys);
            long end2 = Calendar.getInstance().getTimeInMillis();
            log.info("打包oss文件耗时, num:{}, time:{}", num, end2 - end1);

            //重新写入scene.json（去掉密码访问设置）
            this.zipSceneJson(out, this.wwwroot, num, sceneJson);

            //写入启动命令
            this.zipBat(out, num);

            out.close();

            //上传压缩包
            String uploadPath = String.format(this.zipOssFormat, num);
            uploadToOssUtil.uploadBySh(zipPath, uploadPath);

            //更新进度100
            String url = this.publicUrl + uploadPath + "?t=" + Calendar.getInstance().getTimeInMillis();
            this.updateProgress(null, num, SceneDownloadProgressStatus.DOWNLOAD_SUCCESS.code(), url);

            // TODO: 2022/5/24 v3 停止后要开启-----------------------start
            //更新用户场景已下载次数
//            platformUserClient.updateDownloadNum(userId, 1);
//
//            //更新下载log状态为成功
//            sceneUserSceneClient.updateSceneDownloadLog(num, DownloadStatus.SUCCESS.code(), url, null);
            // TODO: 2022/5/24 v3 停止后要开启-----------------------end

        }catch (Exception e){
            //更新进度为下载失败
            this.updateProgress( null, num, SceneDownloadProgressStatus.DOWNLOAD_FAILED.code(), null);
            //更新下载log状态为成功
            // TODO: 2022/5/24 v3 停止后要开启-----------------------start
//            sceneUserSceneClient.updateSceneDownloadLog(num, DownloadStatus.FAILD.code(), null, ExceptionUtil.stacktraceToString(e));
            // TODO: 2022/5/24 v3 停止后要开启-----------------------send
            throw e;
        }finally {
            if(StrUtil.isNotBlank(zipPath)){
                //删除本地zip包
                FileUtils.deleteFile(zipPath);
            }
        }
    }

    private void zipOssFiles(ZipOutputStream out, List<String> ossFilePaths, String num, AtomicInteger count,
        int total, String resolution, int imagesVersion, Set<String> cacheKeys) throws Exception{
        if(CollUtil.isEmpty(ossFilePaths)){
            return;
        }
        String imageNumPath = String.format(UploadFilePath.IMG_VIEW_PATH, num);
        ExecutorService executorService = Executors.newFixedThreadPool(this.zipNthreads);
        List<Future> futureList = new ArrayList<>();
        for (String filePath : ossFilePaths) {
            Callable<Boolean> call = new Callable() {
                @Override
                public Boolean call() throws Exception {
                    try {
                        zipOssFilesHandler(out, num, count, total, resolution,
                            imagesVersion, cacheKeys,filePath, imageNumPath);
                    }catch (Exception e){
                        log.error("下载失败", e);
                        return false;
                    }

                    return true;
                }
            };
            futureList.add(executorService.submit(call));
        }
        //这里一定要加阻塞，不然会导致oss文件还没打包好，主程序已经结束返回了
        Boolean zipSuccess = true;
        for (Future future : futureList) {
            try {
                future.get();
            }catch (Exception e){
                log.error("打包oss文件失败", e);
                zipSuccess = false;
            }
        }
        if(!zipSuccess){
            throw new Exception("打包oss文件失败");
        }

    }

    private void zipOssFilesHandler(ZipOutputStream out, String num,
        AtomicInteger count, int total, String resolution,
        int imagesVersion, Set<String> cacheKeys,
        String filePath, String imageNumPath) throws Exception{

        //更新进度
        this.updateProgress(new BigDecimal(count.incrementAndGet()).divide(new BigDecimal(total), 6, BigDecimal.ROUND_HALF_UP),
            num, SceneDownloadProgressStatus.DOWNLOADING.code(), null);

        //某个目录不需要打包
        if(filePath.contains(imageNumPath + "panorama/panorama_edit/"))
            return;

        //切图
        if(!"notNeadCut".equals(resolution)){
            if((filePath.contains(imageNumPath + "panorama/") && filePath.contains("tiles/" + resolution))
                || filePath.contains(imageNumPath + "tiles/" + resolution + "/")) {
                this.processImage(filePath, out, resolution, imagesVersion, cacheKeys);
                return;
            }
        }

        //其他文件打包
        this.ProcessFiles(num, filePath, out, this.wwwroot, cacheKeys);
    }

    private void zipLocalFiles(ZipOutputStream out, List<String> v3localFilePaths, String v3localPath, String num, AtomicInteger count, int total) throws Exception{
        for (String v3localFilePath : v3localFilePaths) {
            try (FileInputStream in = new FileInputStream(new File(v3localFilePath));){
                this.zipInputStream(out, v3localFilePath.replace(v3localPath, ""), in);
            }catch (Exception e){
                throw e;
            }
            //更新进度
            this.updateProgress(
                new BigDecimal(count.incrementAndGet()).divide(new BigDecimal(total), 6, BigDecimal.ROUND_HALF_UP),
                num, SceneDownloadProgressStatus.DOWNLOAD_COMPRESSING.code(), null);
        }
        //写入code.txt
        this.zipBytes(out, "code.txt", num.getBytes());
    }

    private void zipBat(ZipOutputStream out, String num) throws Exception{
        String batContent = String.format(this.exeContent, num);
        this.zipBytes(out, exeName, batContent.getBytes());

        //更新进度为90%
        this.updateProgress(new BigDecimal("0.9").divide(new BigDecimal("0.8"), 6, BigDecimal.ROUND_HALF_UP), num,
            SceneDownloadProgressStatus.DOWNLOAD_COMPRESSING.code(), null);
    }

    private Map<String, List<String>> getAllFiles(String num, String v3localPath) throws Exception{
        //列出oss所有文件路径
        List<String> ossFilePaths = new ArrayList<>();
        for (String prefix : prefixArr) {
            prefix = String.format(prefix, num);
            List<String> keys = uploadToOssUtil.listKeys(prefix);
            if(CollUtil.isEmpty(keys)){
                continue;
            }
            if(StorageType.AWS.code().equals(this.uploadType)){
                keys = keys.stream().filter(key->{
                    if(key.contains("x-oss-process")){
                        return false;
                    }
                    return true;
                }).collect(Collectors.toList());
            }
            ossFilePaths.addAll(keys);
        }

        //列出v3local所有文件路径
        File file = new File(v3localPath);
        List<String> v3localFilePaths = FileUtils.list(file);

        HashMap<String, List<String>> map = new HashMap<>();
        map.put("ossFilePaths", ossFilePaths);
        map.put("v3localFilePaths", v3localFilePaths);

        return map;
    }

    private JSONObject zipGetInfoJson(ZipOutputStream out, String root, String num) throws Exception{

        ResultData<SceneInfoDTO> sceneViewInfo = sceneUserSceneClient.getSceneViewInfo(num);
        if(!sceneViewInfo.getSuccess()){
            throw new Exception(ServerCode.FEIGN_REQUEST_FAILD.message());
        }
        SceneInfoDTO data = sceneViewInfo.getData();
        JSONObject getInfoJson = null;
        if(Objects.isNull(data)){
            getInfoJson = new JSONObject();
        }else {
            getInfoJson = JSONUtil.parseObj(data);
        }

        getInfoJson.set("sceneScheme", 3);
        getInfoJson.set("needKey", 0);
        getInfoJson.set("sceneKey","");
        //写入getInfo.json
        String getInfoJsonPath = root + String.format(UploadFilePath.DATA_VIEW_PATH, num) + "getInfo.json";
        this.zipBytes(out, getInfoJsonPath, getInfoJson.toString().getBytes());
        return getInfoJson;
    }

    private void zipSceneJson(ZipOutputStream out, String root, String num, JSONObject sceneJson) throws Exception{

        //访问密码置0
        JSONObject controls = sceneJson.getJSONObject("controls");
        controls.set("showLock", 0);

        String sceneJsonPath = root + String.format(UploadFilePath.DATA_VIEW_PATH, num) + "scene.json";
        this.zipBytes(out, sceneJsonPath, sceneJson.toString().getBytes());
    }

    private void processImage(String key, ZipOutputStream out, String resolution, int imagesVersion, Set<String> imgKeys) throws Exception{

        if(key.contains("x-oss-process") || key.endsWith("/")){
            return;
        }

        String fileName = key.substring(key.lastIndexOf("/")+1, key.indexOf("."));
        String ext = key.substring(key.lastIndexOf("."));
        String[] arr = fileName.split("_skybox");
        String dir = arr[0];
        String num = arr[1];
        if(StrUtil.isEmpty(fileName)
            || StrUtil.isEmpty(ext)
            || (".jpg".equals(ext) && ".png".equals(ext))
            ||  StrUtil.isEmpty(dir)
            || StrUtil.isEmpty(num)){
            throw new Exception("本地下载图片资源不符合规则，key：" + key);
        }
        for (ImageType imageType : imageTypes) {

            List<ImageTypeDetail> items = Lists.newArrayList();
            String[] ranges = imageType.getRanges();
            for(int i = 0; i < ranges.length; i++){
                String x = ranges[i];
                for(int j = 0; j < ranges.length; j++){
                    String y = ranges[j];
                    items.add(
                        ImageTypeDetail.builder()
                            .i(String.valueOf(i))
                            .j(String.valueOf(j))
                            .x(x)
                            .y(y)
                            .build()
                    );
                }
            }
            for (ImageTypeDetail item : items) {
                String par = "?x-oss-process=image/resize,m_lfit,w_" + imageType.getSize() + "/crop,w_512,h_512,x_" + item.getX() + ",y_" + item.getY();
                if(StorageType.AWS.code().equals(uploadType)){
                    par += "&imagesVersion="+ imagesVersion;
                }

                var url = this.
                    resourceUrl + key;
                StorageType storageType = StorageType.get(uploadType);
                switch (storageType){
                    case OSS:
                        url += par;
                        break;
                    case AWS:
                        url += URLEncoder.encode(par.replace("/", "@"), "UTF-8");
                        break;
                }
                var fky = key.split("/" + resolution + "/")[0] + "/" + dir + "/" + imageType.getName() +  num + "_" + item.getI()  + "_" + item.getJ() + ext;
                if(imgKeys.contains(fky)){
                    continue;
                }
                imgKeys.add(fky);
                this.zipBytes(out, wwwroot + fky, FileUtils.getBytesFromUrl(url));
            }

        }

    }

    public void ProcessFiles(String num, String key, ZipOutputStream out, String prefix, Set<String> cacheKeys) throws Exception{
        if(cacheKeys.contains(key)){
            return;
        }
        if(key.equals(String.format(UploadFilePath.DATA_VIEW_PATH, num) + "scene.json")){
            return;
        }
        cacheKeys.add(key);
        String fileName = key.substring(key.lastIndexOf("/") + 1);
        String url = this.resourceUrl + key.replace(fileName, URLEncoder.encode(fileName, "UTF-8")) + "?t=" + Calendar.getInstance().getTimeInMillis();
        if(key.contains("hot.json") || key.contains("link-scene.json")){
            String content = FileUtils.getStringFromUrl(url);

            content.replace(publicUrl, "")
//                .replace(publicUrl+"v3/", "")
                .replace("https://spc.html","spc.html")
                .replace("https://smobile.html", "smobile.html");

            zipBytes(out, prefix + key, content.getBytes());
        }else{
            try {
                zipBytes(out, prefix + key, FileUtils.getBytesFromUrl(url));
            }catch (Exception e){
                log.error("获取字节数据出错，url：{}", url);
            }
        }
    }


    public void updateProgress(BigDecimal precent, String num, Integer status, String url){

        SceneDownloadProgressStatus progressStatus = SceneDownloadProgressStatus.get(status);
        switch (progressStatus){
            case DOWNLOAD_SUCCESS:
                precent = new BigDecimal("100");
                break;
            case DOWNLOAD_FAILED:
                precent = new BigDecimal("0");
                break;
            default:
                precent = precent.multiply(new BigDecimal("0.8")).multiply(new BigDecimal("100"));
        }

        DownLoadProgressBean progress = null;
        String key = String.format(RedisKey.PREFIX_DOWNLOAD_PROGRESS_V4, num);
        String progressStr = redisUtil.get(key);
        if(StrUtil.isEmpty(progressStr)){
            progress =  DownLoadProgressBean.builder().percent(precent.intValue()).status(status).url(url).build();
        }else{
            progress = JSONUtil.toBean(progressStr, DownLoadProgressBean.class);
            //如果下载失败，进度不变
            if(status == SceneDownloadProgressStatus.DOWNLOAD_FAILED.code() && progress.getPercent() != null){
                precent = new BigDecimal(progress.getPercent());
            }
            progress.setPercent(precent.intValue());
            progress.setStatus(status);
            progress.setUrl(url);
        }
        if(status == 1003){
            log.info(JSONUtil.toJsonStr(progress));
        }
        redisUtil.set(key, JSONUtil.toJsonStr(progress));

    }

    public void zipInputStream(ZipOutputStream out, String key, FileInputStream in) throws Exception {
        out.putNextEntry(new org.apache.tools.zip.ZipEntry(key));
        byte[] bytes = new byte[1024];
        int b = 0;
        while ((b = in.read(bytes)) != -1) {
            out.write(bytes, 0, b);
        }
    }

    public synchronized void zipBytes(ZipOutputStream out, String key, byte[] bytes) throws Exception {
        out.putNextEntry(new org.apache.tools.zip.ZipEntry(key));
        out.write(bytes);
    }


}