4dkankan-center-modeling-control/src/main/java/com/fdkankan/contro/service/impl/JmgaServiceImpl.java

package com.fdkankan.contro.service.impl;

import cn.hutool.core.collection.CollUtil;
import cn.hutool.core.io.FileUtil;
import cn.hutool.core.io.file.FileMode;
import cn.hutool.core.lang.UUID;
import cn.hutool.core.util.StrUtil;
import cn.hutool.core.util.ZipUtil;
import cn.hutool.http.HttpUtil;
import cn.hutool.json.JSONUtil;
import com.alibaba.excel.EasyExcel;
import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject;
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
import com.baomidou.mybatisplus.core.conditions.update.LambdaUpdateWrapper;
import com.fdkankan.common.constant.ErrorCode;
import com.fdkankan.common.constant.PayStatus;
import com.fdkankan.common.constant.RecStatus;
import com.fdkankan.common.constant.SceneStatus;
import com.fdkankan.common.exception.BusinessException;
import com.fdkankan.common.util.DateExtUtil;
import com.fdkankan.common.util.DateUtil;
import com.fdkankan.contro.bean.SendCallAlgorithmDetail;
import com.fdkankan.contro.constant.ModelingControlRespCode;
import com.fdkankan.contro.constant.ZipConstant;
import com.fdkankan.contro.entity.*;
import com.fdkankan.contro.mq.service.IBuildLogService;
import com.fdkankan.contro.service.*;
import com.fdkankan.contro.util.SceneSourceUtil;
import com.fdkankan.contro.vo.SendCallAlgorithmParam;
import com.fdkankan.contro.vo.UploadSceneOrigParamVo;
import com.fdkankan.fyun.face.FYunFileServiceInterface;
import com.fdkankan.model.constants.ConstantFilePath;
import com.fdkankan.rabbitmq.util.RabbitMqProducer;
import lombok.extern.slf4j.Slf4j;
import net.lingala.zip4j.exception.ZipException;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Service;

import javax.annotation.Resource;
import java.io.File;
import java.io.IOException;
import java.io.RandomAccessFile;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Paths;
import java.util.*;
import java.util.stream.Collectors;
import java.util.stream.Stream;

@Slf4j
@Service
public class JmgaServiceImpl implements IJmgaService {

    public static final String ipv4Pattern = "\\b(?:[0-9]{1,3}\\.){3}[0-9]{1,3}\\b";
    @Value("${jmga.eventNotice:false}")
    private boolean eventNotice;
    @Value("${scene.pro.new.url}")
    private String sceneProNewUrl;

    @Autowired
    private IOrigFileUploadBatchService origFileUploadBatchService;

    @Autowired
    private IOrigFileUploadService origFileUploadService;
    @Resource
    private FYunFileServiceInterface fileServiceInterface;
    @Resource
    private RabbitMqProducer mqProducer;
    @Resource
    private FYunFileServiceInterface fYunFileService;
    @Autowired
    private ISceneFileBuildService sceneFileBuildService;
    @Autowired
    private IUserService userService;
    @Autowired
    private IScenePlusService scenePlusService;
    @Autowired
    private IScenePlusExtService scenePlusExtService;
    @Autowired
    private ISceneBuildProcessLogService sceneBuildProcessLogService;
    @Autowired
    private IBuildLogService buildLogService;
    @Autowired
    private IJyUserService jyUserService;
    @Autowired
    private IScene3dNumService scene3dNumService;
    @Autowired
    private ICameraService cameraService;
    @Autowired
    private ICameraDetailService cameraDetailService;
    @Autowired
    private ISceneEditInfoService sceneEditInfoService;
    @Autowired
    private ISceneEditInfoExtService sceneEditInfoExtService;
    @Autowired
    private ISceneEditControlsService sceneEditControlsService;

    @Override
    public void checkFileWhole(String uuid, String dataSource, JSONObject fdageData) {

        int camType = fdageData.getJSONObject("cam").getIntValue("type");

        //从data.fdage中获取需要上传的文件列表
        JSONArray points = fdageData.getJSONArray("points");
        if(CollUtil.isEmpty(points)){
            return;
        }

        List<String> fileList = new ArrayList<>();
        for (Object point : points) {
            JSONObject pointJson = (JSONObject) point;
            if(camType == 5){//圆周率相机
                String name = pointJson.getString("name");
                fileList.add(name);
            }else{//四维相机
                JSONArray imgs = pointJson.getJSONArray("imgs");
                if(CollUtil.isEmpty(imgs)){
                    continue;
                }
                for (Object img : imgs) {
                    JSONObject imgJson = (JSONObject) img;
                    String name = imgJson.getString("name");
                    fileList.add(name);
                }
            }
        }

        //查询当前待通知计算批次文件是否存在
        List<OrigFileUploadBatch> batchList = origFileUploadBatchService.list(new LambdaQueryWrapper<OrigFileUploadBatch>().eq(OrigFileUploadBatch::getUuid, uuid));
        if(CollUtil.isNotEmpty(batchList)){
            Set<String> batchIds = batchList.stream().map(v -> v.getBatchId()).collect(Collectors.toSet());
            List<OrigFileUpload> origFileList = origFileUploadService.list(new LambdaQueryWrapper<OrigFileUpload>().in(OrigFileUpload::getBatchId, batchIds));
            //过滤出不存在的
            fileList = fileList.stream().filter(v -> origFileList.stream().noneMatch(k -> k.getFileName().equals(v))).collect(Collectors.toList());
        }

        if(CollUtil.isNotEmpty(fileList)){
            String args = JSON.toJSONString(fileList);
            if(fileList.size() > 5){
                args =  fileList.size() +  "个文件";
            }
            throw new BusinessException(ErrorCode.FAILURE_CODE_4001, args);
        }
    }

    @Override
    public void sendStatus(String num, String eventContent) {
        if(!eventNotice){
            return;
        }
        Map<String, Object> param = new HashMap<>();
        try {
            ScenePlus scenePlus = scenePlusService.getScenePlusByNum(num);
            JyUser jyUser = new JyUser();
            param.put("event_type", "scene");
            param.put("event_content", eventContent);
            param.put("scene_num", num);
            param.put("event_time", new Date());
            if(Objects.nonNull(scenePlus)){
                param.put("scene_title", scenePlus.getTitle());
                jyUser = jyUserService.getByUserId(scenePlus.getUserId());
            }
            param.put("ryid", jyUser.getRyId());
            param.put("ryno", jyUser.getRyNo());
            param.put("nick_name", jyUser.getRyNickName());
            mqProducer.sendByWorkQueue("jmga-event-notice", param);
        }catch (Exception e){
            log.info("推送事件失败,param:{}", param);
        }
    }

    @Override
    public void checkLackFile(String dataSource) {

        String dataFdagePath = dataSource.concat(File.separator).concat("capture").concat(File.separator).concat("data.fdage");
        if(!FileUtil.exist(dataFdagePath)){
            throw new BusinessException(ErrorCode.FAILURE_CODE_4001.code(), ErrorCode.FAILURE_CODE_4001.formatMessage("data.fdage"));
        }
        JSONObject fdageData = null;
        try {
            fdageData = JSON.parseObject(FileUtil.readUtf8String(dataFdagePath));
        }catch (Exception e){
            throw new BusinessException(ErrorCode.FAILURE_CODE_4001.code(), ErrorCode.FAILURE_CODE_4001.formatMessage("data.fdage"));
        }

        int camType = fdageData.getJSONObject("cam").getIntValue("type");
        if(camType == 1 || camType == 2 || camType == 9 || camType == 10 || camType == 11){
            Integer shootCount = fdageData.getJSONArray("points").size();
            if(shootCount > 0){//有点位代表架站式
                String parametersPath = dataSource.concat(File.separator).concat("capture").concat(File.separator).concat("parameters.json");
                if(!FileUtil.exist(parametersPath)){
                    throw new BusinessException(ErrorCode.FAILURE_CODE_4001.code(), ErrorCode.FAILURE_CODE_4001.formatMessage("parameters.json"));
                }

                String sfmDataBinPath = dataSource.concat(File.separator).concat("capture").concat(File.separator).concat("sfm_data.bin");
                if(!FileUtil.exist(sfmDataBinPath)){
                    throw new BusinessException(ErrorCode.FAILURE_CODE_4001.code(), ErrorCode.FAILURE_CODE_4001.formatMessage("sfm_data.bin"));
                }

                String upName = camType > 2 ? "Up.txt" : "Up.xml";
                String upTxtPath = dataSource.concat(File.separator).concat("capture").concat(File.separator).concat(upName);
                if(!FileUtil.exist(upTxtPath)){
                    throw new BusinessException(ErrorCode.FAILURE_CODE_4001.code(), ErrorCode.FAILURE_CODE_4001.formatMessage(upName));
                }
            }
        }
        if(camType == 5){
            String parametersPath = dataSource.concat(File.separator).concat("capture").concat(File.separator).concat("images/parameters.json");
            if(!FileUtil.exist(parametersPath)){
                throw new BusinessException(ErrorCode.FAILURE_CODE_4001.code(), ErrorCode.FAILURE_CODE_4001.formatMessage("parameters.json"));
            }

            String sfmDataBinPath = dataSource.concat(File.separator).concat("capture").concat(File.separator).concat("result/reconstruction/sfm_data.bin");
            if(!FileUtil.exist(sfmDataBinPath)){
                throw new BusinessException(ErrorCode.FAILURE_CODE_4001.code(), ErrorCode.FAILURE_CODE_4001.formatMessage("sfm_data.bin"));
            }
        }

        //从data.fdage中获取需要上传的文件列表
        JSONArray points = fdageData.getJSONArray("points");
        if(CollUtil.isEmpty(points)){
            return;
        }

        for (Object point : points) {
            JSONObject pointJson = (JSONObject) point;
            if(camType == 5){//圆周率相机
                String name = pointJson.getString("name");
                this.checkFileAvailable(dataSource, name);
            }else{//四维相机
                JSONArray imgs = pointJson.getJSONArray("imgs");
                for (Object img : imgs) {
                    JSONObject imgJson = (JSONObject) img;
                    String name = imgJson.getString("name");
                    this.checkFileAvailable(dataSource, name);
                }
            }
        }
    }

    private void checkFileAvailable(String dataSource, String fileName){
        List<File> fileList = FileUtil.loopFiles(dataSource.concat(File.separator).concat("capture"));
        if(CollUtil.isEmpty(fileList)){
            throw new BusinessException(ErrorCode.FAILURE_CODE_4001.code(), ErrorCode.FAILURE_CODE_4001.formatMessage(fileName));
        }
        for (File file : fileList) {
            if(file.getName().equals(fileName) && file.length() > 0){
                return;
            }
        }
        throw new BusinessException(ErrorCode.FAILURE_CODE_4001.code(), ErrorCode.FAILURE_CODE_4001.formatMessage(fileName));
    }

    @Override
    public void uploadScene(UploadSceneOrigParamVo param) throws Exception {

        User user = userService.getById(param.getUserId());
        if(Objects.isNull(user)){
            throw new BusinessException(ErrorCode.USER_NOT_EXIST);
        }

        String uuid = UUID.randomUUID().toString();
        String zipName = uuid + ".zip";
        String zipDir = "/oss/4dkankan/bd_server/";
        fYunFileService.downloadFileByCommand(zipDir + zipName, param.getFilePath());

        Camera camera = null;
        CameraDetail cameraDetail = null;
        String sceneNum = null;
        if(param.getSourceType().equals("orig")){//原始资源
            String dataFdageStr = com.fdkankan.contro.util.ZipUtil.readUtf8(zipDir + zipName, "data.fdage");
            if(StrUtil.isEmpty(dataFdageStr) || !JSONUtil.isJson(dataFdageStr)){
                throw new BusinessException(60027, "data.fdage文件数据异常");
            }
            JSONObject dataFdage = JSON.parseObject(dataFdageStr);
            String snCode = dataFdage.getJSONObject("cam").getString("uuid");
            String uuidTime = dataFdage.getString("uuidtime");
            String uniCode = snCode + "_" + uuidTime;
            int camType = dataFdage.getJSONObject("cam").getIntValue("type");
            Map<String, Object> instorage = cameraService.instorage(camType, snCode);
            cameraDetail = (CameraDetail) instorage.get("cameraDetail");

            ScenePlus scenePlus = scenePlusService.getByFileId(uniCode);
            if(scenePlus == null){
                scenePlus = new ScenePlus();
                sceneNum = scene3dNumService.generateSceneNum(cameraDetail.getType());
                scenePlus.setNum(sceneNum);
                scenePlus.setCameraId(cameraDetail.getCameraId());
                scenePlus.setSceneSource(SceneSourceUtil.getSceneSourceByCamType(cameraDetail.getType()));
            }else{
                if(scenePlus.getSceneStatus() == SceneStatus.wait.code()){
                    throw new BusinessException(ErrorCode.FAILURE_CODE_5033);
                }
                sceneNum = scenePlus.getNum();
            }
            scenePlus.setTitle(dataFdage.getString("name"));
            scenePlus.setUserId(user.getId());
            scenePlus.setUpdateTime(null);
            scenePlus.setSceneStatus(SceneStatus.wait.code());
            scenePlusService.saveOrUpdate(scenePlus);

            ScenePlusExt scenePlusExt = scenePlusExtService.getScenePlusExtByPlusId(scenePlus.getId());
            if(scenePlusExt == null){
                scenePlusExt = new ScenePlusExt();
                scenePlusExt.setPlusId(scenePlus.getId());
                scenePlusExt.setWebSite("/" + sceneProNewUrl + scenePlus.getNum());
                String fileId = sceneFileBuildService.getFileId(snCode, uniCode);
                scenePlusExt.setDataSource(ConstantFilePath.BUILD_MODEL_PATH + snCode + File.separator + fileId + File.separator + uniCode);
                scenePlusExtService.save(scenePlusExt);
            }

            SceneEditInfo sceneEditInfo = sceneEditInfoService.getByScenePlusId(scenePlus.getId());
            if(sceneEditInfo == null){
                sceneEditInfo = new SceneEditInfo();
                sceneEditInfo.setScenePlusId(scenePlus.getId());
                sceneEditInfoService.save(sceneEditInfo);
            }
            SceneEditInfoExt sceneEditInfoExt = sceneEditInfoExtService.getByEditInfoId(sceneEditInfo.getId());
            if(sceneEditInfoExt == null){
                sceneEditInfoExt = new SceneEditInfoExt();
                sceneEditInfoExt.setScenePlusId(scenePlus.getId());
                sceneEditInfoExt.setEditInfoId(sceneEditInfo.getId());
                sceneEditInfoExtService.save(sceneEditInfoExt);
            }
            SceneEditControls sceneEditControls = sceneEditControlsService.getBySceneEditId(sceneEditInfo.getId());
            if(sceneEditControls == null){
                sceneEditControls = new SceneEditControls();
                sceneEditControls.setEditInfoId(sceneEditInfo.getId());
                sceneEditControlsService.save(sceneEditControls);
            }

        }else{//离线包
            String sceneJsonStr = com.fdkankan.contro.util.ZipUtil.readUtf8(zipDir + zipName, "scene.json");
            if(StrUtil.isEmpty(sceneJsonStr) || !JSONUtil.isJson(sceneJsonStr)){
                throw new BusinessException(60027, "scene.json文件数据异常");
            }
            JSONObject sceneJson = JSON.parseObject(sceneJsonStr);
            sceneNum = sceneJson.getString("num");

            ScenePlus scenePlusDb = scenePlusService.getDeletedByNum(sceneNum);
            if(scenePlusDb != null && scenePlusDb.getRecStatus().equalsIgnoreCase(RecStatus.VALID.code())){
                throw new BusinessException(ModelingControlRespCode.NUM_EXISTS.code(), ModelingControlRespCode.NUM_EXISTS.message());
            }
            if(scenePlusDb != null && scenePlusDb.getRecStatus().equalsIgnoreCase(RecStatus.DISABLE.code())){
                scenePlusService.updateNumById(scenePlusDb.getId(), scenePlusDb.getNum() + "-" + scenePlusDb.getId());
            }

            int camType = sceneJson.getInteger("camType");
            String snCode = sceneJson.getString("snCode");
            Map<String, Object> instorage = cameraService.instorage(camType, snCode);
            cameraDetail = (CameraDetail) instorage.get("cameraDetail");
            ScenePlus scenePlus = new ScenePlus();
            scenePlus.setNum(sceneNum);
            scenePlus.setCameraId(cameraDetail.getCameraId());
            scenePlus.setSceneSource(SceneSourceUtil.getSceneSourceByCamType(cameraDetail.getType()));
            scenePlus.setTitle(sceneJson.getString("title"));
            scenePlus.setUserId(user.getId());
            scenePlus.setUpdateTime(null);
            scenePlus.setSceneStatus(SceneStatus.wait.code());
            scenePlus.setUploadType("offline");
            scenePlus.setPayStatus(PayStatus.PAY.code());
            scenePlus.setStartBuildTime(new Date());
            scenePlusService.save(scenePlus);
            ScenePlusExt scenePlusExt = new ScenePlusExt();
            scenePlusExt.setPlusId(scenePlus.getId());
            String uniCode = snCode + DateExtUtil.format(new Date(), DateExtUtil.dateStyle11);
            String fileId = sceneFileBuildService.getFileId(snCode, uniCode);
            scenePlusExt.setDataSource(ConstantFilePath.BUILD_MODEL_PATH + snCode + File.separator + fileId + File.separator + uniCode);
            scenePlusExtService.save(scenePlusExt);
        }

        //发mq做异步处理
        JSONObject content = new JSONObject();
        content.put("zipPath", zipDir + zipName);
        content.put("num", sceneNum);
        content.put("sourceType", param.getSourceType());
        mqProducer.sendByWorkQueue("manage-upload-scene", content);
    }


    @Override
    public void noticeBuildBd(String ossHost, String uuid, String dir) {
        String prefix = "http://" + ossHost + "/";
        String path = "/oss/4dkankan/bd_server/" + dir;
        List<File> files = FileUtil.loopFiles(path).stream().filter(file -> !file.getAbsolutePath().contains("backup")).collect(Collectors.toList());
        files.parallelStream().forEach(file ->{
            String url = prefix.concat(file.getAbsolutePath().substring(1).replace("/4dkankan",""));
            SendCallAlgorithmDetail detail = new SendCallAlgorithmDetail();
            detail.setFileName(FileUtil.getName(file.getAbsolutePath()));
            detail.setUuid(uuid);
            SendCallAlgorithmParam param = new SendCallAlgorithmParam();
            param.setFilepath(url);
            param.setDetails(detail);
            HttpUtil.post("http://127.0.0.1:8085/api/scene/file/sendCallAlgorithm", JSON.toJSONString(param));
        });
    }

    @Override
    public void updateSceneFailAndZipLog(String num, String failReason) throws IOException {
        scenePlusService.update(new LambdaUpdateWrapper<ScenePlus>()
                .set(ScenePlus::getSceneStatus, SceneStatus.FAILD.code())
                .set(ScenePlus::getBuildFailReason, failReason)
                .eq(ScenePlus::getNum, num));

        String logBasePath = File.separator + "mnt" + File.separator + "logs" + File.separator;
        String tmpPath = logBasePath + "tmp" + File.separator;
        String logPath = tmpPath + num + "-" + Calendar.getInstance().getTimeInMillis() + File.separator;
        String zipPath = tmpPath + num + "-" + Calendar.getInstance().getTimeInMillis() + ".zip";
        FileUtil.mkdir(logPath);
        //打包服务端日志
        File[] ls = FileUtil.ls(logBasePath);
        for (File l : ls) {
            if(l.isFile() || !l.getName().startsWith("modeling-control")){
                continue;
            }
            this.copyLog(l.getAbsolutePath(), logPath, "info","sql","error");
        }
        //打包算法日志
        ScenePlus scenePlus = scenePlusService.getScenePlusByNum(num);
        ScenePlusExt scenePlusExt = scenePlusExtService.getScenePlusExtByPlusId(scenePlus.getId());
        String consoleLogPath = scenePlusExt.getDataSource() + "/console.log";
        String consoleOldLogPath = scenePlusExt.getDataSource() + "/console.old.log";
        if(FileUtil.exist(consoleLogPath)){
            FileUtil.copy(consoleLogPath, logPath,true);
        }
        if(FileUtil.exist(consoleOldLogPath)){
            FileUtil.copy(consoleOldLogPath, logPath,true);
        }
        //打包数据库记录
        String[] split = scenePlusExt.getDataSource().split("/");
        String uuid = split[split.length -1];
        List<OrigFileUploadBatch> batchList = origFileUploadBatchService.list(new LambdaQueryWrapper<OrigFileUploadBatch>().eq(OrigFileUploadBatch::getUuid, uuid));
        if(CollUtil.isNotEmpty(batchList)){
            EasyExcel.write(logPath.concat("origFileUploadBatch.xlsx"), OrigFileUploadBatch.class).sheet("origFileUploadBatch").doWrite(batchList);
            List<String> batchIdList = batchList.stream().map(v -> v.getBatchId()).collect(Collectors.toList());
            List<OrigFileUpload> uploadList = origFileUploadService.list(new LambdaQueryWrapper<OrigFileUpload>().in(OrigFileUpload::getBatchId, batchIdList));
            if(CollUtil.isNotEmpty(uploadList)){
                uploadList.stream().forEach(v->{
                    v.setFileUrl(v.getFileUrl().replaceAll(ipv4Pattern, "ip"));
                });
                EasyExcel.write(logPath.concat("origFileUpload.xlsx"), OrigFileUpload.class).sheet("origFileUpload").doWrite(uploadList);
            }
        }
        List<SceneBuildProcessLog> processLogList = sceneBuildProcessLogService.listByNum(num);
        if(CollUtil.isNotEmpty(processLogList)){
            EasyExcel.write(logPath.concat("sceneBuildProcessLog.xlsx"), SceneBuildProcessLog.class).sheet("sceneBuildProcessLog").doWrite(processLogList);
        }
        List<BuildLog> buildLogList = buildLogService.list(new LambdaQueryWrapper<BuildLog>().eq(BuildLog::getSceneNum, num));
        if(CollUtil.isNotEmpty(buildLogList)){
            EasyExcel.write(logPath.concat("buildLog.xlsx"), BuildLog.class).sheet("buildLog").doWrite(buildLogList);
        }

        //上传日志，删除本地压缩包
        ZipUtil.zip(logPath, zipPath, false);
        String logName = "build_log/" + num + "/" + num + "-log.zip";
        if(fYunFileService.fileExist(logName)){
            fYunFileService.copyFileInBucket(logName, logName + "-bak-" + DateExtUtil.format(Calendar.getInstance().getTime(), DateExtUtil.dateStyle11));
        }
        fYunFileService.uploadFile(zipPath, logName);

    }

    private void copyLog(String serverPath, String tmpPath, String... levels) throws IOException {
        level: for (String level : levels) {
            String infoPath = serverPath + File.separator + level + File.separator;
            List<File> fileList = FileUtil.loopFiles(infoPath);
            if(CollUtil.isEmpty(fileList)){
                continue;
            }
            CollUtil.reverse(fileList);
            for (File file : fileList) {
                int i = fileList.indexOf(file);
                if(i > 7){
                    continue level;
                }
                String targetPath = file.getAbsolutePath().replace(FileUtil.getParent(serverPath, 1), tmpPath);
                File targetLog = new File(targetPath);
                targetLog.createNewFile();
//                FileUtil.copy(file.getAbsolutePath(), targetPath, true);
//                String content = FileUtil.readUtf8String(targetPath);
//                content = content.replace("公安", "xx").replaceAll(ipv4Pattern, "ip");
//                FileUtil.writeUtf8String(content, targetPath);

                try (Stream<String> lines = Files.lines(Paths.get(file.getAbsolutePath()), StandardCharsets.UTF_8)) {
                    lines.map(line -> line.replace("公安", "xx").replaceAll(ipv4Pattern, "ip"))
                            .forEach(line -> FileUtil.appendUtf8String(line, targetLog)); // 或者进行其他处理
                } catch (IOException e) {
                    e.printStackTrace();
                }
            }
        }
    }

}