package com.fdkankan.contro.service.impl; import cn.hutool.core.collection.CollUtil; import cn.hutool.core.io.FileUtil; import cn.hutool.core.io.file.FileMode; import cn.hutool.core.lang.UUID; import cn.hutool.core.util.StrUtil; import cn.hutool.core.util.ZipUtil; import cn.hutool.http.HttpUtil; import cn.hutool.json.JSONUtil; import com.alibaba.excel.EasyExcel; import com.alibaba.fastjson.JSON; import com.alibaba.fastjson.JSONArray; import com.alibaba.fastjson.JSONObject; import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper; import com.baomidou.mybatisplus.core.conditions.update.LambdaUpdateWrapper; import com.fdkankan.common.constant.ErrorCode; import com.fdkankan.common.constant.PayStatus; import com.fdkankan.common.constant.RecStatus; import com.fdkankan.common.constant.SceneStatus; import com.fdkankan.common.exception.BusinessException; import com.fdkankan.common.util.DateExtUtil; import com.fdkankan.common.util.DateUtil; import com.fdkankan.contro.bean.SendCallAlgorithmDetail; import com.fdkankan.contro.constant.ModelingControlRespCode; import com.fdkankan.contro.constant.ZipConstant; import com.fdkankan.contro.entity.*; import com.fdkankan.contro.mq.service.IBuildLogService; import com.fdkankan.contro.service.*; import com.fdkankan.contro.util.SceneSourceUtil; import com.fdkankan.contro.vo.SendCallAlgorithmParam; import com.fdkankan.contro.vo.UploadSceneOrigParamVo; import com.fdkankan.fyun.face.FYunFileServiceInterface; import com.fdkankan.model.constants.ConstantFilePath; import com.fdkankan.rabbitmq.util.RabbitMqProducer; import lombok.extern.slf4j.Slf4j; import net.lingala.zip4j.exception.ZipException; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; import javax.annotation.Resource; import java.io.File; import java.io.IOException; import java.io.RandomAccessFile; import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Paths; import java.util.*; import java.util.stream.Collectors; import java.util.stream.Stream; @Slf4j @Service public class JmgaServiceImpl implements IJmgaService { public static final String ipv4Pattern = "\\b(?:[0-9]{1,3}\\.){3}[0-9]{1,3}\\b"; @Value("${jmga.eventNotice:false}") private boolean eventNotice; @Value("${scene.pro.new.url}") private String sceneProNewUrl; @Autowired private IOrigFileUploadBatchService origFileUploadBatchService; @Autowired private IOrigFileUploadService origFileUploadService; @Resource private FYunFileServiceInterface fileServiceInterface; @Resource private RabbitMqProducer mqProducer; @Resource private FYunFileServiceInterface fYunFileService; @Autowired private ISceneFileBuildService sceneFileBuildService; @Autowired private IUserService userService; @Autowired private IScenePlusService scenePlusService; @Autowired private IScenePlusExtService scenePlusExtService; @Autowired private ISceneBuildProcessLogService sceneBuildProcessLogService; @Autowired private IBuildLogService buildLogService; @Autowired private IJyUserService jyUserService; @Autowired private IScene3dNumService scene3dNumService; @Autowired private ICameraService cameraService; @Autowired private ICameraDetailService cameraDetailService; @Autowired private ISceneEditInfoService sceneEditInfoService; @Autowired private ISceneEditInfoExtService sceneEditInfoExtService; @Autowired private ISceneEditControlsService sceneEditControlsService; @Override public void checkFileWhole(String uuid, String dataSource, JSONObject fdageData) { int camType = fdageData.getJSONObject("cam").getIntValue("type"); //从data.fdage中获取需要上传的文件列表 JSONArray points = fdageData.getJSONArray("points"); if(CollUtil.isEmpty(points)){ return; } List fileList = new ArrayList<>(); for (Object point : points) { JSONObject pointJson = (JSONObject) point; if(camType == 5){//圆周率相机 String name = pointJson.getString("name"); fileList.add(name); }else{//四维相机 JSONArray imgs = pointJson.getJSONArray("imgs"); if(CollUtil.isEmpty(imgs)){ continue; } for (Object img : imgs) { JSONObject imgJson = (JSONObject) img; String name = imgJson.getString("name"); fileList.add(name); } } } //查询当前待通知计算批次文件是否存在 List batchList = origFileUploadBatchService.list(new LambdaQueryWrapper().eq(OrigFileUploadBatch::getUuid, uuid)); if(CollUtil.isNotEmpty(batchList)){ Set batchIds = batchList.stream().map(v -> v.getBatchId()).collect(Collectors.toSet()); List origFileList = origFileUploadService.list(new LambdaQueryWrapper().in(OrigFileUpload::getBatchId, batchIds)); //过滤出不存在的 fileList = fileList.stream().filter(v -> origFileList.stream().noneMatch(k -> k.getFileName().equals(v))).collect(Collectors.toList()); } if(CollUtil.isNotEmpty(fileList)){ String args = JSON.toJSONString(fileList); if(fileList.size() > 5){ args = fileList.size() + "个文件"; } throw new BusinessException(ErrorCode.FAILURE_CODE_4001, args); } } @Override public void sendStatus(String num, String eventContent) { if(!eventNotice){ return; } Map param = new HashMap<>(); try { ScenePlus scenePlus = scenePlusService.getScenePlusByNum(num); JyUser jyUser = new JyUser(); param.put("event_type", "scene"); param.put("event_content", eventContent); param.put("scene_num", num); param.put("event_time", new Date()); if(Objects.nonNull(scenePlus)){ param.put("scene_title", scenePlus.getTitle()); jyUser = jyUserService.getByUserId(scenePlus.getUserId()); } param.put("ryid", jyUser.getRyId()); param.put("ryno", jyUser.getRyNo()); param.put("nick_name", jyUser.getRyNickName()); mqProducer.sendByWorkQueue("jmga-event-notice", param); }catch (Exception e){ log.info("推送事件失败,param:{}", param); } } @Override public void checkLackFile(String dataSource) { String dataFdagePath = dataSource.concat(File.separator).concat("capture").concat(File.separator).concat("data.fdage"); if(!FileUtil.exist(dataFdagePath)){ throw new BusinessException(ErrorCode.FAILURE_CODE_4001.code(), ErrorCode.FAILURE_CODE_4001.formatMessage("data.fdage")); } JSONObject fdageData = null; try { fdageData = JSON.parseObject(FileUtil.readUtf8String(dataFdagePath)); }catch (Exception e){ throw new BusinessException(ErrorCode.FAILURE_CODE_4001.code(), ErrorCode.FAILURE_CODE_4001.formatMessage("data.fdage")); } int camType = fdageData.getJSONObject("cam").getIntValue("type"); if(camType == 1 || camType == 2 || camType == 9 || camType == 10 || camType == 11){ Integer shootCount = fdageData.getJSONArray("points").size(); if(shootCount > 0){//有点位代表架站式 String parametersPath = dataSource.concat(File.separator).concat("capture").concat(File.separator).concat("parameters.json"); if(!FileUtil.exist(parametersPath)){ throw new BusinessException(ErrorCode.FAILURE_CODE_4001.code(), ErrorCode.FAILURE_CODE_4001.formatMessage("parameters.json")); } String sfmDataBinPath = dataSource.concat(File.separator).concat("capture").concat(File.separator).concat("sfm_data.bin"); if(!FileUtil.exist(sfmDataBinPath)){ throw new BusinessException(ErrorCode.FAILURE_CODE_4001.code(), ErrorCode.FAILURE_CODE_4001.formatMessage("sfm_data.bin")); } String upName = camType > 2 ? "Up.txt" : "Up.xml"; String upTxtPath = dataSource.concat(File.separator).concat("capture").concat(File.separator).concat(upName); if(!FileUtil.exist(upTxtPath)){ throw new BusinessException(ErrorCode.FAILURE_CODE_4001.code(), ErrorCode.FAILURE_CODE_4001.formatMessage(upName)); } } } if(camType == 5){ String parametersPath = dataSource.concat(File.separator).concat("capture").concat(File.separator).concat("images/parameters.json"); if(!FileUtil.exist(parametersPath)){ throw new BusinessException(ErrorCode.FAILURE_CODE_4001.code(), ErrorCode.FAILURE_CODE_4001.formatMessage("parameters.json")); } String sfmDataBinPath = dataSource.concat(File.separator).concat("capture").concat(File.separator).concat("result/reconstruction/sfm_data.bin"); if(!FileUtil.exist(sfmDataBinPath)){ throw new BusinessException(ErrorCode.FAILURE_CODE_4001.code(), ErrorCode.FAILURE_CODE_4001.formatMessage("sfm_data.bin")); } } //从data.fdage中获取需要上传的文件列表 JSONArray points = fdageData.getJSONArray("points"); if(CollUtil.isEmpty(points)){ return; } for (Object point : points) { JSONObject pointJson = (JSONObject) point; if(camType == 5){//圆周率相机 String name = pointJson.getString("name"); this.checkFileAvailable(dataSource, name); }else{//四维相机 JSONArray imgs = pointJson.getJSONArray("imgs"); for (Object img : imgs) { JSONObject imgJson = (JSONObject) img; String name = imgJson.getString("name"); this.checkFileAvailable(dataSource, name); } } } } private void checkFileAvailable(String dataSource, String fileName){ List fileList = FileUtil.loopFiles(dataSource.concat(File.separator).concat("capture")); if(CollUtil.isEmpty(fileList)){ throw new BusinessException(ErrorCode.FAILURE_CODE_4001.code(), ErrorCode.FAILURE_CODE_4001.formatMessage(fileName)); } for (File file : fileList) { if(file.getName().equals(fileName) && file.length() > 0){ return; } } throw new BusinessException(ErrorCode.FAILURE_CODE_4001.code(), ErrorCode.FAILURE_CODE_4001.formatMessage(fileName)); } @Override public void uploadScene(UploadSceneOrigParamVo param) throws Exception { User user = userService.getById(param.getUserId()); if(Objects.isNull(user)){ throw new BusinessException(ErrorCode.USER_NOT_EXIST); } String uuid = UUID.randomUUID().toString(); String zipName = uuid + ".zip"; String zipDir = "/oss/4dkankan/bd_server/"; fYunFileService.downloadFileByCommand(zipDir + zipName, param.getFilePath()); Camera camera = null; CameraDetail cameraDetail = null; String sceneNum = null; if(param.getSourceType().equals("orig")){//原始资源 String dataFdageStr = com.fdkankan.contro.util.ZipUtil.readUtf8(zipDir + zipName, "data.fdage"); if(StrUtil.isEmpty(dataFdageStr) || !JSONUtil.isJson(dataFdageStr)){ throw new BusinessException(60027, "data.fdage文件数据异常"); } JSONObject dataFdage = JSON.parseObject(dataFdageStr); String snCode = dataFdage.getJSONObject("cam").getString("uuid"); String uuidTime = dataFdage.getString("uuidtime"); String uniCode = snCode + "_" + uuidTime; int camType = dataFdage.getJSONObject("cam").getIntValue("type"); Map instorage = cameraService.instorage(camType, snCode); cameraDetail = (CameraDetail) instorage.get("cameraDetail"); ScenePlus scenePlus = scenePlusService.getByFileId(uniCode); if(scenePlus == null){ scenePlus = new ScenePlus(); sceneNum = scene3dNumService.generateSceneNum(cameraDetail.getType()); scenePlus.setNum(sceneNum); scenePlus.setCameraId(cameraDetail.getCameraId()); scenePlus.setSceneSource(SceneSourceUtil.getSceneSourceByCamType(cameraDetail.getType())); }else{ if(scenePlus.getSceneStatus() == SceneStatus.wait.code()){ throw new BusinessException(ErrorCode.FAILURE_CODE_5033); } sceneNum = scenePlus.getNum(); } scenePlus.setTitle(dataFdage.getString("name")); scenePlus.setUserId(user.getId()); scenePlus.setUpdateTime(null); scenePlus.setSceneStatus(SceneStatus.wait.code()); scenePlusService.saveOrUpdate(scenePlus); ScenePlusExt scenePlusExt = scenePlusExtService.getScenePlusExtByPlusId(scenePlus.getId()); if(scenePlusExt == null){ scenePlusExt = new ScenePlusExt(); scenePlusExt.setPlusId(scenePlus.getId()); scenePlusExt.setWebSite("/" + sceneProNewUrl + scenePlus.getNum()); String fileId = sceneFileBuildService.getFileId(snCode, uniCode); scenePlusExt.setDataSource(ConstantFilePath.BUILD_MODEL_PATH + snCode + File.separator + fileId + File.separator + uniCode); scenePlusExtService.save(scenePlusExt); } SceneEditInfo sceneEditInfo = sceneEditInfoService.getByScenePlusId(scenePlus.getId()); if(sceneEditInfo == null){ sceneEditInfo = new SceneEditInfo(); sceneEditInfo.setScenePlusId(scenePlus.getId()); sceneEditInfoService.save(sceneEditInfo); } SceneEditInfoExt sceneEditInfoExt = sceneEditInfoExtService.getByEditInfoId(sceneEditInfo.getId()); if(sceneEditInfoExt == null){ sceneEditInfoExt = new SceneEditInfoExt(); sceneEditInfoExt.setScenePlusId(scenePlus.getId()); sceneEditInfoExt.setEditInfoId(sceneEditInfo.getId()); sceneEditInfoExtService.save(sceneEditInfoExt); } SceneEditControls sceneEditControls = sceneEditControlsService.getBySceneEditId(sceneEditInfo.getId()); if(sceneEditControls == null){ sceneEditControls = new SceneEditControls(); sceneEditControls.setEditInfoId(sceneEditInfo.getId()); sceneEditControlsService.save(sceneEditControls); } }else{//离线包 String sceneJsonStr = com.fdkankan.contro.util.ZipUtil.readUtf8(zipDir + zipName, "scene.json"); if(StrUtil.isEmpty(sceneJsonStr) || !JSONUtil.isJson(sceneJsonStr)){ throw new BusinessException(60027, "scene.json文件数据异常"); } JSONObject sceneJson = JSON.parseObject(sceneJsonStr); sceneNum = sceneJson.getString("num"); ScenePlus scenePlusDb = scenePlusService.getDeletedByNum(sceneNum); if(scenePlusDb != null && scenePlusDb.getRecStatus().equalsIgnoreCase(RecStatus.VALID.code())){ throw new BusinessException(ModelingControlRespCode.NUM_EXISTS.code(), ModelingControlRespCode.NUM_EXISTS.message()); } if(scenePlusDb != null && scenePlusDb.getRecStatus().equalsIgnoreCase(RecStatus.DISABLE.code())){ scenePlusService.updateNumById(scenePlusDb.getId(), scenePlusDb.getNum() + "-" + scenePlusDb.getId()); } int camType = sceneJson.getInteger("camType"); String snCode = sceneJson.getString("snCode"); Map instorage = cameraService.instorage(camType, snCode); cameraDetail = (CameraDetail) instorage.get("cameraDetail"); ScenePlus scenePlus = new ScenePlus(); scenePlus.setNum(sceneNum); scenePlus.setCameraId(cameraDetail.getCameraId()); scenePlus.setSceneSource(SceneSourceUtil.getSceneSourceByCamType(cameraDetail.getType())); scenePlus.setTitle(sceneJson.getString("title")); scenePlus.setUserId(user.getId()); scenePlus.setUpdateTime(null); scenePlus.setSceneStatus(SceneStatus.wait.code()); scenePlus.setUploadType("offline"); scenePlus.setPayStatus(PayStatus.PAY.code()); scenePlus.setStartBuildTime(new Date()); scenePlusService.save(scenePlus); ScenePlusExt scenePlusExt = new ScenePlusExt(); scenePlusExt.setPlusId(scenePlus.getId()); String uniCode = snCode + DateExtUtil.format(new Date(), DateExtUtil.dateStyle11); String fileId = sceneFileBuildService.getFileId(snCode, uniCode); scenePlusExt.setDataSource(ConstantFilePath.BUILD_MODEL_PATH + snCode + File.separator + fileId + File.separator + uniCode); scenePlusExtService.save(scenePlusExt); } //发mq做异步处理 JSONObject content = new JSONObject(); content.put("zipPath", zipDir + zipName); content.put("num", sceneNum); content.put("sourceType", param.getSourceType()); mqProducer.sendByWorkQueue("manage-upload-scene", content); } @Override public void noticeBuildBd(String ossHost, String uuid, String dir) { String prefix = "http://" + ossHost + "/"; String path = "/oss/4dkankan/bd_server/" + dir; List files = FileUtil.loopFiles(path).stream().filter(file -> !file.getAbsolutePath().contains("backup")).collect(Collectors.toList()); files.parallelStream().forEach(file ->{ String url = prefix.concat(file.getAbsolutePath().substring(1).replace("/4dkankan","")); SendCallAlgorithmDetail detail = new SendCallAlgorithmDetail(); detail.setFileName(FileUtil.getName(file.getAbsolutePath())); detail.setUuid(uuid); SendCallAlgorithmParam param = new SendCallAlgorithmParam(); param.setFilepath(url); param.setDetails(detail); HttpUtil.post("http://127.0.0.1:8085/api/scene/file/sendCallAlgorithm", JSON.toJSONString(param)); }); } @Override public void updateSceneFailAndZipLog(String num, String failReason) throws IOException { scenePlusService.update(new LambdaUpdateWrapper() .set(ScenePlus::getSceneStatus, SceneStatus.FAILD.code()) .set(ScenePlus::getBuildFailReason, failReason) .eq(ScenePlus::getNum, num)); String logBasePath = File.separator + "mnt" + File.separator + "logs" + File.separator; String tmpPath = logBasePath + "tmp" + File.separator; String logPath = tmpPath + num + "-" + Calendar.getInstance().getTimeInMillis() + File.separator; String zipPath = tmpPath + num + "-" + Calendar.getInstance().getTimeInMillis() + ".zip"; FileUtil.mkdir(logPath); //打包服务端日志 File[] ls = FileUtil.ls(logBasePath); for (File l : ls) { if(l.isFile() || !l.getName().startsWith("modeling-control")){ continue; } this.copyLog(l.getAbsolutePath(), logPath, "info","sql","error"); } //打包算法日志 ScenePlus scenePlus = scenePlusService.getScenePlusByNum(num); ScenePlusExt scenePlusExt = scenePlusExtService.getScenePlusExtByPlusId(scenePlus.getId()); String consoleLogPath = scenePlusExt.getDataSource() + "/console.log"; String consoleOldLogPath = scenePlusExt.getDataSource() + "/console.old.log"; if(FileUtil.exist(consoleLogPath)){ FileUtil.copy(consoleLogPath, logPath,true); } if(FileUtil.exist(consoleOldLogPath)){ FileUtil.copy(consoleOldLogPath, logPath,true); } //打包数据库记录 String[] split = scenePlusExt.getDataSource().split("/"); String uuid = split[split.length -1]; List batchList = origFileUploadBatchService.list(new LambdaQueryWrapper().eq(OrigFileUploadBatch::getUuid, uuid)); if(CollUtil.isNotEmpty(batchList)){ EasyExcel.write(logPath.concat("origFileUploadBatch.xlsx"), OrigFileUploadBatch.class).sheet("origFileUploadBatch").doWrite(batchList); List batchIdList = batchList.stream().map(v -> v.getBatchId()).collect(Collectors.toList()); List uploadList = origFileUploadService.list(new LambdaQueryWrapper().in(OrigFileUpload::getBatchId, batchIdList)); if(CollUtil.isNotEmpty(uploadList)){ uploadList.stream().forEach(v->{ v.setFileUrl(v.getFileUrl().replaceAll(ipv4Pattern, "ip")); }); EasyExcel.write(logPath.concat("origFileUpload.xlsx"), OrigFileUpload.class).sheet("origFileUpload").doWrite(uploadList); } } List processLogList = sceneBuildProcessLogService.listByNum(num); if(CollUtil.isNotEmpty(processLogList)){ EasyExcel.write(logPath.concat("sceneBuildProcessLog.xlsx"), SceneBuildProcessLog.class).sheet("sceneBuildProcessLog").doWrite(processLogList); } List buildLogList = buildLogService.list(new LambdaQueryWrapper().eq(BuildLog::getSceneNum, num)); if(CollUtil.isNotEmpty(buildLogList)){ EasyExcel.write(logPath.concat("buildLog.xlsx"), BuildLog.class).sheet("buildLog").doWrite(buildLogList); } //上传日志,删除本地压缩包 ZipUtil.zip(logPath, zipPath, false); String logName = "build_log/" + num + "/" + num + "-log.zip"; if(fYunFileService.fileExist(logName)){ fYunFileService.copyFileInBucket(logName, logName + "-bak-" + DateExtUtil.format(Calendar.getInstance().getTime(), DateExtUtil.dateStyle11)); } fYunFileService.uploadFile(zipPath, logName); } private void copyLog(String serverPath, String tmpPath, String... levels) throws IOException { level: for (String level : levels) { String infoPath = serverPath + File.separator + level + File.separator; List fileList = FileUtil.loopFiles(infoPath); if(CollUtil.isEmpty(fileList)){ continue; } CollUtil.reverse(fileList); for (File file : fileList) { int i = fileList.indexOf(file); if(i > 7){ continue level; } String targetPath = file.getAbsolutePath().replace(FileUtil.getParent(serverPath, 1), tmpPath); File targetLog = new File(targetPath); targetLog.createNewFile(); // FileUtil.copy(file.getAbsolutePath(), targetPath, true); // String content = FileUtil.readUtf8String(targetPath); // content = content.replace("公安", "xx").replaceAll(ipv4Pattern, "ip"); // FileUtil.writeUtf8String(content, targetPath); try (Stream lines = Files.lines(Paths.get(file.getAbsolutePath()), StandardCharsets.UTF_8)) { lines.map(line -> line.replace("公安", "xx").replaceAll(ipv4Pattern, "ip")) .forEach(line -> FileUtil.appendUtf8String(line, targetLog)); // 或者进行其他处理 } catch (IOException e) { e.printStackTrace(); } } } } }