冷归档

src/main/java/com/fdkankan/scene/controller/SceneColdStorageLogController.java

@@ -0,0 +1,21 @@
+package com.fdkankan.scene.controller;
+
+
+import org.springframework.web.bind.annotation.RequestMapping;
+
+import org.springframework.web.bind.annotation.RestController;
+
+/**
+ * <p>
+ * 场景冷归档日志表 前端控制器
+ * </p>
+ *
+ * @author 
+ * @since 2023-07-12
+ */
+@RestController
+@RequestMapping("/scene/sceneColdStorageLog")
+public class SceneColdStorageLogController {
+
+}
+

src/main/java/com/fdkankan/scene/entity/SceneColdStorageLog.java

@@ -0,0 +1,75 @@
+package com.fdkankan.scene.entity;
+
+import com.baomidou.mybatisplus.annotation.IdType;
+import com.baomidou.mybatisplus.annotation.TableField;
+import com.baomidou.mybatisplus.annotation.TableId;
+import com.baomidou.mybatisplus.annotation.TableLogic;
+import com.baomidou.mybatisplus.annotation.TableName;
+import java.io.Serializable;
+import java.util.Date;
+import lombok.Getter;
+import lombok.Setter;
+
+/**
+ * <p>
+ * 场景冷归档日志表
+ * </p>
+ *
+ * @author 
+ * @since 2023-07-12
+ */
+@Getter
+@Setter
+@TableName("t_scene_cold_storage_log")
+public class SceneColdStorageLog implements Serializable {
+
+    private static final long serialVersionUID = 1L;
+
+    @TableId(value = "id", type = IdType.AUTO)
+    private Long id;
+
+    /**
+     * 场景码
+     */
+    @TableField("num")
+    private String num;
+
+    /**
+     * 封存地址
+     */
+    @TableField("path")
+    private String path;
+
+    /**
+     * 操作类型（1-封存，2-解封存）
+     */
+    @TableField("type")
+    private Integer type;
+
+    @TableField("state")
+    private Integer state;
+
+    @TableField("reason")
+    private String reason;
+
+    /**
+     * 创建时间
+     */
+    @TableField("create_time")
+    private Date createTime;
+
+    /**
+     * 修改时间
+     */
+    @TableField("update_time")
+    private Date updateTime;
+
+    /**
+     * A-有效，I-无效
+     */
+    @TableField("rec_status")
+    @TableLogic(value = "A", delval = "I")
+    private String recStatus;
+
+
+}

src/main/java/com/fdkankan/scene/generate/AutoGenerate.java

@@ -18,7 +18,7 @@ public class AutoGenerate {
         String path =System.getProperty("user.dir");
 
         generate(path,"scene", getTables(new String[]{
-                "t_scene_clean_orig"
+                "t_scene_cold_storage_log"
         }));
 
 //        generate(path,"goods", getTables(new String[]{

src/main/java/com/fdkankan/scene/mapper/ISceneColdStorageLogMapper.java

@@ -0,0 +1,18 @@
+package com.fdkankan.scene.mapper;
+
+import com.fdkankan.scene.entity.SceneColdStorageLog;
+import com.baomidou.mybatisplus.core.mapper.BaseMapper;
+import org.apache.ibatis.annotations.Mapper;
+
+/**
+ * <p>
+ * 场景冷归档日志表 Mapper 接口
+ * </p>
+ *
+ * @author 
+ * @since 2023-07-12
+ */
+@Mapper
+public interface ISceneColdStorageLogMapper extends BaseMapper<SceneColdStorageLog> {
+
+}

src/main/java/com/fdkankan/scene/mapper/IScenePlusMapper.java

@@ -26,4 +26,6 @@ public interface IScenePlusMapper extends BaseMapper<ScenePlus> {
 
     List<SceneBean> listCleanOss4TestCamera(Set<Long> cameraIds, Date time);
 
+    List<SceneBean> selectColdStorageScene(Date time);
+
 }

src/main/java/com/fdkankan/scene/mapper/ISceneProMapper.java

@@ -26,6 +26,8 @@ public interface ISceneProMapper extends BaseMapper<ScenePro> {
 
     List<SceneBean> listCleanOss4TestCamera(Set<Long> cameraIds, Date time);
 
+    List<SceneBean> selectColdStorageScene(Date time);
+
 
 
 }

src/main/java/com/fdkankan/scene/schedule/ScheduleJob.java

@@ -1,10 +1,7 @@
 package com.fdkankan.scene.schedule;
 
-import cn.hutool.core.util.StrUtil;
-import com.fdkankan.redis.constant.RedisKey;
 import com.fdkankan.redis.constant.RedisLockKey;
 import com.fdkankan.redis.util.RedisLockUtil;
-import com.fdkankan.redis.util.RedisUtil;
 import com.fdkankan.scene.service.ISceneAsynOperLogService;
 import com.fdkankan.scene.service.ISceneCleanOrigService;
 import lombok.extern.log4j.Log4j2;
@@ -41,45 +38,45 @@ public class ScheduleJob {
         log.info("定时清除全景图压缩包完毕");
     }
 
-    /**
-     * 删除场景原始资源
-     * 每天凌晨执行
-     */
-    @Scheduled(cron="0 0 1 * * ?")
-    public void cleanOssHomeV3() {
-        log.info("删除v3场景原始资源开始");
-        String lockKey = RedisLockKey.LOCK_CLEAN_SCENE_ORIG_V3;
-        try {
-            boolean lock = redisLockUtil.lock(lockKey, 24*60*60);
-            if(!lock){
-                return;
-            }
-            sceneCleanOrigService.cleanOrigV3();
-        }finally {
-            redisLockUtil.unlockLua(lockKey);
-        }
-        log.info("删除v3场景原始资源结束");
-    }
-
-    /**
-     * 删除场景原始资源
-     * 每天凌晨执行
-     */
-    @Scheduled(cron="0 0 1 * * ?")
-    public void cleanOssHomeV4() {
-        log.info("删除v4场景原始资源开始");
-        String lockKey = RedisLockKey.LOCK_CLEAN_SCENE_ORIG_V4;
-        try {
-            boolean lock = redisLockUtil.lock(lockKey, 24*60*60);
-            if(!lock){
-                return;
-            }
-            sceneCleanOrigService.cleanOrigV4();
-        }finally {
-            redisLockUtil.unlockLua(lockKey);
-        }
-        log.info("删除v4场景原始资源结束");
-    }
+//    /**
+//     * 删除场景原始资源
+//     * 每天凌晨执行
+//     */
+//    @Scheduled(cron="0 0 1 * * ?")
+//    public void cleanOssHomeV3() {
+//        log.info("删除v3场景原始资源开始");
+//        String lockKey = RedisLockKey.LOCK_CLEAN_SCENE_ORIG_V3;
+//        try {
+//            boolean lock = redisLockUtil.lock(lockKey, 24*60*60);
+//            if(!lock){
+//                return;
+//            }
+//            sceneCleanOrigService.cleanOrigV3();
+//        }finally {
+//            redisLockUtil.unlockLua(lockKey);
+//        }
+//        log.info("删除v3场景原始资源结束");
+//    }
+//
+//    /**
+//     * 删除场景原始资源
+//     * 每天凌晨执行
+//     */
+//    @Scheduled(cron="0 0 1 * * ?")
+//    public void cleanOssHomeV4() {
+//        log.info("删除v4场景原始资源开始");
+//        String lockKey = RedisLockKey.LOCK_CLEAN_SCENE_ORIG_V4;
+//        try {
+//            boolean lock = redisLockUtil.lock(lockKey, 24*60*60);
+//            if(!lock){
+//                return;
+//            }
+//            sceneCleanOrigService.cleanOrigV4();
+//        }finally {
+//            redisLockUtil.unlockLua(lockKey);
+//        }
+//        log.info("删除v4场景原始资源结束");
+//    }
 
     /**
      * 删除已删除场景原始资源及caches
@@ -150,4 +147,44 @@ public class ScheduleJob {
 //        sceneCleanOrigService.cleanOss4TestCameraV4();
 //        log.info("删除已删除V4场景原始资源结束");
 //    }
+
+    /**
+     * 删除场景原始资源
+     * 每天凌晨执行
+     */
+    @Scheduled(cron="0 0 1 * * ?")
+    public void coldStorageHomeV3() {
+        log.info("冻结v3场景原始资源开始");
+        String lockKey = RedisLockKey.LOCK_CLEAN_SCENE_ORIG_V3;
+        try {
+            boolean lock = redisLockUtil.lock(lockKey, 24*60*60);
+            if(!lock){
+                return;
+            }
+            sceneCleanOrigService.coldStorageHomeV3();
+        }finally {
+            redisLockUtil.unlockLua(lockKey);
+        }
+        log.info("冻结v3场景原始资源结束");
+    }
+
+    /**
+     * 删除场景原始资源
+     * 每天凌晨执行
+     */
+    @Scheduled(cron="0 0 1 * * ?")
+    public void coldStorageHomeV4() {
+        log.info("冻结v4场景原始资源开始");
+        String lockKey = RedisLockKey.LOCK_CLEAN_SCENE_ORIG_V4;
+        try {
+            boolean lock = redisLockUtil.lock(lockKey, 24*60*60);
+            if(!lock){
+                return;
+            }
+            sceneCleanOrigService.coldStorageHomeV4();
+        }finally {
+            redisLockUtil.unlockLua(lockKey);
+        }
+        log.info("冻结v4场景原始资源结束");
+    }
 }

src/main/java/com/fdkankan/scene/service/ISceneCleanOrigService.java

@@ -25,4 +25,8 @@ public interface ISceneCleanOrigService extends IService<SceneCleanOrig> {
 
     void cleanOss4TestCameraV4();
 
+    void coldStorageHomeV3();
+
+    void coldStorageHomeV4();
+
 }

src/main/java/com/fdkankan/scene/service/ISceneColdStorageLogService.java

@@ -0,0 +1,16 @@
+package com.fdkankan.scene.service;
+
+import com.fdkankan.scene.entity.SceneColdStorageLog;
+import com.baomidou.mybatisplus.extension.service.IService;
+
+/**
+ * <p>
+ * 场景冷归档日志表 服务类
+ * </p>
+ *
+ * @author 
+ * @since 2023-07-12
+ */
+public interface ISceneColdStorageLogService extends IService<SceneColdStorageLog> {
+    void saveLog(String num, String path, Integer type, Integer state, String reason);
+}

src/main/java/com/fdkankan/scene/service/IScenePlusService.java

@@ -43,4 +43,6 @@ public interface IScenePlusService extends IService<ScenePlus> {
 
     List<SceneBean> listCleanOss4TestCamera(Set<Long> cameraIds, int cleanOrigMonth);
 
+    List<SceneBean> listColdStorageScene(int month);
+
 }

src/main/java/com/fdkankan/scene/service/ISceneProService.java

@@ -58,4 +58,6 @@ public interface ISceneProService extends IService<ScenePro> {
 
     List<SceneBean> listCleanOss4TestCamera(Set<Long> cameraIds, int month);
 
+    List<SceneBean> listColdStorageScene(int month);
+
 }

src/main/java/com/fdkankan/scene/service/impl/SceneCleanOrigServiceImpl.java

@@ -49,12 +49,18 @@ public class SceneCleanOrigServiceImpl extends ServiceImpl<ISceneCleanOrigMapper
 
     @Value("${scene.cleanOrig.month}")
     private Integer cleanOrigMonth;
+    @Value("${scene.coldStorage.month}")
+    private Integer coldStorageMonth;
     @Value("${scene.cleanDeleted.month}")
     private Integer cleanDeletedMonth;
     @Value("#{'${scene.cleanTestCamera.snCode:}'.split(',')}")
     private List<String> testSnCodeList;
     @Value("${scene.cleanTestCamera.month}")
     private Integer cleanTestCameraMonth;
+    @Value("${fyun.bucket}")
+    private String bucket;
+    @Value("${fyun.coldBucket}")
+    private String coldBucket;
     @Autowired
     private ICameraService cameraService;
 
@@ -69,6 +75,8 @@ public class SceneCleanOrigServiceImpl extends ServiceImpl<ISceneCleanOrigMapper
     private FYunFileServiceInterface fYunFileService;
     @Autowired
     private RedisUtil redisUtil;
+    @Autowired
+    private ISceneColdStorageLogService sceneColdStorageLogService;
 
     @Override
     public void cleanOrigV4() {
@@ -231,4 +239,57 @@ public class SceneCleanOrigServiceImpl extends ServiceImpl<ISceneCleanOrigMapper
         this.cleanOrig4Delete(sceneBeans, true, 3);
     }
 
+    @Override
+    public void coldStorageHomeV3() {
+        //查询所有计算时间超过限定时间的场景,计算成功、未被删除
+        List<SceneBean> sceneBeans = sceneProService.listColdStorageScene(coldStorageMonth);
+        this.coldStorage(sceneBeans);
+    }
+
+    @Override
+    public void coldStorageHomeV4() {
+        //查询所有计算时间超过限定时间的场景,计算成功、未被删除
+        List<SceneBean> sceneBeans = scenePlusService.listColdStorageScene(coldStorageMonth);
+        this.coldStorage(sceneBeans);
+    }
+
+    private void coldStorage(List<SceneBean> sceneBeans){
+        if(CollUtil.isEmpty(sceneBeans)){
+            return;
+        }
+        sceneBeans.parallelStream().forEach(scene->{
+            boolean lock = this.lock(scene.getDataSource());
+            try {
+                if(lock) {
+                    this.coldStorageHandler(scene);
+                    sceneColdStorageLogService.saveLog(scene.getNum(), scene.getDataSource(), 1, 1, null);
+                }
+            }catch (Exception e){
+                log.error("冷归档失败,num:{}" + scene.getNum(), e);
+                sceneColdStorageLogService.saveLog(scene.getNum(), scene.getDataSource(),1, CommonSuccessStatus.FAIL.code(), ExceptionUtil.stacktraceToString(e, 3000));
+            }finally {
+                this.releaseLock(scene.getDataSource());
+            }
+        });
+    }
+
+    private void coldStorageHandler(SceneBean scene){
+        String dataSource = scene.getDataSource();
+        if(StrUtil.isEmpty(dataSource) || dataSource.length() < 10) {
+            return;
+        }
+        String homePath = dataSource.replace(ConstantFilePath.BUILD_MODEL_PATH, ConstantFilePath.OSS_PREFIX);
+
+        //将文件复制到冷归档bucket
+        fYunFileService.copyFileBetweenBucket(bucket, homePath, coldBucket, homePath);
+
+        List<String> origList = fYunFileService.listRemoteFiles(bucket, homePath);
+        List<String> coldList = fYunFileService.listRemoteFiles(coldBucket, homePath);
+        if(origList.size() != coldList.size()){
+            throw new RuntimeException("复制文件到冷归档bucket失败");
+        }
+
+        //删除标准bucket文件
+        fYunFileService.deleteFolder(homePath);
+    }
 }

src/main/java/com/fdkankan/scene/service/impl/SceneColdStorageLogServiceImpl.java

@@ -0,0 +1,33 @@
+package com.fdkankan.scene.service.impl;
+
+import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
+import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
+import com.fdkankan.scene.entity.SceneColdStorageLog;
+import com.fdkankan.scene.mapper.ISceneColdStorageLogMapper;
+import com.fdkankan.scene.service.ISceneColdStorageLogService;
+import org.springframework.stereotype.Service;
+
+/**
+ * <p>
+ * 场景冷归档日志表 服务实现类
+ * </p>
+ *
+ * @author 
+ * @since 2023-07-12
+ */
+@Service
+public class SceneColdStorageLogServiceImpl extends ServiceImpl<ISceneColdStorageLogMapper, SceneColdStorageLog> implements ISceneColdStorageLogService {
+
+    public void saveLog(String num, String path, Integer type, Integer state, String reason){
+        //清除旧的日志
+        this.remove(new LambdaQueryWrapper<SceneColdStorageLog>().eq(SceneColdStorageLog::getNum, num));
+        SceneColdStorageLog sceneColdStorageLog = new SceneColdStorageLog();
+        sceneColdStorageLog.setNum(num);
+        sceneColdStorageLog.setPath(path);
+        sceneColdStorageLog.setType(type);
+        sceneColdStorageLog.setState(state);
+        sceneColdStorageLog.setReason(reason);
+        this.saveOrUpdate(sceneColdStorageLog);
+    }
+
+}

src/main/java/com/fdkankan/scene/service/impl/ScenePlusServiceImpl.java

@@ -266,4 +266,11 @@ public class ScenePlusServiceImpl extends ServiceImpl<IScenePlusMapper, ScenePlu
         return this.baseMapper.listCleanOss4TestCamera(cameraIds, time);
 
     }
+
+    @Override
+    public List<SceneBean> listColdStorageScene(int cleanOrigMonth) {
+        Date time = Calendar.getInstance().getTime();
+        time = DateUtil.beginOfDay(DateUtil.offset(time, DateField.MONTH, -cleanOrigMonth));
+        return this.baseMapper.selectColdStorageScene(time);
+    }
 }

src/main/java/com/fdkankan/scene/service/impl/SceneProServiceImpl.java

@@ -1148,4 +1148,11 @@ public class SceneProServiceImpl extends ServiceImpl<ISceneProMapper, ScenePro>
         time = DateUtil.beginOfDay(DateUtil.offset(time, DateField.MONTH, -month));
         return this.baseMapper.listCleanOss4TestCamera(cameraIds, time);
     }
+
+    @Override
+    public List<SceneBean> listColdStorageScene(int cleanOrigMonth) {
+        Date time = Calendar.getInstance().getTime();
+        time = DateUtil.beginOfDay(DateUtil.offset(time, DateField.MONTH, -cleanOrigMonth));
+        return this.baseMapper.selectColdStorageScene(time);
+    }
 }

src/main/resources/bootstrap-test.yml

@@ -33,8 +33,8 @@ spring:
             group: DEFAULT_GROUP
             refresh: true
       discovery:
-        namespace: ${spring.cloud.nacos.namespace}
-#        namespace: public
+#        namespace: ${spring.cloud.nacos.namespace}
+        namespace: public
 
 
 

src/main/resources/mapper/scene/SceneColdStorageLogMapper.xml

@@ -0,0 +1,5 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN" "http://mybatis.org/dtd/mybatis-3-mapper.dtd">
+<mapper namespace="com.fdkankan.scene.mapper.ISceneColdStorageLogMapper">
+
+</mapper>

src/main/resources/mapper/scene/ScenePlusMapper.xml

@@ -33,4 +33,14 @@
         and NOT EXISTS (select o.num from t_scene_clean_orig o where o.rec_status = 'A' and o.num = plus.num and o.type = 3 and ext.algorithm_time <![CDATA[ < ]]> o.create_time)
     </select>
 
+    <select id="selectColdStorageScene" resultType="com.fdkankan.scene.bean.SceneBean">
+        select plus.num, ext.data_source
+        from t_scene_plus plus
+        left join t_scene_plus_ext ext on plus.id = ext.plus_id
+        where plus.scene_status in (1,-2) and ext.algorithm_time <![CDATA[ < ]]> #{time}
+        and ext.data_source is NOT null
+        and NOT EXISTS (select c.num from t_scene_cold_storage_log c where c.rec_status = 'A' and c.num = plus.num and ext.algorithm_time <![CDATA[ < ]]> c.create_time)
+        and NOT EXISTS (select o.num from t_scene_clean_orig o where o.rec_status = 'A' and o.state != -1 and o.num = plus.num and ext.algorithm_time <![CDATA[ < ]]> o.create_time)
+    </select>
+
 </mapper>

src/main/resources/mapper/scene/SceneProMapper.xml

@@ -30,4 +30,14 @@
         and NOT EXISTS (select o.num from t_scene_clean_orig o where o.rec_status = 'A' and o.num = pro.num and o.type = 3 and pro.create_time <![CDATA[ < ]]> o.create_time)
     </select>
 
+    <select id="selectColdStorageScene" resultType="com.fdkankan.scene.bean.SceneBean">
+        select pro.num, pro.data_source
+        from t_scene_pro pro
+        where pro.status in (1,-2) and pro.create_time <![CDATA[ < ]]> #{time}
+        and pro.data_source is not null
+        and pro.is_upgrade = 0
+        and NOT EXISTS (select c.num from t_scene_cold_storage_log c where c.rec_status = 'A' and c.num = pro.num and pro.create_time <![CDATA[ < ]]> c.create_time)
+        and NOT EXISTS (select o.num from t_scene_clean_orig o where o.rec_status = 'A' and o.state != -1 and o.num = pro.num and pro.create_time <![CDATA[ < ]]> o.create_time)
+    </select>
+
 </mapper>