v4.9.0 定时任务删除资源

src/main/java/com/fdkankan/scene/bean/SceneBean.java

@@ -8,5 +8,4 @@ public class SceneBean {
     private String num;
 
     private String dataSource;
-
 }

src/main/java/com/fdkankan/scene/listener/RabbitMqListener.java

@@ -1,77 +1,77 @@
-package com.fdkankan.scene.listener;
-
-import com.alibaba.fastjson.JSON;
-import com.fdkankan.scene.entity.DownloadTourVideo;
-import com.fdkankan.scene.service.IDownloadTourVideoService;
-import com.rabbitmq.client.Channel;
-import java.nio.charset.StandardCharsets;
-import lombok.extern.slf4j.Slf4j;
-import org.springframework.amqp.core.Message;
-import org.springframework.amqp.rabbit.annotation.Queue;
-import org.springframework.amqp.rabbit.annotation.RabbitListener;
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.beans.factory.annotation.Value;
-import org.springframework.stereotype.Component;
-
-/**
- * <p>
- * TODO
- * </p>
- *
- * @author dengsixing
- * @since 2022/4/19
- **/
-@Slf4j
-@Component
-public class RabbitMqListener {
-
-    @Value("${queue.scene.transfer-tour-video}")
-    private String downloadTourVideoQueue;
-
-    @Autowired
-    private IDownloadTourVideoService downloadTourVideoService;
-
-
-
-
-    /**
-     * 开启了手动确认模式，如果没有手动确认，消费者不会重试，当服务重启时会再次消费，因为rabbitmq认为你还没有处理完你的业务
-     * queuesToDeclare = @Queue("${queue.modeling.modeling-test}"),  如果队列不不存在会自动创建队列
-     * concurrency = "3"    设置消费线程数，每个线程每次只拉取一条消息消费
-     */
-    @RabbitListener(
-        queuesToDeclare = @Queue("${queue.scene.transfer-tour-video}")
-    )
-    public void transferTourVideo(Channel channel, Message message) throws Exception {
-        String messageId = message.getMessageProperties().getMessageId();
-        String msg = new String(message.getBody(), StandardCharsets.UTF_8);
-        log.info("开始消费消息,id:{},queue:{},content:{}", messageId, downloadTourVideoQueue, msg);
-        DownloadTourVideo downloadTourVideo = JSON.parseObject(msg, DownloadTourVideo.class);
-        downloadTourVideoService.transferTourVideo(downloadTourVideo);
-        log.info("deliverTag:" + message.getMessageProperties().getDeliveryTag());
-        channel.basicAck(message.getMessageProperties().getDeliveryTag(), false);
-        log.info("结束消费消息,id:{}", messageId);
-    }
-
-    /**
-     * 开启了手动确认模式，如果没有手动确认，消费者不会重试，当服务重启时会再次消费，因为rabbitmq认为你还没有处理完你的业务
-     * queuesToDeclare = @Queue("${queue.modeling.modeling-test}"),  如果队列不不存在会自动创建队列
-     * concurrency = "3"    设置消费线程数，每个线程每次只拉取一条消息消费
-     */
-    @RabbitListener(
-        queuesToDeclare = @Queue("test_dsx")
-    )
-    public void test(Channel channel, Message message) throws Exception {
-        String messageId = message.getMessageProperties().getMessageId();
-        String msg = new String(message.getBody(), StandardCharsets.UTF_8);
-        channel.basicAck(message.getMessageProperties().getDeliveryTag(), false);
-        log.info("开始消费消息,id:{},queue:{},content:{}", messageId, "test_dsx", msg);
-        Thread.sleep(10000L);
-        log.info("deliverTag:" + message.getMessageProperties().getDeliveryTag());
-        log.info("结束消费消息,id:{}", messageId);
-    }
-
-
-
-
-}
+//package com.fdkankan.scene.listener;
+//
+//import com.alibaba.fastjson.JSON;
+//import com.fdkankan.scene.entity.DownloadTourVideo;
+//import com.fdkankan.scene.service.IDownloadTourVideoService;
+//import com.rabbitmq.client.Channel;
+//import java.nio.charset.StandardCharsets;
+//import lombok.extern.slf4j.Slf4j;
+//import org.springframework.amqp.core.Message;
+//import org.springframework.amqp.rabbit.annotation.Queue;
+//import org.springframework.amqp.rabbit.annotation.RabbitListener;
+//import org.springframework.beans.factory.annotation.Autowired;
+//import org.springframework.beans.factory.annotation.Value;
+//import org.springframework.stereotype.Component;
+//
+///**
+// * <p>
+// * TODO
+// * </p>
+// *
+// * @author dengsixing
+// * @since 2022/4/19
+// **/
+//@Slf4j
+//@Component
+//public class RabbitMqListener {
+//
+//    @Value("${queue.scene.transfer-tour-video}")
+//    private String downloadTourVideoQueue;
+//
+//    @Autowired
+//    private IDownloadTourVideoService downloadTourVideoService;
+//
+//
+//
+//
+//    /**
+//     * 开启了手动确认模式，如果没有手动确认，消费者不会重试，当服务重启时会再次消费，因为rabbitmq认为你还没有处理完你的业务
+//     * queuesToDeclare = @Queue("${queue.modeling.modeling-test}"),  如果队列不不存在会自动创建队列
+//     * concurrency = "3"    设置消费线程数，每个线程每次只拉取一条消息消费
+//     */
+//    @RabbitListener(
+//        queuesToDeclare = @Queue("${queue.scene.transfer-tour-video}")
+//    )
+//    public void transferTourVideo(Channel channel, Message message) throws Exception {
+//        String messageId = message.getMessageProperties().getMessageId();
+//        String msg = new String(message.getBody(), StandardCharsets.UTF_8);
+//        log.info("开始消费消息,id:{},queue:{},content:{}", messageId, downloadTourVideoQueue, msg);
+//        DownloadTourVideo downloadTourVideo = JSON.parseObject(msg, DownloadTourVideo.class);
+//        downloadTourVideoService.transferTourVideo(downloadTourVideo);
+//        log.info("deliverTag:" + message.getMessageProperties().getDeliveryTag());
+//        channel.basicAck(message.getMessageProperties().getDeliveryTag(), false);
+//        log.info("结束消费消息,id:{}", messageId);
+//    }
+//
+//    /**
+//     * 开启了手动确认模式，如果没有手动确认，消费者不会重试，当服务重启时会再次消费，因为rabbitmq认为你还没有处理完你的业务
+//     * queuesToDeclare = @Queue("${queue.modeling.modeling-test}"),  如果队列不不存在会自动创建队列
+//     * concurrency = "3"    设置消费线程数，每个线程每次只拉取一条消息消费
+//     */
+//    @RabbitListener(
+//        queuesToDeclare = @Queue("test_dsx")
+//    )
+//    public void test(Channel channel, Message message) throws Exception {
+//        String messageId = message.getMessageProperties().getMessageId();
+//        String msg = new String(message.getBody(), StandardCharsets.UTF_8);
+//        channel.basicAck(message.getMessageProperties().getDeliveryTag(), false);
+//        log.info("开始消费消息,id:{},queue:{},content:{}", messageId, "test_dsx", msg);
+//        Thread.sleep(10000L);
+//        log.info("deliverTag:" + message.getMessageProperties().getDeliveryTag());
+//        log.info("结束消费消息,id:{}", messageId);
+//    }
+//
+//
+//
+//
+//}

src/main/java/com/fdkankan/scene/mapper/ISceneProMapper.java

@@ -1,9 +1,13 @@
 package com.fdkankan.scene.mapper;
 
 import com.baomidou.mybatisplus.core.mapper.BaseMapper;
+import com.fdkankan.scene.bean.SceneBean;
 import com.fdkankan.scene.entity.ScenePro;
 import org.apache.ibatis.annotations.Mapper;
 
+import java.util.Date;
+import java.util.List;
+
 /**
  * <p>
  * pro场景表 Mapper 接口
@@ -15,4 +19,6 @@ import org.apache.ibatis.annotations.Mapper;
 @Mapper
 public interface ISceneProMapper extends BaseMapper<ScenePro> {
 
+    List<SceneBean> selectCleanOrigScene(Date time);
+
 }

src/main/java/com/fdkankan/scene/schedule/ScheduleJob.java

@@ -1,5 +1,9 @@
 package com.fdkankan.scene.schedule;
 
+import com.fdkankan.redis.constant.RedisKey;
+import com.fdkankan.redis.constant.RedisLockKey;
+import com.fdkankan.redis.util.RedisLockUtil;
+import com.fdkankan.redis.util.RedisUtil;
 import com.fdkankan.scene.service.ISceneAsynOperLogService;
 import com.fdkankan.scene.service.ISceneCleanOrigService;
 import lombok.extern.log4j.Log4j2;
@@ -8,6 +12,8 @@ import org.springframework.cloud.context.config.annotation.RefreshScope;
 import org.springframework.scheduling.annotation.Scheduled;
 import org.springframework.stereotype.Component;
 
+import javax.annotation.PostConstruct;
+
 @RefreshScope
 @Log4j2
 @Component
@@ -17,6 +23,8 @@ public class ScheduleJob {
     private ISceneAsynOperLogService sceneAsynOperLogService;
     @Autowired
     private ISceneCleanOrigService sceneCleanOrigService;
+    @Autowired
+    private RedisLockUtil redisLockUtil;
 
     /**
      * 每天凌晨一点执行
@@ -32,19 +40,41 @@ public class ScheduleJob {
      * 删除场景原始资源
      * 每天凌晨执行
      */
-    @Scheduled(cron="1 0 0 * * ?")
+    @PostConstruct
+    @Scheduled(cron="0 0 1 * * ?")
     public void cleanOssHomeV3() {
-        log.info("定时清除全景图压缩包开始");
-        log.info("定时清除全景图压缩包完毕");
+        log.info("删除v3场景原始资源开始");
+        String lockKey = RedisLockKey.LOCK_CLEAN_SCENE_ORIG_V3;
+        try {
+            boolean lock = redisLockUtil.lock(lockKey, RedisKey.CAMERA_EXPIRE_7_TIME);
+            if(!lock){
+                return;
+            }
+            sceneCleanOrigService.cleanOrigV3();
+        }finally {
+            redisLockUtil.unlockLua(lockKey);
+        }
+        log.info("删除v3场景原始资源结束");
     }
 
     /**
      * 删除场景原始资源
      * 每天凌晨执行
      */
-    @Scheduled(cron="1 0 0 * * ?")
+    @PostConstruct
+    @Scheduled(cron="0 0 1 * * ?")
     public void cleanOssHomeV4() {
-        log.info("定时清除全景图压缩包开始");
-        log.info("定时清除全景图压缩包完毕");
+        log.info("删除v4场景原始资源开始");
+        String lockKey = RedisLockKey.LOCK_CLEAN_SCENE_ORIG_V4;
+        try {
+            boolean lock = redisLockUtil.lock(lockKey, RedisKey.CAMERA_EXPIRE_7_TIME);
+            if(!lock){
+                return;
+            }
+            sceneCleanOrigService.cleanOrigV4();
+        }finally {
+            redisLockUtil.unlockLua(lockKey);
+        }
+        log.info("删除v4场景原始资源结束");
     }
 }

src/main/java/com/fdkankan/scene/service/ISceneCleanOrigService.java

@@ -1,7 +1,7 @@
 package com.fdkankan.scene.service;
 
-import com.fdkankan.scene.entity.SceneCleanOrig;
 import com.baomidou.mybatisplus.extension.service.IService;
+import com.fdkankan.scene.entity.SceneCleanOrig;
 
 /**
  * <p>

src/main/java/com/fdkankan/scene/service/IScenePlusService.java

@@ -35,6 +35,6 @@ public interface IScenePlusService extends IService<ScenePlus> {
 
     ResultData uploadShareLogo(String num, MultipartFile file) throws Exception;
 
-    List<SceneBean> listCleanOrigScene();
+    List<SceneBean> listCleanOrigScene(int cleanOrigMonth);
 
 }

src/main/java/com/fdkankan/scene/service/ISceneProService.java

@@ -1,6 +1,7 @@
 package com.fdkankan.scene.service;
 
 import com.baomidou.mybatisplus.extension.service.IService;
+import com.fdkankan.scene.bean.SceneBean;
 import com.fdkankan.web.response.ResultData;
 import com.fdkankan.scene.entity.ScenePro;
 import com.fdkankan.scene.vo.BaseDataParamVO;
@@ -49,4 +50,6 @@ public interface ISceneProService extends IService<ScenePro> {
 
     ScenePro getByNum(String num);
 
+    List<SceneBean> listCleanOrigScene(int cleanOrigMonth);
+
 }

src/main/java/com/fdkankan/scene/service/impl/SceneCleanOrigServiceImpl.java

@@ -1,5 +1,14 @@
 package com.fdkankan.scene.service.impl;
 
+import cn.hutool.core.collection.CollUtil;
+import cn.hutool.core.exceptions.ExceptionUtil;
+import cn.hutool.core.util.StrUtil;
+import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
+import com.fdkankan.common.constant.CommonStatus;
+import com.fdkankan.common.constant.CommonSuccessStatus;
+import com.fdkankan.common.constant.OperationType;
+import com.fdkankan.fyun.face.FYunFileServiceInterface;
+import com.fdkankan.model.constants.ConstantFilePath;
 import com.fdkankan.scene.bean.SceneBean;
 import com.fdkankan.scene.entity.SceneCleanOrig;
 import com.fdkankan.scene.mapper.ISceneCleanOrigMapper;
@@ -9,10 +18,15 @@ import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
 import com.fdkankan.scene.service.IScenePlusExtService;
 import com.fdkankan.scene.service.IScenePlusService;
 import com.fdkankan.scene.service.ISceneProService;
+import lombok.extern.slf4j.Slf4j;
 import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Value;
 import org.springframework.stereotype.Service;
+import sun.font.TextRecord;
 
+import java.util.Date;
 import java.util.List;
+import java.util.Objects;
 
 /**
  * <p>
@@ -22,54 +36,73 @@ import java.util.List;
  * @author 
  * @since 2023-03-29
  */
+@Slf4j
 @Service
 public class SceneCleanOrigServiceImpl extends ServiceImpl<ISceneCleanOrigMapper, SceneCleanOrig> implements ISceneCleanOrigService {
 
+    @Value("${scene.cleanOrig.month}")
+    private Integer cleanOrigMonth;
+
     @Autowired
     private ISceneProService sceneProService;
     @Autowired
     private IScenePlusService scenePlusService;
     @Autowired
     private IScenePlusExtService scenePlusExtService;
-
+    @Autowired
+    private FYunFileServiceInterface fYunFileService;
 
     @Override
     public void cleanOrigV4() {
-
-        //查询所有计算时间超过限定时间的场景,计算成功、未被删除
-        List<SceneBean> sceneBeans = scenePlusService.listCleanOrigScene();
-
-        //写入t_scene_clean_orig表
-
-
-        //列举原始资源目录
-
-
-        //如果目录文件不为空，则执行批量删除
-
-        //更新t_scene_clean_orig表
+        //查询所有计算时间超过限定时间的场景,计算成功、未被删除、最后一次计算后未被删除过的
+        List<SceneBean> sceneBeans = scenePlusService.listCleanOrigScene(cleanOrigMonth);
+        if(CollUtil.isEmpty(sceneBeans)){
+            return;
+        }
+
+        sceneBeans.parallelStream().forEach(scene->{
+            this.cleanOrigHandler(scene);
+        });
     }
 
     @Override
     public void cleanOrigV3() {
-
         //查询所有计算时间超过限定时间的场景,计算成功、未被删除
+        List<SceneBean> sceneBeans = sceneProService.listCleanOrigScene(cleanOrigMonth);
+        if(CollUtil.isEmpty(sceneBeans)){
+            return;
+        }
+
+        sceneBeans.parallelStream().forEach(scene->{
+            this.cleanOrigHandler(scene);
+        });
+    }
 
-        //写入t_scene_clean_orig表
-
-
-        //列举原始资源目录
-
-
-        //如果目录文件不为空，则执行批量删除
-
-        //更新t_scene_clean_orig表
-
-
-
+    private void cleanOrigHandler(SceneBean scene){
+        try {
+            String dataSource = scene.getDataSource();
+            if(StrUtil.isNotEmpty(dataSource)){
+                String homePath = dataSource.replace(ConstantFilePath.BUILD_MODEL_PATH, ConstantFilePath.OSS_PREFIX);
+                fYunFileService.deleteFolder(homePath);
+            }
+            this.saveLog(scene.getNum(), CommonSuccessStatus.SUCCESS.code(), null);
+        }catch (Exception e){
+            log.error("删除原始资源失败,num : " + scene.getNum(), e);
+            this.saveLog(scene.getNum(), CommonSuccessStatus.Fail.code(), ExceptionUtil.stacktraceToString(e, 3000));
+        }
+    }
 
 
 
 
+    private void saveLog(String num, int status, String reason){
+        //清除旧的日志
+        this.remove(new LambdaQueryWrapper<SceneCleanOrig>().eq(SceneCleanOrig::getNum, num));
+        SceneCleanOrig sceneCleanOrig = new SceneCleanOrig();
+        sceneCleanOrig.setNum(num);
+        sceneCleanOrig.setState(status);
+        sceneCleanOrig.setReason(reason);
+        this.saveOrUpdate(sceneCleanOrig);
     }
+
 }

src/main/java/com/fdkankan/scene/service/impl/ScenePlusServiceImpl.java

@@ -64,7 +64,6 @@ public class ScenePlusServiceImpl extends ServiceImpl<IScenePlusMapper, ScenePlu
     private String mainUrl;
     @Value("${scene.pro.new.url}")
     private String sceneProNewUrl;
-    @Value("${}")
 
     @Autowired
     private IScenePlusExtService scenePlusExtService;
@@ -247,9 +246,9 @@ public class ScenePlusServiceImpl extends ServiceImpl<IScenePlusMapper, ScenePlu
     }
 
     @Override
-    public List<SceneBean> listCleanOrigScene() {
+    public List<SceneBean> listCleanOrigScene(int cleanOrigMonth) {
         Date time = Calendar.getInstance().getTime();
-        time = DateUtil.beginOfDay(DateUtil.offset(time, DateField.MONTH, 6));
+        time = DateUtil.beginOfDay(DateUtil.offset(time, DateField.MONTH, -cleanOrigMonth));
         return this.baseMapper.selectCleanOrigScene(time);
     }
 }

src/main/java/com/fdkankan/scene/service/impl/SceneProServiceImpl.java

@@ -1,6 +1,8 @@
 package com.fdkankan.scene.service.impl;
 
 import cn.hutool.core.collection.CollUtil;
+import cn.hutool.core.date.DateField;
+import cn.hutool.core.date.DateUtil;
 import cn.hutool.core.io.FileUtil;
 import cn.hutool.core.util.RuntimeUtil;
 import cn.hutool.core.util.StrUtil;
@@ -16,6 +18,7 @@ import com.fdkankan.model.constants.ConstantFileName;
 import com.fdkankan.model.constants.ConstantFilePath;
 import com.fdkankan.model.constants.UploadFilePath;
 import com.fdkankan.common.exception.BusinessException;
+import com.fdkankan.scene.bean.SceneBean;
 import com.fdkankan.web.response.ResultData;
 import com.fdkankan.model.utils.ComputerUtil;
 import com.fdkankan.model.utils.ConvertUtils;
@@ -1122,4 +1125,11 @@ public class SceneProServiceImpl extends ServiceImpl<ISceneProMapper, ScenePro>
         FileUtil.del(zipPath);
         return ResultData.ok(url);
     }
+
+    @Override
+    public List<SceneBean> listCleanOrigScene(int cleanOrigMonth) {
+        Date time = Calendar.getInstance().getTime();
+        time = DateUtil.beginOfDay(DateUtil.offset(time, DateField.MONTH, -cleanOrigMonth));
+        return this.baseMapper.selectCleanOrigScene(time);
+    }
 }

src/main/resources/bootstrap-test.yml

@@ -29,7 +29,8 @@ spring:
             group: DEFAULT_GROUP
             refresh: true
       discovery:
-        namespace: ${spring.cloud.nacos.namespace}
+#        namespace: ${spring.cloud.nacos.namespace}
+        namespace: public
 
 
 

src/main/resources/mapper/scene/ScenePlusMapper.xml

@@ -6,9 +6,9 @@
         select plus.num, ext.data_source
         from t_scene_plus plus
         left join t_scene_plus_ext ext on plus.id = ext.plus_id
-        where plus.rec_status = 'A' and plus.scene_status in (1,-2) and ext.algorithm_time <![CDATA[ < ]]> #{time}
+        where plus.scene_status in (1,-2) and ext.algorithm_time <![CDATA[ < ]]> #{time}
         and ext.data_source is NOT null
-        and NOT EXISTS (select o.num from t_scene_clean_orig o where o.rec_status = 'A' and o.num = plus.num and ext.algorithm_time <![CDATA[ < ]]> o.create_time);
+        and NOT EXISTS (select o.num from t_scene_clean_orig o where o.rec_status = 'A' and o.state = 1 and o.num = plus.num and ext.algorithm_time <![CDATA[ < ]]> o.create_time)
     </select>
 
 </mapper>

src/main/resources/mapper/scene/SceneProMapper.xml

@@ -1,13 +1,13 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN" "http://mybatis.org/dtd/mybatis-3-mapper.dtd">
-<mapper namespace="com.fdkankan.com.fdkankan.scene.mapper.ISceneProMapper">
+<mapper namespace="com.fdkankan.scene.mapper.ISceneProMapper">
 
     <select id="selectCleanOrigScene" resultType="com.fdkankan.scene.bean.SceneBean">
         select pro.num, pro.data_source
         from t_scene_pro pro
-        where pro.rec_status = 'A' and pro.status in (1,-2) and pro.create_time <![CDATA[ < ]]> #{time}
+        where pro.status in (1,-2) and pro.create_time <![CDATA[ < ]]> #{time}
         and pro.data_source is not null
-        and NOT EXISTS (select o.num from t_scene_clean_orig o where o.rec_status = 'A' and o.num = pro.num and pro.create_time <![CDATA[ < ]]> o.create_time);
+        and NOT EXISTS (select o.num from t_scene_clean_orig o where o.rec_status = 'A' and o.state = 1 and o.num = pro.num and pro.create_time <![CDATA[ < ]]> o.create_time)
     </select>
 
 </mapper>