修改资源清除逻辑

src/main/java/com/fdkankan/scene/mapper/ISceneCleanMapper.java

@@ -7,6 +7,7 @@ import java.util.Date;
 import java.util.List;
 import org.apache.ibatis.annotations.Mapper;
 import org.apache.ibatis.annotations.Param;
+import org.apache.ibatis.annotations.Select;
 
 /**
  * <p>
@@ -20,4 +21,6 @@ import org.apache.ibatis.annotations.Param;
 public interface ISceneCleanMapper extends BaseMapper<SceneClean> {
 
     List<SceneBean> selectNeadCleanScene(@Param("index") long index, @Param("size") int size, @Param("time") Date time);
+
+    List<SceneClean> findAllByDateSource(@Param("dataSource") String dataSource);
 }

src/main/java/com/fdkankan/scene/schedule/ScheduleJob.java

@@ -19,7 +19,7 @@ public class ScheduleJob {
      * 执行时间：每天凌晨
      * 删除条件：场景计算时间大于6个月且未删除过资源
      */
-//    @Scheduled(cron = "0 0 0 * * ?")
+    @Scheduled(cron = "0 0 0/2 * * ?")
     public void sceneCleanResource() {
         log.info("每天凌晨12:00开始执行定时任务：清除场景计算目录");
         try {

src/main/java/com/fdkankan/scene/service/impl/SceneCleanServiceImpl.java

@@ -1,33 +1,20 @@
 package com.fdkankan.scene.service.impl;
 
-import cn.hutool.core.collection.CollUtil;
-import cn.hutool.core.date.DateTime;
-import cn.hutool.core.date.DateUtil;
-import cn.hutool.core.io.FileUtil;
-import cn.hutool.core.lang.UUID;
-import cn.hutool.core.util.RandomUtil;
 import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
 import com.baomidou.mybatisplus.core.conditions.update.LambdaUpdateWrapper;
+import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
 import com.fdkankan.common.constant.CommonStatus;
-import com.fdkankan.common.util.FileUtils;
+import com.fdkankan.common.util.DateUtil;
+import com.fdkankan.model.constants.ConstantFilePath;
 import com.fdkankan.model.utils.CreateObjUtil;
 import com.fdkankan.redis.constant.RedisKey;
 import com.fdkankan.redis.constant.RedisLockKey;
 import com.fdkankan.redis.util.RedisLockUtil;
-import com.fdkankan.redis.util.RedisUtil;
 import com.fdkankan.scene.bean.SceneBean;
 import com.fdkankan.scene.entity.SceneClean;
 import com.fdkankan.scene.mapper.ISceneCleanMapper;
 import com.fdkankan.scene.service.ISceneCleanService;
-import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
-import com.google.common.collect.Lists;
-import com.google.common.collect.Sets;
-import java.util.Calendar;
-import java.util.Date;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Set;
-import java.util.stream.Collectors;
+import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.lang3.StringUtils;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.beans.factory.annotation.Value;
@@ -36,6 +23,9 @@ import org.springframework.stereotype.Service;
 import org.springframework.transaction.annotation.Transactional;
 import org.springframework.util.CollectionUtils;
 
+import java.util.*;
+import java.util.stream.Collectors;
+
 /**
  * <p>
  *  服务实现类
@@ -46,91 +36,79 @@ import org.springframework.util.CollectionUtils;
  */
 @RefreshScope
 @Service
+@Slf4j
 public class SceneCleanServiceImpl extends ServiceImpl<ISceneCleanMapper, SceneClean> implements ISceneCleanService {
 
     @Autowired
     private RedisLockUtil redisLockUtil;
-    @Autowired
-    private RedisUtil redisUtil;
+
     @Value("${scene.clean.size:10}")
     private Integer sceneCleanSize;
 
     @Transactional
     @Override
     public void sceneCleanResource(){
+        // 获取redis 锁
+        boolean lock = redisLockUtil.lock(RedisLockKey.LOCK_SCENE_CLEAN, RedisKey.EXPIRE_TIME_2_HOUR);
+        if(!lock){
+            log.warn("未获取到清除资源锁，退出任务！");
+            return;
+        }
 
-        //这里考虑到日后会集群部署，所以查询需要清理资源的场景的写法是负载均衡写法，为了避免幻读影响到查询结果，所以写库放在循环外层
-        List<SceneClean> allInsertList = Lists.newArrayList();
-        Set<String> allUpdateSet = Sets.newHashSet();
-
-        DateTime dateTime = DateUtil.beginOfDay(DateUtil.offsetMonth(Calendar.getInstance().getTime(), -7));
+        Date time = DateUtil.delay(Calendar.getInstance().getTime(), -7, Calendar.MONTH);
+        time.setHours(0);
+        time.setMinutes(0);
+        time.setSeconds(0);
 
-        while (true){
-            String uuid = UUID.randomUUID().toString();
-            boolean lock = redisLockUtil.lock(RedisLockKey.LOCK_SCENE_CLEAN, uuid, RedisKey.EXPIRE_TIME_1_MINUTE);
-            if(!lock){
-                try {
-                    Thread.sleep(RandomUtil.randomLong(1000L, 2000L));
-                    continue;
-                } catch (InterruptedException e) {
-                    e.printStackTrace();
-                }
-            }
-            long index = 0;
-            String indexStr =  redisUtil.get(RedisKey.SCENE_CLEAN_INDEX);
-            if(StringUtils.isNotBlank(indexStr)){
-                index = Long.valueOf(indexStr);
-            }
-            //下标增加1000
-            redisUtil.incr(RedisKey.SCENE_CLEAN_INDEX, sceneCleanSize);
-            //解锁
-            redisLockUtil.unlockLua(RedisLockKey.LOCK_SCENE_CLEAN, uuid);
-            //查询需要清理资源的场景
-            List<SceneBean> sceneBeanList = this.selectNeadCleanScene(index, sceneCleanSize, dateTime);
-            //如果查出来的场景集合是空，证明已经没有场景资源需要删除，则需要把查询下表删除，等待下一次定时任务执行
-            if(CollectionUtils.isEmpty(sceneBeanList)){
-                redisUtil.del(RedisKey.SCENE_CLEAN_INDEX);
-                break;
-            }
-            List<SceneClean> insertList = Lists.newArrayList();
-            List<String> numList = sceneBeanList.stream().map(scene -> scene.getNum()).collect(Collectors.toList());
-            List<SceneClean> updateList = this.list(new LambdaQueryWrapper<SceneClean>().in(SceneClean::getNum, numList));
-            Set<String> updateSet = new HashSet<>();
-            if(!CollectionUtils.isEmpty(updateList)){
-                updateList.stream().forEach(item->{
-                    updateSet.add(item.getNum());
-                });
-            }
-            sceneBeanList.stream().forEach(scene -> {
-                if(StringUtils.isNotBlank(scene.getDataSource())){
+        //查询需要清理资源的场景
+        List<SceneBean> sceneProEntityList = selectNeadCleanScene(0,sceneCleanSize, time);
+        //如果查出来的场景集合是空，证明已经没有场景资源需要删除，则需要把查询下表删除，等待下一次定时任务执行
+        if (CollectionUtils.isEmpty(sceneProEntityList)) {
+            return;
+        }
+        List<String> numList = sceneProEntityList.parallelStream().map(SceneBean::getNum).collect(Collectors.toList());
+        List<SceneClean> insertList = new ArrayList<>();
+        List<SceneClean> updateList = this.list(new LambdaQueryWrapper<SceneClean>().in(SceneClean::getNum, numList));
+        Set<String> updateSet = updateList.stream().map(SceneClean::getNum).collect(Collectors.toSet());
+        sceneProEntityList.stream().forEach(scene -> {
+            if (StringUtils.isNotBlank(scene.getDataSource())) {
+                // 查询该资源的最新计算时间
+                List<SceneClean> scenes = findAllByDateSource(scene.getDataSource());
+                if (scenes.get(0).getCreateTime().before(time)) {
                     try {
-                        CreateObjUtil.deleteFile(scene.getDataSource());
+                        CreateObjUtil.deleteFile(scene.getDataSource().replace(ConstantFilePath.BUILD_MODEL_PATH, "/").replace(ConstantFilePath.BUILD_MODEL_LASER_PATH, "/"));
                     } catch (Exception e) {
                         e.printStackTrace();
+                        updateSet.remove(scene.getNum());
+                        return;
                     }
                 }
-                if(!updateSet.contains(scene.getNum())){
-                    SceneClean sceneClean = new SceneClean();
-                    sceneClean.setNum(scene.getNum());
-                    insertList.add(sceneClean);
-                }
-            });
-            if(!CollectionUtils.isEmpty(insertList)){
-                allInsertList.addAll(insertList);
             }
-            if(!CollectionUtils.isEmpty(updateSet)){
-                allUpdateSet.addAll(updateSet);
+            if (!updateSet.contains(scene.getNum())) {
+                SceneClean sceneCleanEntity = new SceneClean();
+                sceneCleanEntity.setState(1);
+                sceneCleanEntity.setNum(scene.getNum());
+                sceneCleanEntity.setCreateTime(new Date());
+                sceneCleanEntity.setUpdateTime(new Date());
+                sceneCleanEntity.setRecStatus("A");
+                insertList.add(sceneCleanEntity);
             }
-        }
+        });
 
         //写库
-        if(CollUtil.isNotEmpty(allInsertList)){
-            this.saveBatch(allInsertList);
+        if(!CollectionUtils.isEmpty(insertList)){
+            this.saveBatch(insertList);
         }
-        if(CollUtil.isNotEmpty(allUpdateSet)){
-                this.update(new LambdaUpdateWrapper<SceneClean>().set(SceneClean::getState,
-                    CommonStatus.YES.code()).in(SceneClean::getNum, allUpdateSet));
+        if(!CollectionUtils.isEmpty(updateSet)){
+            this.update(new LambdaUpdateWrapper<SceneClean>().set(SceneClean::getState,
+                    CommonStatus.YES.code()).in(SceneClean::getNum, updateSet));
         }
+
+        redisLockUtil.unlock(RedisLockKey.LOCK_SCENE_CLEAN);
+    }
+
+    private List<SceneClean> findAllByDateSource(String dataSource) {
+        return this.baseMapper.findAllByDateSource(dataSource);
     }
 
     @Override

src/main/resources/mapper/scene/SceneCleanMapper.xml

@@ -8,7 +8,7 @@
       FROM t_scene_pro p
       WHERE p.status != 0 and p.is_upgrade != 1 and p.create_time <![CDATA[ <= ]]> #{time}
       and p.num NOT IN (
-          SELECT c.num FROM t_scene_clean c WHERE p.num = c.num AND c.rec_status = 'A' AND c.state = 1
+          SELECT c.num FROM t_scene_clean c WHERE c.state = 1
       )
 
       union
@@ -18,10 +18,24 @@
       left join t_scene_plus_ext e on p.id = e.plus_id
       WHERE p.scene_status != 0 and e.algorithm_time <![CDATA[ <= ]]> #{time}
       and p.num NOT IN (
-          SELECT c.num FROM t_scene_clean c WHERE p.num = c.num AND c.rec_status = 'A' AND c.state = 1
+          SELECT c.num FROM t_scene_clean c WHERE c.state = 1
       )
     ) t
     LIMIT #{index},#{size}
   </select>
 
+  <select id="findAllByDateSource" resultType="com.fdkankan.scene.entity.SceneClean">
+    select t.create_time from (
+    SELECT p.create_time
+    FROM t_scene_pro p
+    WHERE p.data_source = #{dataSource}
+
+    union
+
+    SELECT p.algorithm_time
+    FROM t_scene_plus_ext p
+    WHERE p.data_source = #{dataSource}
+    ) t order by create_time desc
+  </select>
+
 </mapper>