|
@@ -8,21 +8,22 @@ import com.alibaba.fastjson.JSONObject;
|
|
|
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
|
|
|
import com.fdkankan.common.constant.CommonStatus;
|
|
|
import com.fdkankan.common.constant.CommonSuccessStatus;
|
|
|
+import com.fdkankan.common.constant.ErrorCode;
|
|
|
import com.fdkankan.common.constant.OperationType;
|
|
|
+import com.fdkankan.common.exception.BusinessException;
|
|
|
import com.fdkankan.fyun.face.FYunFileServiceInterface;
|
|
|
import com.fdkankan.model.constants.ConstantFilePath;
|
|
|
+import com.fdkankan.model.constants.UploadFilePath;
|
|
|
import com.fdkankan.model.utils.SceneUtil;
|
|
|
import com.fdkankan.redis.constant.RedisKey;
|
|
|
import com.fdkankan.redis.util.RedisUtil;
|
|
|
import com.fdkankan.scene.bean.SceneBean;
|
|
|
+import com.fdkankan.scene.entity.Camera;
|
|
|
import com.fdkankan.scene.entity.SceneCleanOrig;
|
|
|
import com.fdkankan.scene.mapper.ISceneCleanOrigMapper;
|
|
|
import com.fdkankan.scene.mapper.IScenePlusExtMapper;
|
|
|
-import com.fdkankan.scene.service.ISceneCleanOrigService;
|
|
|
+import com.fdkankan.scene.service.*;
|
|
|
import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
|
|
|
-import com.fdkankan.scene.service.IScenePlusExtService;
|
|
|
-import com.fdkankan.scene.service.IScenePlusService;
|
|
|
-import com.fdkankan.scene.service.ISceneProService;
|
|
|
import lombok.extern.slf4j.Slf4j;
|
|
|
import org.springframework.beans.factory.annotation.Autowired;
|
|
|
import org.springframework.beans.factory.annotation.Value;
|
|
@@ -30,10 +31,8 @@ import org.springframework.cloud.context.config.annotation.RefreshScope;
|
|
|
import org.springframework.stereotype.Service;
|
|
|
import sun.font.TextRecord;
|
|
|
|
|
|
-import java.util.Date;
|
|
|
-import java.util.List;
|
|
|
-import java.util.Map;
|
|
|
-import java.util.Objects;
|
|
|
+import java.util.*;
|
|
|
+import java.util.stream.Collectors;
|
|
|
|
|
|
/**
|
|
|
* <p>
|
|
@@ -50,6 +49,15 @@ public class SceneCleanOrigServiceImpl extends ServiceImpl<ISceneCleanOrigMapper
|
|
|
|
|
|
@Value("${scene.cleanOrig.month}")
|
|
|
private Integer cleanOrigMonth;
|
|
|
+ @Value("${scene.cleanDeleted.month}")
|
|
|
+ private Integer cleanDeletedMonth;
|
|
|
+ @Value("#{'${scene.cleanTestCamera.snCode:}'.split(',')}")
|
|
|
+ private List<String> testSnCodeList;
|
|
|
+ @Value("${scene.cleanTestCamera.month}")
|
|
|
+ private Integer cleanTestCameraMonth;
|
|
|
+ @Autowired
|
|
|
+ private ICameraService cameraService;
|
|
|
+
|
|
|
|
|
|
@Autowired
|
|
|
private ISceneProService sceneProService;
|
|
@@ -66,73 +74,68 @@ public class SceneCleanOrigServiceImpl extends ServiceImpl<ISceneCleanOrigMapper
|
|
|
public void cleanOrigV4() {
|
|
|
//查询所有计算时间超过限定时间的场景,计算成功、未被删除、最后一次计算后未被删除过的
|
|
|
List<SceneBean> sceneBeans = scenePlusService.listCleanOrigScene(cleanOrigMonth);
|
|
|
- if(CollUtil.isEmpty(sceneBeans)){
|
|
|
- return;
|
|
|
- }
|
|
|
-
|
|
|
- sceneBeans.parallelStream().forEach(scene->{
|
|
|
- this.cleanOrigHandler(scene);
|
|
|
- });
|
|
|
+ this.cleanOrig(sceneBeans);
|
|
|
}
|
|
|
|
|
|
@Override
|
|
|
public void cleanOrigV3() {
|
|
|
//查询所有计算时间超过限定时间的场景,计算成功、未被删除
|
|
|
List<SceneBean> sceneBeans = sceneProService.listCleanOrigScene(cleanOrigMonth);
|
|
|
+ this.cleanOrig(sceneBeans);
|
|
|
+
|
|
|
+ }
|
|
|
+
|
|
|
+ private void cleanOrig(List<SceneBean> sceneBeans){
|
|
|
if(CollUtil.isEmpty(sceneBeans)){
|
|
|
return;
|
|
|
}
|
|
|
-
|
|
|
sceneBeans.parallelStream().forEach(scene->{
|
|
|
- this.cleanOrigHandler(scene);
|
|
|
+ boolean lock = this.lock(scene.getDataSource());
|
|
|
+ try {
|
|
|
+ if(lock) {
|
|
|
+ this.cleanOrigHandler(scene);
|
|
|
+ this.saveLog(scene.getNum(), 1, CommonSuccessStatus.SUCCESS.code(), null);
|
|
|
+ }
|
|
|
+ }catch (Exception e){
|
|
|
+ log.error("删除原始资源失败,num : " + scene.getNum(), e);
|
|
|
+ this.saveLog(scene.getNum(), 1, CommonSuccessStatus.FAIL.code(), ExceptionUtil.stacktraceToString(e, 3000));
|
|
|
+ }finally {
|
|
|
+ this.releaseLock(scene.getDataSource());
|
|
|
+ }
|
|
|
});
|
|
|
}
|
|
|
|
|
|
private void cleanOrigHandler(SceneBean scene){
|
|
|
- try {
|
|
|
- boolean lock = this.lock(scene.getDataSource());
|
|
|
- if(!lock){
|
|
|
- throw new RuntimeException("场景正在上传");
|
|
|
- }
|
|
|
- int successStatus = CommonSuccessStatus.SUCCESS.code();
|
|
|
- String reason = null;
|
|
|
- String dataSource = scene.getDataSource();
|
|
|
- if(StrUtil.isNotEmpty(dataSource)){
|
|
|
- String homePath = dataSource.replace(ConstantFilePath.BUILD_MODEL_PATH, ConstantFilePath.OSS_PREFIX);
|
|
|
- //由于国内测试和生产用的bucket是同一个,这里需要做一个安全校验,保证不会删错
|
|
|
- String fileContent = fYunFileService.getFileContent(homePath.concat("/").concat("data.fdage"));
|
|
|
- if(StrUtil.isNotBlank(fileContent)){
|
|
|
- JSONObject jsonObject = JSON.parseObject(fileContent);
|
|
|
- String snCode = jsonObject.getJSONObject("cam").getString("uuid");
|
|
|
- String uuidTime = jsonObject.getString("uuidtime");
|
|
|
- if(StrUtil.isEmpty(snCode)
|
|
|
- || StrUtil.isEmpty(uuidTime)
|
|
|
- || !homePath.contains(snCode)
|
|
|
- || !homePath.contains(uuidTime)){
|
|
|
- successStatus = CommonSuccessStatus.FAIL.code();
|
|
|
- reason = "dataSource与data.fdage文件不匹配";
|
|
|
- }else{
|
|
|
- fYunFileService.deleteFolder(homePath);
|
|
|
- }
|
|
|
+ String dataSource = scene.getDataSource();
|
|
|
+ if(StrUtil.isNotEmpty(dataSource)){
|
|
|
+ String homePath = dataSource.replace(ConstantFilePath.BUILD_MODEL_PATH, ConstantFilePath.OSS_PREFIX);
|
|
|
+ //由于国内测试和生产用的bucket是同一个,这里需要做一个安全校验,保证不会删错
|
|
|
+ String fileContent = fYunFileService.getFileContent(homePath.concat("/").concat("data.fdage"));
|
|
|
+ if(StrUtil.isNotBlank(fileContent)){
|
|
|
+ JSONObject jsonObject = JSON.parseObject(fileContent);
|
|
|
+ String snCode = jsonObject.getJSONObject("cam").getString("uuid");
|
|
|
+ String uuidTime = jsonObject.getString("uuidtime");
|
|
|
+ if(StrUtil.isEmpty(snCode)
|
|
|
+ || StrUtil.isEmpty(uuidTime)
|
|
|
+ || !homePath.contains(snCode)
|
|
|
+ || !homePath.contains(uuidTime)){
|
|
|
+ throw new RuntimeException("dataSource与data.fdage文件不匹配");
|
|
|
+ }else{
|
|
|
+ fYunFileService.deleteFolder(homePath);
|
|
|
}
|
|
|
}
|
|
|
- this.saveLog(scene.getNum(), successStatus, reason);
|
|
|
- }catch (Exception e){
|
|
|
- log.error("删除原始资源失败,num : " + scene.getNum(), e);
|
|
|
- this.saveLog(scene.getNum(), CommonSuccessStatus.FAIL.code(), ExceptionUtil.stacktraceToString(e, 3000));
|
|
|
- }finally {
|
|
|
- this.releaseLock(scene.getDataSource());
|
|
|
}
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
- private void saveLog(String num, int status, String reason){
|
|
|
+ private void saveLog(String num, int type, int status, String reason){
|
|
|
//清除旧的日志
|
|
|
this.remove(new LambdaQueryWrapper<SceneCleanOrig>().eq(SceneCleanOrig::getNum, num));
|
|
|
SceneCleanOrig sceneCleanOrig = new SceneCleanOrig();
|
|
|
sceneCleanOrig.setNum(num);
|
|
|
+ sceneCleanOrig.setType(type);
|
|
|
sceneCleanOrig.setState(status);
|
|
|
sceneCleanOrig.setReason(reason);
|
|
|
this.saveOrUpdate(sceneCleanOrig);
|
|
@@ -158,4 +161,74 @@ public class SceneCleanOrigServiceImpl extends ServiceImpl<ISceneCleanOrigMapper
|
|
|
redisUtil.del(String.format(RedisKey.SCENE_OSS_HOME_DIR_DELETE, uuid));
|
|
|
}
|
|
|
|
|
|
+
|
|
|
+ @Override
|
|
|
+ public void cleanOss4DeletedSceneV3() {
|
|
|
+ List<SceneBean> sceneBeans = sceneProService.listCleanOss4DeletedScene(cleanDeletedMonth);
|
|
|
+ this.cleanOrig4Delete(sceneBeans, false, 2);
|
|
|
+ }
|
|
|
+
|
|
|
+ @Override
|
|
|
+ public void cleanOss4DeletedSceneV4() {
|
|
|
+ //查询所有计算时间超过限定时间的场景,计算成功、未被删除、最后一次计算后未被删除过的
|
|
|
+ List<SceneBean> sceneBeans = scenePlusService.listCleanOss4DeletedScene(cleanDeletedMonth);
|
|
|
+ this.cleanOrig4Delete(sceneBeans, true, 2);
|
|
|
+ }
|
|
|
+
|
|
|
+ /**
|
|
|
+ * 删除已删除场景的原始资源及caches目录(v3场景不需要删除caches目录)
|
|
|
+ * @param sceneBeans
|
|
|
+ * @param deleteCaches 是否需要删除caches目录
|
|
|
+ */
|
|
|
+ private void cleanOrig4Delete(List<SceneBean> sceneBeans, boolean deleteCaches, Integer type){
|
|
|
+ if(CollUtil.isEmpty(sceneBeans)){
|
|
|
+ return;
|
|
|
+ }
|
|
|
+ sceneBeans.parallelStream().forEach(scene->{
|
|
|
+ try {
|
|
|
+ //删除caches文件
|
|
|
+ if(deleteCaches){
|
|
|
+ this.deleteResultCaches(scene.getNum());
|
|
|
+ }
|
|
|
+ //删除原始资源
|
|
|
+ this.cleanOrigHandler(scene);
|
|
|
+
|
|
|
+ this.saveLog(scene.getNum(), type, CommonSuccessStatus.SUCCESS.code(), null);
|
|
|
+ }catch (Exception e){
|
|
|
+ log.error("删除已删除场景资源失败,num : " + scene.getNum(), e);
|
|
|
+ this.saveLog(scene.getNum(), type, CommonSuccessStatus.FAIL.code(), ExceptionUtil.stacktraceToString(e, 3000));
|
|
|
+ }
|
|
|
+ });
|
|
|
+ }
|
|
|
+
|
|
|
+ private void deleteResultCaches(String num){
|
|
|
+ String cachesPath = String.format(UploadFilePath.scene_result_data_path, num).concat("caches");
|
|
|
+ if(CollUtil.isEmpty(fYunFileService.listRemoteFiles(cachesPath))){
|
|
|
+ return;
|
|
|
+ }
|
|
|
+ fYunFileService.deleteFolder(cachesPath);
|
|
|
+ }
|
|
|
+
|
|
|
+ @Override
|
|
|
+ public void cleanOss4TestCameraV3() {
|
|
|
+ List<Camera> cameras = cameraService.listBySnCodes(testSnCodeList);
|
|
|
+ if(CollUtil.isEmpty(cameras)){
|
|
|
+ return;
|
|
|
+ }
|
|
|
+ Set<Long> cameraIds = cameras.stream().map(Camera::getId).collect(Collectors.toSet());
|
|
|
+ List<SceneBean> sceneBeans = sceneProService.listCleanOss4TestCamera(cameraIds, cleanTestCameraMonth);
|
|
|
+ this.cleanOrig4Delete(sceneBeans, false, 3);
|
|
|
+ }
|
|
|
+
|
|
|
+ @Override
|
|
|
+ public void cleanOss4TestCameraV4() {
|
|
|
+ List<Camera> cameras = cameraService.listBySnCodes(testSnCodeList);
|
|
|
+ if(CollUtil.isEmpty(cameras)){
|
|
|
+ return;
|
|
|
+ }
|
|
|
+ Set<Long> cameraIds = cameras.stream().map(Camera::getId).collect(Collectors.toSet());
|
|
|
+ List<SceneBean> sceneBeans = scenePlusService.listCleanOss4TestCamera(cameraIds, cleanTestCameraMonth);
|
|
|
+ this.cleanOrig4Delete(sceneBeans, true, 3);
|
|
|
+ }
|
|
|
+
|
|
|
}
|