download_scene/browser_code/downloadController.js

let OSS = require('ali-oss');
let fs = require('fs');
let path = require('path');
var archiver = require('archiver');
let request = require("request");
let http = require('http');

let compressPercent = 0;
let taskCodeQueue = [];
let taskDownloaderQueue = {};
let pathConfig = {
  ossPrefix: 'http://4dkankan.oss-cn-shenzhen.aliyuncs.com/',  // oss前缀
  // serverPrefix: 'http://192.168.0.61:8888/',  // 服务器前缀
  serverPrefix: 'https://test.4dkankan.com/back/down/server/controller/',  // 服务器前缀
  localDataName: 'localData',
  layer: './',  // 层级
  rootFold: 'tmpData',   // 根目录文件夹
  staticPath: [   // 静态数据路径
    'static'
  ],
  paths: [  // images/data路径
    'images/images',
    'data/data'
  ],
  filters: ['tiles'],
  dynamicPath: [   // 一些使用api加载的数据, 提前写入json中
    'http://pro.4dkankan.com/api/scene/getInfo'
  ],
}

function downloader() {
  this.objArr = [];
  this.completeFolds = 0;   // 记录已经递归完成的目录, completeFolds == foldNum, 表示所有目录递归完成
  this.fileNum = 0;  // 需要下载的文件总数
  this.completeFileNum = 0;  // 已经下载完成的文件总数
  this.threadNum = 10;   // 同时并发的线程总数
  this.curActiveThread = 0;
  this.sta = 0;
  this.downloadProcess = 0;  // 下载进度
  this.zipProcess = 0;   // 压缩进度
  this.sceneCode = '';   // 场景码
  this.sceneInfo = '';   // 场景的其他json数据

  this.downloadResponse = null;
  this.timer = null;   // 定时器
  this.dirPath = path.join(__dirname, pathConfig.rootFold);

  this.foldNum = pathConfig.paths.length + pathConfig.staticPath.length;   // 记录目录的总数, 用于判断是否目录是否递归完成
  this.client = new OSS({   // 配置阿里云oss
    region: 'oss-cn-shenzhen',
    accessKeyId: 'LTAIUrvuHqj8pvry',
    accessKeySecret: 'JLOVl0k8Ke0aaM8nLMMiUAZ3EiiqI4',
    bucket: '4dkankan',
  })
}

downloader.prototype.createRootFold = function (data) {
  //console.log('pathpath' + this.dirPath);
  if (!fs.existsSync(this.dirPath)) {
    fs.mkdirSync(this.dirPath);
  } else {
    console.log('文件夹已存在');
  }
  this.dirPath = path.join(this.dirPath, '/' + data);
  // 创建文件夹目录

  if (!fs.existsSync(this.dirPath)) {
    fs.mkdirSync(this.dirPath);
  } else {
    console.log('文件夹已存在');
  }
  // this.dirPath = path.join(this.dirPath, '/' + 'static');  
  // // 创建文件夹目录
  // if (!fs.existsSync(this.dirPath)) {
  //     fs.mkdirSync(this.dirPath);
  // } else {
  //     console.log('文件夹已存在');
  // }
}

downloader.prototype.listDir = async function (dir, prefixDir) {
  let that = this;
  for (let i = 0; i < pathConfig.filters.length; i++) {
    let filter = pathConfig.filters[i];
    if (dir.indexOf(filter) > 0) {
      console.log('过滤的路径' + dir);
      that.completeFolds++;
      if (that.completeFolds === that.foldNum) {
        that.fileNum = that.objArr.length;
        that.timer = setInterval(that.download, 16);
      }
      return;
    }
  }

  let result = await this.client.list({
    prefix: dir,
    delimiter: '/',
    'max-keys': 1000 // 最大限制1000
  });
  // let dirArr = [];
  // if(staticSta){
  //     dirArr = `static/${dir}`.split('/');
  // }else{

  // }
  dirArr = `${prefixDir}${dir}`.split('/');
  let tmpDir = '';
  let childDirPath;
  dirArr.forEach((item) => {
    tmpDir += '/' + item;
    childDirPath = path.join(this.dirPath, '/' + tmpDir);
    if (!fs.existsSync(childDirPath)) {
      fs.mkdirSync(childDirPath);
      console.log('创建文件夹成功:  ' + childDirPath);
    } else {
      console.log('文件夹已存在');
    }
  });
  result.objects && result.objects.forEach(async function (obj) {
    let downloadURL = pathConfig.ossPrefix + obj.name;
    let arr = obj.name.split('/');
    let objName = arr[arr.length - 1];
    if (objName === '') {
      return;   // 过滤非法文件, 因为阿里云有时候会把文件夹当作文件返回, 
    }
    // if(obj.name.indexOf('images') >-1){
    //     console.log(obj);
    // }
    // let writeURL = childDirPath + '/' + objName;
    let writeURL = path.join(childDirPath, objName);
    let tack = {
      downloadURL: downloadURL,
      writeURL: writeURL
    }
    that.objArr.push(tack);
  });

  if (result.prefixes) {   // 如果当前目录还有子目录存在, 则继续递归
    this.foldNum--;    // 移除当前目录
    let that = this;
    result.prefixes.forEach(function (subDir) {
      that.foldNum++;  // 记录当前目录的子目录
      that.listDir(subDir, prefixDir);
    });
  } else {   // 当前目录已经递归完成
    this.completeFolds++;
    console.log(this.completeFolds + '  ' + this.foldNum)
    if (this.completeFolds === this.foldNum) {
      // console.log(this.objArr)
      this.fileNum = this.objArr.length;
      this.timer = setInterval(this.download.bind(this), 16);
    }
  }
}

downloader.prototype.download = function () {

  if (this.objArr.length === 0) {
    clearInterval(this.timer);
    this.timer = null;
    return;
  }
  if (this.curActiveThread <= this.threadNum) {
    let task = this.objArr.shift();
    let stream = fs.createWriteStream(task.writeURL);
    let readStream = request(task.downloadURL).on('error', function () {
      console.log("文件[" + task.downloadURL + "]下载出错  ");
    }).pipe(stream);

    let that = this;
    readStream.on('finish', function () {
      that.curActiveThread--;  // 释放线程
    })
    stream.on('finish', function () {
      that.completeFileNum++;
      that.downloadProcess = that.completeFileNum / that.fileNum * 100;
      if (that.completeFileNum === that.fileNum) {   // oss文件已下载完
        let sceneCode = that.sceneCode;
        let sceneJsonPath = path.join(__dirname, `tmpData/${sceneCode}/static/images/images${sceneCode}/sceneData.json`);
        let sceneCodePath = path.join(__dirname, `tmpData/${sceneCode}/code.txt`);
        fs.writeFile(sceneJsonPath, that.sceneInfo, function (err) {  // 写入sceneInfo.json
          if (err) {
            return console.log(`写入文件出错: ${err}`);
          }
          fs.writeFile(sceneCodePath , sceneCode , function (err) {   // 将场景码写入
            console.log('开始压缩' + that.sceneCode)
            // console.log(`scene: ${that.sceneCode}`);
            // let zipStream = fs.createWriteStream(`${__dirname}/tmpData/${that.sceneCode}/` + `/localData.zip`);
            let zipStream = fs.createWriteStream(`${__dirname}/tmpData/zip/` + `/${that.sceneCode}.zip`);
            pathConfig.localDataName = that.sceneCode;

            let archive = archiver('zip', {
              zlib: {
                level: 9
              }
            });
            zipStream.on('close', function () {
              let respData = 'archiver has been finalized and the output file descriptor has closed.';
              console.log(archive.pointer() + ' total bytes');
              console.log('文件压缩已写入完成');
              // 重置两个进度
              downloadPercent = 0;
              compressPercent = 0;
            });
            archive.on('progress', function (process) {
              compressPercent = that.zipProcess = process.fs.processedBytes / process.fs.totalBytes * 100;
            })
            archive.on('error', function (err) {
              console.log(err);
            })
            archive.pipe(zipStream);
            archive.directory(`${__dirname}/tmpData/${that.sceneCode}/`, false);  // 放入static目录中
            archive.directory(`${__dirname}/static/page`, false);  // 放入static目录中
            archive.directory(`${__dirname}/browser/`, false);  // 将浏览器放入压缩包中
            // archive.directory(pathConfig.staticPath, 'static');   // 追加固定的文件资源
            // archive.directory(`${__dirname}/static/`, 'static');
            archive.finalize();
          })
        })
      }
    })
    this.curActiveThread++;
  }
}

downloader.prototype.getJson = function (url, callback) {
  console.log(url);
  http.get(url, res => {
    let body = '';
    res.on('data', chunk => {
      body += chunk;
    });
    res.on('end', () => {
      // let resp = JSON.parse(body);
      console.log(body);
      // callback(resp);
    })
  })
}

downloader.prototype.loadDynamicInfo = function (sceneCode) {
  for (let i = 0; i < pathConfig.dynamicPath.length; i++) {
    let time = new Date().getTime();
    let url = `${pathConfig.dynamicPath[i]}?num=${sceneCode}&t=${time}`;
    this.getJson(url);
  }
}

downloader.prototype.execute = function (data) {
  console.log(data);
  this.createRootFold(data);
  pathConfig.paths.forEach(path => {
    this.listDir(path + data, `static/`);
  })
  pathConfig.staticPath.forEach(path => {
    this.listDir(path, '');
  })
  // this.sceneInfo = data.sceneInfo
  // this.loadDynamicInfo(this.sceneInfo);
}

function start(response, data) {
  let sceneCode = data.sceneCode;
  let respData;
  console.log(`${__dirname}/${pathConfig.rootFold}/${sceneCode}/${pathConfig.localDataName}.zip`)
  if (fs.existsSync(`${__dirname}/${pathConfig.rootFold}/${sceneCode}/${pathConfig.localDataName}.zip`)) {
    //console.log(`${__dirname}/zip/${sceneCode}/${pathConfig.localDataName}.zip`)
    //if (fs.existsSync(`${__dirname}/zip/${sceneCode}/${pathConfig.localDataName}.zip`)) {
    respData = {
      sta: 1003,   // 1003 文件已存在
      data: {
        percent: 100,
        url: `${pathConfig.serverPrefix}${pathConfig.rootFold}/${sceneCode}/${pathConfig.localDataName}.zip`
      },
      msg: '文件已存在, 直接返回url'
    };
    console.log('文件已存在, 直接返回url');
  } else {
    respData = {
      sta: 1002,   // 1002 
      data: {
        percent: 0
      },
      msg: '已加入任务队列'
    };
    taskCodeQueue.push(sceneCode);
    taskDownloaderQueue[sceneCode] = new downloader();
    taskDownloaderQueue[sceneCode].sceneCode = data.sceneCode;
    taskDownloaderQueue[sceneCode].sceneInfo = data.sceneInfo;
  }
  let json = JSON.stringify(respData)
  // response.writeHead(200, { "Content-Type": "application/json" });
  response.send(json);
  // response.end();
}

function downloadProcess(response, data) {
  let sceneCode = data.sceneCode;
  let downloader = taskDownloaderQueue[sceneCode];
  if (!downloader) return;
  if (downloader.downloadProcess < 100) {
    let respData = {
      sta: 1000,   // 状态码 1000-文件正在下载 1001-文件正在压缩
      data: {
        percent: downloader.downloadProcess
      },
      msg: '文件下载中'
    };
    let json = JSON.stringify(respData)
    // response.writeHead(200, { "Content-Type": "application/json" });
    response.send(json);
    // response.end();
  } else {
    let respData = {
      sta: 1001,   // 状态码 1000-文件正在下载 1001-文件正在压缩
      data: {
        percent: downloader.zipProcess
      },
      msg: '文件压缩中'
    };
    if (downloader.zipProcess === 100) {
      // respData.data.url = `${pathConfig.serverPrefix}${pathConfig.rootFold}/${downloader.sceneCode}/${pathConfig.localDataName}.zip`
      respData.data.url = `${pathConfig.serverPrefix}${pathConfig.rootFold}/zip/${pathConfig.localDataName}.zip`
    }
    let json = JSON.stringify(respData)
    // response.writeHead(200, { "Content-Type": "application/json" });
    response.send(json);
    // response.end();
  }
}

(function () {
  setInterval(function () {
    let taskCode = taskCodeQueue.shift();
    if (taskCode) {
      taskDownloaderQueue[taskCode].execute(taskCode);
    }
  }, 1000)
})()

exports.start = start;
exports.downloadProcess = downloadProcess;