update

package-lock.json

@@ -12,7 +12,9 @@
         "axios": "^0.26.1",
         "buffer": "^5.2.1",
         "eventemitter3": "^3.1.0",
-        "tinyh264": "^0.0.7"
+        "lodash-es": "^4.17.21",
+        "tinyh264": "^0.0.7",
+        "uuid": "^8.3.2"
       },
       "devDependencies": {
         "@babel/core": "^7.13.14",
@@ -3457,6 +3459,11 @@
       "dev": true,
       "license": "MIT"
     },
+    "node_modules/lodash-es": {
+      "version": "4.17.21",
+      "resolved": "https://registry.npmmirror.com/lodash-es/-/lodash-es-4.17.21.tgz",
+      "integrity": "sha512-mKnC+QJ9pWVzv+C4/U3rRsHapFfHvQFoFB92e52xeyGMcX6/OlIl78je1u8vePzYZSkkogMPJ2yjxxsb89cxyw=="
+    },
     "node_modules/lodash.debounce": {
       "version": "4.0.8",
       "resolved": "https://registry.npmmirror.com/lodash.debounce/-/lodash.debounce-4.0.8.tgz",
@@ -4563,6 +4570,14 @@
       "dev": true,
       "license": "MIT"
     },
+    "node_modules/uuid": {
+      "version": "8.3.2",
+      "resolved": "https://registry.npmmirror.com/uuid/-/uuid-8.3.2.tgz",
+      "integrity": "sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg==",
+      "bin": {
+        "uuid": "dist/bin/uuid"
+      }
+    },
     "node_modules/validate-npm-package-license": {
       "version": "3.0.4",
       "resolved": "https://registry.npmmirror.com/validate-npm-package-license/-/validate-npm-package-license-3.0.4.tgz",
@@ -6910,6 +6925,11 @@
       "integrity": "sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==",
       "dev": true
     },
+    "lodash-es": {
+      "version": "4.17.21",
+      "resolved": "https://registry.npmmirror.com/lodash-es/-/lodash-es-4.17.21.tgz",
+      "integrity": "sha512-mKnC+QJ9pWVzv+C4/U3rRsHapFfHvQFoFB92e52xeyGMcX6/OlIl78je1u8vePzYZSkkogMPJ2yjxxsb89cxyw=="
+    },
     "lodash.debounce": {
       "version": "4.0.8",
       "resolved": "https://registry.npmmirror.com/lodash.debounce/-/lodash.debounce-4.0.8.tgz",
@@ -7699,6 +7719,11 @@
       "integrity": "sha512-jk1+QP6ZJqyOiuEI9AEWQfju/nB2Pw466kbA0LEZljHwKeMgd9WrAEgEGxjPDD2+TNbbb37rTyhEfrCXfuKXnA==",
       "dev": true
     },
+    "uuid": {
+      "version": "8.3.2",
+      "resolved": "https://registry.npmmirror.com/uuid/-/uuid-8.3.2.tgz",
+      "integrity": "sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg=="
+    },
     "validate-npm-package-license": {
       "version": "3.0.4",
       "resolved": "https://registry.npmmirror.com/validate-npm-package-license/-/validate-npm-package-license-3.0.4.tgz",

package.json

@@ -14,13 +14,16 @@
     "build": "node --no-warnings ./scripts/build.js --env=production",
     "serve": "http-server ./dist -p 8088 -c 0 -P http://192.168.0.47 --cors",
     "start": "npm-run-all --parallel serve dev",
-    "ba":"rollup -c rollup.config.ba.js --environment development && npm run serve"
+    "ba-serve": "rollup -c rollup.config.ba.js -w",
+    "ba": "npm-run-all --parallel ba-serve serve"
   },
   "dependencies": {
     "axios": "^0.26.1",
-    "tinyh264": "^0.0.7",
     "buffer": "^5.2.1",
-    "eventemitter3": "^3.1.0"
+    "eventemitter3": "^3.1.0",
+    "lodash-es": "^4.17.21",
+    "tinyh264": "^0.0.7",
+    "uuid": "^8.3.2"
   },
   "devDependencies": {
     "@babel/core": "^7.13.14",

rollup.config.ba.js

@@ -51,7 +51,7 @@ if (isProd) {
 
 export default [
     {
-        input: 'src/video/index.js',
+        input: 'src/h264Decoder/index.js',
         //external: ['three'],
         output: [
             {

src/h264Decoder/VDecoder.js

@@ -0,0 +1,152 @@
+import EventEmitter from "eventemitter3";
+import H264Worker from "web-worker:./h264.worker.js";
+import { range, isArray } from "lodash-es";
+import { v4 as uuidv4 } from "uuid";
+
+export class VDecoder extends EventEmitter {
+  constructor({ chunkSize = 256 * 1024, maxChip = 100 }) {
+    super();
+    // this.cacheSegmentCount = cacheSegmentCount;
+    // this.chunkSize = chunkSize;
+    this.cacheBuffer = [];
+    this.cacheBufferTotal = null;
+    this.worker = new H264Worker();
+    this.initWorker();
+    this.tempVideos = [];
+    this.ready = false;
+    this.decoding = false;
+    this.decodingId = null;
+    this.start = null;
+    this.maxChip = maxChip;
+  }
+
+  static isSupport() {
+    return !!(
+      // UC and Quark browser (iOS/Android) support wasm/asm limited,
+      // its iOS version make wasm/asm performance very slow （maybe hook something）
+      // its Android version removed support for wasm/asm, it just run pure javascript codes,
+      // so it is very easy to cause memory leaks
+      (
+        !/UCBrowser|Quark/.test(window.navigator.userAgent) &&
+        window.fetch &&
+        window.ReadableStream &&
+        window.Promise &&
+        window.URL &&
+        window.URL.createObjectURL &&
+        window.Blob &&
+        window.Worker &&
+        !!new Audio().canPlayType("audio/aac;").replace(/^no$/, "") &&
+        (window.AudioContext || window.webkitAudioContext)
+      )
+    );
+  }
+  initWorker() {
+    this.worker.addEventListener("message", (e) => {
+      const message =
+        /** @type {{type:string, width:number, height:number, data:ArrayBuffer, renderStateId:number}} */ e.data;
+
+      switch (message.type) {
+        case "pictureReady":
+          //   onPictureReady(message);
+          this.emit(
+            "decodeData",
+            Object.assign(message, { clipId: this.decodingId })
+          );
+          if (this.decoding && this.decodingId) {
+            this.decodeNext(this.decodingId);
+          }
+          break;
+        case "decoderReady":
+          this.ready = true;
+          this.emit("ready");
+          break;
+      }
+    });
+  }
+
+  /**
+   *
+   * @param {*} rangeArray array [2,100]
+   */
+  fetch({ path, range: rangeArray, decode = true }) {
+    if (!this.ready) {
+      throw new Error("decoder is not ready");
+    }
+    const url = path;
+    if (!(isArray(rangeArray) && rangeArray.length === 2)) {
+      throw new Error("range must is an array!");
+    }
+
+    if (this.tempVideos.length > this.maxChip) {
+      this.flush();
+      console.log("flush");
+    }
+    const rangeFetch = range(rangeArray[0], rangeArray[1] + 1);
+    // console.log("url", url);
+    // console.log("rangeFetch", rangeFetch);
+    const allFetch = rangeFetch.map((i) => {
+      return fetch(`${url}/${i}`).then((response) => {
+        return response.arrayBuffer().then(function (buffer) {
+          return new Uint8Array(buffer);
+        });
+      });
+    });
+
+    return Promise.all(allFetch).then((data) => {
+      const clip = { id: uuidv4(), data };
+      this.emit("fetchDone", clip);
+      this.cacheBuffer = data.slice();
+      this.tempVideos.push(clip);
+      if (decode) {
+        this.start = Date.now();
+        this.cacheBufferTotal = clip.data.length;
+        this.decodeNext(clip.id);
+      }
+      return Promise.resolve(clip);
+    });
+  }
+  /**
+   * @param {Uint8Array} h264Nal
+   */
+  decode(h264Nal, id) {
+    this.worker.postMessage(
+      {
+        type: "decode",
+        data: h264Nal.buffer,
+        offset: h264Nal.byteOffset,
+        length: h264Nal.byteLength,
+        renderStateId: id,
+      },
+      [h264Nal.buffer]
+    );
+  }
+
+  decodeNext(clipId) {
+    const nextFrame = this.cacheBuffer.shift();
+    this.decodingId = clipId;
+    this.decoding = true;
+    let tempId = this.cacheBufferTotal - this.cacheBuffer.length - 1;
+
+    if (nextFrame) {
+      this.decode(nextFrame, tempId);
+    } else {
+      const clip = this.tempVideos.find(({ id }) => id === this.decodingId);
+      const fps = (1000 / (Date.now() - this.start)) * clip.data.length;
+      console.log(
+        `Decoded ${clip.data.length} frames in ${Date.now() - this.start}ms @ ${
+          fps >> 0
+        }FPS`
+      );
+
+      this.decoding = false;
+      this.decodingId = null;
+      tempId = 0;
+      this.emit("decodeDone", clip.id);
+    }
+  }
+  flush() {
+    this.tempVideos = [];
+  }
+
+  preloader(preload) {}
+}

src/h264Decoder/h264.worker.js

@@ -0,0 +1,3 @@
+import { init } from 'tinyh264'
+
+init()

src/h264Decoder/index.js

@@ -0,0 +1,41 @@
+// vDecoder example
+import { VDecoder } from "./VDecoder.js";
+// 测试用
+import { initWebGLCanvas, draw } from "../video/test.js";
+
+// decoder
+
+const vDecoder = new VDecoder({
+  maxChip: 100,
+});
+
+vDecoder.on("ready", () => {
+  console.log("ready");
+  // 测试canvas
+  initWebGLCanvas();
+
+  vDecoder.fetch({
+    path: "https://laser-data.oss-cn-shenzhen.aliyuncs.com/test-video/earth",
+    range: [8, 100],
+  });
+
+  vDecoder.on("fetchDone", (clip) => {
+    console.log("fetchDone", clip);
+  });
+  //监听 decodeData
+  vDecoder.on("decodeData", (data) => {
+    // console.log("decodeData", data);
+    const { width, height, data: buffer } = data;
+    draw(new Uint8Array(buffer), width, height);
+  });
+
+  vDecoder.on("decodeDone", async (id) => {
+    let clipId = null;
+    // vDecoder.flush();
+    clipId = await vDecoder.fetch({
+      path: "https://laser-data.oss-cn-shenzhen.aliyuncs.com/test-video/14_test",
+      range: [0, 28],
+    });
+    // console.log("clipId", clipId);
+  });
+});

src/h264Decoder/utils/ticker.js

@@ -0,0 +1,78 @@
+const raf = (callback, delay) => {
+  return setTimeout(callback, delay);
+};
+
+raf.cancel = (handler) => {
+  clearTimeout(handler);
+};
+
+class Ticker {
+  constructor() {
+    this.extra = 0;
+    this.timestamp = 0;
+    this.interval = 1000 / 60;
+    this.callbacks = [];
+    this.handler = raf(this._tick.bind(this), this.interval);
+  }
+
+  setFps(fps) {
+    if (fps) {
+      this.interval = 1000 / fps;
+    }
+  }
+
+  add(func) {
+    this.callbacks.push(func);
+  }
+
+  remove(func) {
+    for (let i = this.callbacks.length - 1; i >= 0; i--) {
+      if (this.callbacks[i] == func) {
+        this.callbacks.splice(i, 1);
+      }
+    }
+  }
+
+  destroy() {
+    raf.cancel(this.handler);
+    this.handler = null;
+    this.callbacks = [];
+  }
+
+  _tick() {
+    const now = this._now();
+    if (!this.timestamp) {
+      this.timestamp = now;
+    }
+
+    let extra = this.extra;
+    const diff = now - this.timestamp + extra;
+    let loop = Math.floor(diff / this.interval);
+    extra = diff - this.interval * loop;
+    if (extra >= this.interval / 1.5) {
+      loop = Math.ceil(diff / this.interval);
+      extra = 0;
+    }
+
+    if (loop) {
+      for (let i = 0; i < this.callbacks.length; i++) {
+        for (let j = 0; j < loop; j++) {
+          this.callbacks[i](now - this.timestamp);
+        }
+      }
+      this.extra = extra;
+      this.timestamp = now;
+    }
+
+    this.handler = raf(this._tick.bind(this), this.interval);
+  }
+
+  _now() {
+    if (window.performance && window.performance.now) {
+      return window.performance.now();
+    }
+    return +new Date();
+  }
+}
+
+export default Ticker;

src/h264Decoder/utils/util.js

@@ -0,0 +1,112 @@
+/********************************************************
+Copyright (c) <2019> <copyright ErosZy>
+
+"Anti 996" License Version 1.0 (Draft)
+
+Permission is hereby granted to any individual or legal entity
+obtaining a copy of this licensed work (including the source code,
+documentation and/or related items, hereinafter collectively referred
+to as the "licensed work"), free of charge, to deal with the licensed
+work for any purpose, including without limitation, the rights to use,
+reproduce, modify, prepare derivative works of, distribute, publish
+and sublicense the licensed work, subject to the following conditions:
+
+1. The individual or the legal entity must conspicuously display,
+without modification, this License and the notice on each redistributed
+or derivative copy of the Licensed Work.
+
+2. The individual or the legal entity must strictly comply with all
+applicable laws, regulations, rules and standards of the jurisdiction
+relating to labor and employment where the individual is physically
+located or where the individual was born or naturalized; or where the
+legal entity is registered or is operating (whichever is stricter). In
+case that the jurisdiction has no such laws, regulations, rules and
+standards or its laws, regulations, rules and standards are
+unenforceable, the individual or the legal entity are required to
+comply with Core International Labor Standards.
+
+3. The individual or the legal entity shall not induce, suggest or force
+its employee(s), whether full-time or part-time, or its independent
+contractor(s), in any methods, to agree in oral or written form, to
+directly or indirectly restrict, weaken or relinquish his or her
+rights or remedies under such laws, regulations, rules and standards
+relating to labor and employment as mentioned above, no matter whether
+such written or oral agreements are enforceable under the laws of the
+said jurisdiction, nor shall such individual or the legal entity
+limit, in any methods, the rights of its employee(s) or independent
+contractor(s) from reporting or complaining to the copyright holder or
+relevant authorities monitoring the compliance of the license about
+its violation(s) of the said license.
+
+THE LICENSED WORK IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+IN NO EVENT SHALL THE COPYRIGHT HOLDER BE LIABLE FOR ANY CLAIM,
+DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
+OTHERWISE, ARISING FROM, OUT OF OR IN ANY WAY CONNECTION WITH THE
+LICENSED WORK OR THE USE OR OTHER DEALINGS IN THE LICENSED WORK.
+*********************************************************/
+
+import EventEmitter from 'eventemitter3';
+import inherits from 'inherits';
+
+export default {
+  isWeChat() {
+    return /MicroMessenger/i.test(window.navigator.userAgent);
+  },
+  workerify(func, methods = []) {
+    const funcStr = this.getFuncBody(func.toString());
+    function __Worker__(data) {
+      EventEmitter.call(this);
+      this.id = 0;
+      this.resolves = [];
+
+      const blob = new Blob([funcStr], { type: 'text/javascript' });
+      this.url = URL.createObjectURL(blob);
+      this.worker = new Worker(this.url);
+      this.worker.onmessage = message => {
+        const { id, data, destroy, type } = message.data;
+        if (destroy) {
+          this.resolves = [];
+          URL.revokeObjectURL(this.url);
+          this.worker.terminate();
+          this.worker = null;
+        } else if (type == 'event') {
+          this.emit(data.type, data.data);
+        } else {
+          for (let i = this.resolves.length - 1; i >= 0; i--) {
+            if (id == this.resolves[i].id) {
+              this.resolves[i].resolve(data);
+              this.resolves.splice(i, 1);
+              break;
+            }
+          }
+        }
+      };
+
+      this.worker.postMessage({ type: 'constructor', id: this.id++, data });
+    }
+
+    inherits(__Worker__, EventEmitter);
+
+    for (let i = 0; i < methods.length; i++) {
+      const type = methods[i];
+      __Worker__.prototype[type] = function(data) {
+        return new Promise((resolve, reject) => {
+          const id = this.id++;
+          this.resolves.push({ id, resolve, reject });
+          if (this.worker) {
+            this.worker.postMessage({ type, id, data });
+          }
+        });
+      };
+    }
+
+    return __Worker__;
+  },
+  getFuncBody(funcStr) {
+    return funcStr
+      .trim()
+      .match(/^function\s*\w*\s*\([\w\s,]*\)\s*{([\w\W]*?)}$/)[1];
+  }
+};

src/main.js

@@ -1,6 +1,6 @@
 import Xverse from "./Xverse.js";
 import Codes from "./enum/Codes.js";
-import './video/index.js'
+// import './video/index.js'
 
 const xverse = new Xverse({
   env: "DEV",

src/video/test.js

@@ -0,0 +1,72 @@
+import YUVSurfaceShader from "./YUVSurfaceShader";
+import Texture from "./Texture";
+
+let canvas = null;
+let yuvSurfaceShader = null;
+let yTexture = null;
+let uTexture = null;
+let vTexture = null;
+
+function initWebGLCanvas() {
+  canvas = document.createElement("canvas");
+  canvas.id = "test_canvas";
+  canvas.style = `position: fixed;top:0;left: 0;z-index: 100;`;
+  const gl = canvas.getContext("webgl");
+  yuvSurfaceShader = YUVSurfaceShader.create(gl);
+  yTexture = Texture.create(gl, gl.LUMINANCE);
+  uTexture = Texture.create(gl, gl.LUMINANCE);
+  vTexture = Texture.create(gl, gl.LUMINANCE);
+
+  document.body.append(canvas);
+}
+
+function draw(buffer, width, height) {
+  canvas.width = width;
+  canvas.height = height;
+
+  // the width & height returned are actually padded, so we have to use the frame size to get the real image dimension
+  // when uploading to texture
+  const stride = width; // stride
+  // height is padded with filler rows
+
+  // if we knew the size of the video before encoding, we could cut out the black filler pixels. We don't, so just set
+  // it to the size after encoding
+  const sourceWidth = width;
+  const sourceHeight = height;
+  const maxXTexCoord = sourceWidth / stride;
+  const maxYTexCoord = sourceHeight / height;
+
+  const lumaSize = stride * height;
+  const chromaSize = lumaSize >> 2;
+
+  const yBuffer = buffer.subarray(0, lumaSize);
+  const uBuffer = buffer.subarray(lumaSize, lumaSize + chromaSize);
+  const vBuffer = buffer.subarray(
+    lumaSize + chromaSize,
+    lumaSize + 2 * chromaSize
+  );
+//   console.log("yBuffer", 1);
+  window.updateTexture(yBuffer);
+
+  const chromaHeight = height >> 1;
+  const chromaStride = stride >> 1;
+
+  // we upload the entire image, including stride padding & filler rows. The actual visible image will be mapped
+  // from texture coordinates as to crop out stride padding & filler rows using maxXTexCoord and maxYTexCoord.
+
+  yTexture.image2dBuffer(yBuffer, stride, height);
+  uTexture.image2dBuffer(uBuffer, chromaStride, chromaHeight);
+  vTexture.image2dBuffer(vBuffer, chromaStride, chromaHeight);
+
+  yuvSurfaceShader.setTexture(yTexture, uTexture, vTexture);
+  yuvSurfaceShader.updateShaderData(
+    { w: width, h: height },
+    { maxXTexCoord, maxYTexCoord }
+  );
+  // debugger
+  // data = window.changeTexture(data);
+  // window.updateTexture( data );
+  yuvSurfaceShader.draw();
+}
+
+export { canvas, initWebGLCanvas, draw };

yarn.lock

@@ -1789,6 +1789,11 @@
     "pify" "^3.0.0"
     "strip-bom" "^3.0.0"
 
+"lodash-es@^4.17.21":
+  "integrity" "sha512-mKnC+QJ9pWVzv+C4/U3rRsHapFfHvQFoFB92e52xeyGMcX6/OlIl78je1u8vePzYZSkkogMPJ2yjxxsb89cxyw=="
+  "resolved" "https://registry.npmmirror.com/lodash-es/-/lodash-es-4.17.21.tgz"
+  "version" "4.17.21"
+
 "lodash.debounce@^4.0.8":
   "integrity" "sha512-FT1yDzDYEoYWhnSGnpE/4Kj1fLZkDFyqRb7fNt6FdYOSxlUWAtp42Eh6Wb0rGIv/m9Bgo7x4GhQbm5Ys4SG5ow=="
   "resolved" "https://registry.npmmirror.com/lodash.debounce/-/lodash.debounce-4.0.8.tgz"
@@ -2419,6 +2424,11 @@
   "resolved" "https://registry.npmmirror.com/url-join/-/url-join-4.0.1.tgz"
   "version" "4.0.1"
 
+"uuid@^8.3.2":
+  "integrity" "sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg=="
+  "resolved" "https://registry.npmmirror.com/uuid/-/uuid-8.3.2.tgz"
+  "version" "8.3.2"
+
 "validate-npm-package-license@^3.0.1":
   "integrity" "sha512-DpKm2Ui/xN7/HQKCtpZxoRWBhZ9Z0kqtygG8XCgNQ8ZlDnxuQmWhj566j8fN4Cu3/JmbhsDo7fcAJq4s9h27Ew=="
   "resolved" "https://registry.npmmirror.com/validate-npm-package-license/-/validate-npm-package-license-3.0.4.tgz"