Metaverse/src/XMaterialComponent.js

import XVideoRawYUV from "./XVideoRawYUV"
import Logger from "./Logger.js"

const logger = new Logger('XMaterial')
export default class XMaterialComponent {
    constructor(e, t) {
        E(this, "scene");
        E(this, "engine");
        E(this, "yuvInfo");
        E(this, "shaderMode");
        E(this, "_panoInfo");
        E(this, "_dynamic_size");
        E(this, "_dynamic_babylonpose");
        E(this, "_dynamic_textures");
        E(this, "_dynamic_shaders");
        E(this, "_scenemanager");
        E(this, "_videoTexture");
        E(this, "_videoElement");
        E(this, "_lowModelShader");
        E(this, "_defaultShader");
        E(this, "_inputYUV420", !0);
        E(this, "_inputPanoYUV420", !0);
        E(this, "_videoRawYUVTexArray");
        E(this, "_isUpdateYUV", !0);
        E(this, "initMaterial", async()=>new Promise((e,t)=>{
            this._initDefaultShader(),
            this.shaderMode == 2 ? this.initDynamicData(this._panoInfo.dynamicRange, this._panoInfo.width, this._panoInfo.height).then(()=>{
                this._initPureVideoShader(),
                this._prepareRender(this.yuvInfo)
            }
            ) : this.shaderMode == 1 ? (this._initPureVideoShader(),
            this._prepareRender(this.yuvInfo)) : this.shaderMode == 0,
            e(!0)
        }
        ));
        E(this, "_initPureVideoContent", e=>{
            this._inputYUV420 ? this._videoRawYUVTexArray.getVideoYUVTex(0) != null && (this._lowModelShader.setTexture("texture_video", this._videoRawYUVTexArray.getVideoYUVTex(0)),
            this._lowModelShader.setFloat("isYUV", 1),
            BABYLON.Texture.WhenAllReady([this._videoRawYUVTexArray.getVideoYUVTex(0)], ()=>{
                this._changePureVideoLowModelShaderCanvasSize(e)
            }
            )) : (this._videoElement = e.videoElement,
            this._videoTexture || (this._videoTexture = new VideoTexture("InterVideoTexture",this._videoElement,this.scene,!0,!1)),
            BABYLON.Texture.WhenAllReady([this._videoTexture], ()=>{
                this._changePureVideoLowModelShaderCanvasSize({
                    width: this._videoElement.height,
                    height: this._videoElement.width,
                    fov: e.fov
                })
            }
            ),
            this._lowModelShader.setTexture("texture_video", this._videoTexture),
            this._lowModelShader.setFloat("isYUV", 0))
        }
        );
        E(this, "_changePureVideoLowModelShaderCanvasSize", e=>{
            var a;
            const t = e.fov || 50
              , r = e.width || 720
              , n = e.height || 1280
              , o = r / (2 * Math.tan(Math.PI * t / 360));
            (a = this._lowModelShader) == null || a.setVector3("focal_width_height", new BABYLON.Vector3(o,r,n))
        }
        );
        E(this, "updateRawYUVData", (e,t,r,n=-1)=>{
            var o, a;
            if (n == -1 && (n = this.yuvInfo.fov),
            this._isUpdateYUV == !0) {
                const s = {
                    width: t,
                    height: r,
                    fov: n
                }
                  , l = this._videoRawYUVTexArray.findId(t, r)
                  , u = this._videoRawYUVTexArray.getCurrentVideoTexId();
                (u < 0 || l != u || n != this.yuvInfo.fov) && (this.yuvInfo.width = t,
                this.yuvInfo.height = r,
                this.yuvInfo.fov = n,
                this._videoRawYUVTexArray.setCurrentVideoTexId(l),
                this._changeVideoRes(l),
                this._changePureVideoLowModelShaderCanvasSize(s),
                this._scenemanager.cameraComponent.cameraFovChange(s),
                this._scenemanager.yuvInfo = s),
                (o = this._videoRawYUVTexArray.getVideoYUVTex(l)) == null || o.update(e),
                (a = this._videoRawYUVTexArray.getVideoYUVTex(l)) == null || a.updateSamplingMode(BABYLON.Texture.BILINEAR_SAMPLINGMODE)
            }
        }
        );
        E(this, "_changeVideoRes", e=>{
            this._lowModelShader.setTexture("texture_video", this._videoRawYUVTexArray.getVideoYUVTex(e))
        }
        );
        E(this, "initDynamicData", (e,t,r)=>new Promise((n,o)=>{
            this.setDynamicSize(e).then(a=>{
                if (a) {
                    for (let s = 0; s < e; ++s)
                        (l=>{
                            this.initDynamicTexture(l, t, r),
                            this.initDynamicShaders(l).then(()=>{
                                this._updatePanoShaderInput(l)
                            }
                            )
                        }
                        )(s);
                    n(!0)
                } else
                    o(new XMaterialError(`[Engine] DynamicRoomSize (${e}) is too small`))
            }
            )
        }
        ).catch(n=>logger.error(`[Engine] ${n}`)));
        E(this, "_initDefaultShader", ()=>{
            this._defaultShader == null && (this._defaultShader = new BABYLON.GridMaterial("GridShader",this.scene),
            this._defaultShader.gridRatio = 50,
            this._defaultShader.lineColor = new BABYLON.Color3(0,0,.5),
            this._defaultShader.majorUnitFrequency = 1,
            this._defaultShader.mainColor = new BABYLON.Color3(.6,.6,.6),
            this._defaultShader.backFaceCulling = !1)
        }
        );
        E(this, "_initPureVideoShader", ()=>{
            if (this._lowModelShader == null) {
                const e = new BABYLON.ShaderMaterial("PureVideoShader",this.scene,{
                    vertexSource: pureVideoVertex,
                    fragmentSource: pureVideoFragment
                },{
                    attributes: ["uv", "position", "world0", "world1", "world2", "world3"],
                    uniforms: ["view", "projection", "worldViewProjection", "world"],
                    defines: ["#define SHADOWFULLFLOAT"]
                });
                e.setTexture("shadowSampler", null),
                e.setMatrix("lightSpaceMatrix", null),
                e.setFloat("haveShadowLight", 0),
                e.setTexture("texture_video", null),
                e.setFloat("isYUV", this._inputYUV420 ? 1 : 0),
                e.setFloat("fireworkLight", 0),
                e.setVector3("fireworkLightPosition", new BABYLON.Vector3(0,0,0)),
                e.setVector3("focal_width_height", new BABYLON.Vector3(772.022491,720,1280)),
                e.backFaceCulling = !1,
                this._lowModelShader = e
            }
        }
        );
        E(this, "setDynamicSize", e=>new Promise((t,r)=>{
            e >= 1 && e <= 100 ? (this._dynamic_size = e,
            t(!0)) : (this._dynamic_size = 1,
            t(!1))
        }
        ));
        E(this, "_isInDynamicRange", e=>e < this._dynamic_size && e >= 0);
        E(this, "initDynamicTexture", (e,t,r)=>{
            this._isInDynamicRange(e) && (this._dynamic_textures[e] != null && (this._dynamic_textures[e].dispose(),
            this._dynamic_textures[e] = null),
            this._dynamic_textures[e] = new BABYLON.RawTexture(null,t,r * 1.5,BABYLON.Engine.TEXTUREFORMAT_LUMINANCE,this.scene,!1,!0,BABYLON.Texture.NEAREST_SAMPLINGMODE,BABYLON.Engine.TEXTURETYPE_UNSIGNED_BYTE),
            this._dynamic_textures[e].name = "Pano_Dynamic_" + e + "_" + Date.now())
        }
        );
        E(this, "initDynamicShaders", e=>(logger.info("[Engine] Material init dynamic shader."),
        new Promise((t,r)=>{
            this._dynamic_shaders[e] != null && this._dynamic_shaders[e].dispose();
            const n = new BABYLON.ShaderMaterial("Pano_Shader_" + e,this.scene,{
                vertexSource: panoVertex,
                fragmentSource: panoFragment
            },{
                attributes: ["uv", "position", "world0", "world1", "world2", "world3"],
                uniforms: ["view", "projection", "worldViewProjection", "world"],
                defines: ["#define SHADOWFULLFLOAT"]
            });
            n.setTexture("texture_pano", null),
            n.setVector3("centre_pose", new BABYLON.Vector3(0,0,0)),
            n.setFloat("isYUV", this._inputPanoYUV420 ? 1 : 0),
            n.setTexture("shadowSampler", null),
            n.setMatrix("lightSpaceMatrix", null),
            n.setFloat("haveShadowLight", 0),
            n.setFloat("fireworkLight", 0),
            n.setVector3("fireworkLightPosition", new BABYLON.Vector3(0,0,0)),
            n.backFaceCulling = !1,
            this._dynamic_shaders[e] = n,
            t(!0)
        }
        )));
        this._scenemanager = e,
        this.scene = e.Scene,
        this.engine = this.scene.getEngine(),
        this.shaderMode = 1,
        this._dynamic_textures = [],
        this._dynamic_shaders = [],
        this._dynamic_babylonpose = [],
        this._videoRawYUVTexArray = new XVideoRawYUV(this.scene,t.videoResOriArray),
        this.shaderMode = t.shaderMode,
        t.yuvInfo != null && (this.yuvInfo = t.yuvInfo),
        t.panoInfo != null && this.setPanoInfo(t.panoInfo)
    }
    stopYUVUpdate() {
        this._isUpdateYUV = !1
    }
    allowYUVUpdate() {
        this._isUpdateYUV = !0
    }
    setPanoInfo(e) {
        this._panoInfo = e
    }
    _prepareRender(e) {
        e && (this._initPureVideoContent(e),
        this._updatePureVideoShaderInput())
    }
    getPureVideoShader() {
        return this._lowModelShader
    }
    getDefaultShader() {
        return this._defaultShader
    }
    updatePanoPartYUV(e, t, r) {
        const n = t.subarray(0, r.width * r.height)
          , o = t.subarray(r.width * r.height, r.width * r.height * 1.25)
          , a = t.subarray(r.width * r.height * 1.25)
          , s = this._panoInfo.width
          , l = this._panoInfo.height;
        if (this._dynamic_textures[e] != null) {
            const u = this._dynamic_textures[e].getInternalTexture();
            if (u != null && u != null) {
                const c = this.engine._getTextureTarget(u);
                this.engine._bindTextureDirectly(c, u, !0),
                this.engine.updateTextureData(u, n, r.startX, l * 1.5 - r.startY - r.height, r.width, r.height),
                this.engine.updateTextureData(u, o, r.startX * .5, (l - r.startY - r.height) * .5, r.width * .5 - 1, r.height * .5 - 1),
                this.engine.updateTextureData(u, a, r.startX * .5 + s * .5, (l - r.startY - r.height) * .5, r.width * .5, r.height * .5),
                this.engine._bindTextureDirectly(c, null)
            }
        }
    }
    changePanoImg(e, t) {
        if (logger.info(`[Engine] changePanoImg, id=${e}, pose=${t.pose.position.x},${t.pose.position.y},${t.pose.position.z}`),
        !this._isInDynamicRange(e))
            return logger.error(`[Engine] ${e} is bigger than dynamic size set in PanoInfo`),
            Promise.reject(new XMaterialError(`[Engine] ${e} is bigger than dynamic size set in PanoInfo`));
        const r = ue4Position2Xverse(t.pose.position);
        return r && (this._dynamic_babylonpose[e] = {
            position: r
        }),
        new Promise((n,o)=>{
            try {
                typeof t.data == "string" ? (this.setPanoYUV420(!1),
                this._dynamic_textures[e].updateURL(t.data, null, ()=>{
                    this._dynamic_textures[e].updateSamplingMode(BABYLON.Texture.NEAREST_SAMPLINGMODE)
                }
                )) : (this.isPanoYUV420() == !1 && this.initDynamicTexture(e, this._panoInfo.width, this._panoInfo.height),
                this.setPanoYUV420(!0),
                this._dynamic_textures[e].update(t.data),
                this._dynamic_textures[e].updateSamplingMode(BABYLON.Texture.NEAREST_SAMPLINGMODE)),
                n(this)
            } catch (a) {
                o(new XMaterialError(`[Engine] ChangePanoImg Error! ${a}`))
            }
        }
        ).then(n=>(t.fov != null && this._scenemanager.cameraComponent.changeCameraFov(t.fov * Math.PI / 180),
        this._dynamic_shaders[e].setFloat("isYUV", this._inputPanoYUV420 ? 1 : 0),
        this._dynamic_shaders[e].setTexture("texture_pano", this._dynamic_textures[e]),
        this._dynamic_shaders[e].setVector3("centre_pose", this._dynamic_babylonpose[e].position),
        !0))
    }
    setYUV420(e) {
        this._inputYUV420 = e
    }
    isYUV420() {
        return this._inputYUV420
    }
    setPanoYUV420(e) {
        this._inputPanoYUV420 = e
    }
    isPanoYUV420() {
        return this._inputPanoYUV420
    }
    getDynamicShader(e) {
        return this._dynamic_shaders[e]
    }
    _updatePureVideoShaderInput() {
        var e, t, r, n, o, a, s, l, u, c;
        if (this.scene.getLightByName("AvatarLight") ? ((e = this._lowModelShader) == null || e.setFloat("haveShadowLight", 1),
        (n = this._lowModelShader) == null || n.setTexture("shadowSampler", (r = (t = this.scene.getLightByName("AvatarLight")) == null ? void 0 : t.getShadowGenerator()) == null ? void 0 : r.getShadowMapForRendering()),
        (s = this._lowModelShader) == null || s.setMatrix("lightSpaceMatrix", (a = (o = this.scene.getLightByName("AvatarLight")) == null ? void 0 : o.getShadowGenerator()) == null ? void 0 : a.getTransformMatrix())) : ((l = this._lowModelShader) == null || l.setTexture("shadowSampler", this._videoTexture),
        (u = this._lowModelShader) == null || u.setMatrix("lightSpaceMatrix", new Matrix),
        (c = this._lowModelShader) == null || c.setFloat("haveShadowLight", 0)),
        this.scene.getLightByName("fireworkLight"))
            this.scene.registerBeforeRender(()=>{
                var h;
                this._lowModelShader.setFloat("fireworkLight", this.scene.getLightByName("fireworkLight").getScaledIntensity()),
                this._lowModelShader.setVector3("fireworkLightPosition", (h = this.scene.getLightByName("fireworkLight")) == null ? void 0 : h.position)
            }
            );
        else {
            const h = new BABYLON.PointLight("fireworkLight",new BABYLON.Vector3(0,0,0),this.scene);
            h.intensity = 0
        }
    }
    _updatePanoShaderInput(e) {
        var t, r, n, o, a, s, l, u, c, h;
        if (this._isInDynamicRange(e))
            if (this.scene.getLightByName("AvatarLight") ? ((t = this._dynamic_shaders[e]) == null || t.setFloat("haveShadowLight", 1),
            (o = this._dynamic_shaders[e]) == null || o.setTexture("shadowSampler", (n = (r = this.scene.getLightByName("AvatarLight")) == null ? void 0 : r.getShadowGenerator()) == null ? void 0 : n.getShadowMapForRendering()),
            (l = this._dynamic_shaders[e]) == null || l.setMatrix("lightSpaceMatrix", (s = (a = this.scene.getLightByName("AvatarLight")) == null ? void 0 : a.getShadowGenerator()) == null ? void 0 : s.getTransformMatrix())) : ((u = this._dynamic_shaders[e]) == null || u.setTexture("shadowSampler", null),
            (c = this._dynamic_shaders[e]) == null || c.setMatrix("lightSpaceMatrix", new Matrix),
            (h = this._dynamic_shaders[e]) == null || h.setFloat("haveShadowLight", 0)),
            this.scene.getLightByName("fireworkLight"))
                this.scene.registerBeforeRender(()=>{
                    var f;
                    this._dynamic_shaders[e].setFloat("fireworkLight", this.scene.getLightByName("fireworkLight").getScaledIntensity()),
                    this._dynamic_shaders[e].setVector3("fireworkLightPosition", (f = this.scene.getLightByName("fireworkLight")) == null ? void 0 : f.position)
                }
                );
            else {
                const f = new BABYLON.PointLight("fireworkLight",new BABYLON.Vector3(0,0,0),this.scene);
                f.intensity = 0
            }
    }
}