Metaverse/src/XMaterialComponent.js

import XVideoRawYUV from "./XVideoRawYUV"
import Logger from "./Logger.js"

const logger = new Logger('XMaterial')
export default class XMaterialComponent {
    constructor(e, t) {
        E(this, "scene");
        E(this, "engine");
        E(this, "yuvInfo");
        E(this, "shaderMode");
        E(this, "_panoInfo");
        E(this, "_dynamic_size");
        E(this, "_dynamic_babylonpose");
        E(this, "_dynamic_textures");
        E(this, "_dynamic_shaders");
        E(this, "_scenemanager");
        E(this, "_videoTexture");
        E(this, "_videoElement");
        E(this, "_lowModelShader");
        E(this, "_defaultShader");
        E(this, "_inputYUV420", !0);
        E(this, "_inputPanoYUV420", !0);
        E(this, "_videoRawYUVTexArray");
        E(this, "_isUpdateYUV", !0);
        E(this, "initMaterial", async()=>new Promise((resolve,t)=>{
            this._initDefaultShader(),
            this.shaderMode == 2 ? this.initDynamicData(this._panoInfo.dynamicRange, this._panoInfo.width, this._panoInfo.height).then(()=>{
                this._initPureVideoShader(),
                this._prepareRender(this.yuvInfo)
            }
            ) : this.shaderMode == 1 ? (this._initPureVideoShader(),
            this._prepareRender(this.yuvInfo)) : this.shaderMode == 0,
            resolve(!0)
        }));

        E(this, "_initPureVideoContent", focal_width_height=>{
            if(this._inputYUV420){
                if(this._videoRawYUVTexArray.getVideoYUVTex(0) != null){
                    this._lowModelShader.setTexture("texture_video", this._videoRawYUVTexArray.getVideoYUVTex(0));
                    this._lowModelShader.setFloat("isYUV", 1);
                    BABYLON.Texture.WhenAllReady([this._videoRawYUVTexArray.getVideoYUVTex(0)], ()=>{
                        this._changePureVideoLowModelShaderCanvasSize(focal_width_height)
                    })
                }
            }
            else{
                this._videoElement = e.videoElement;
                this._videoTexture || (this._videoTexture = new VideoTexture("InterVideoTexture",this._videoElement,this.scene,!0,!1));
                BABYLON.Texture.WhenAllReady([this._videoTexture], ()=>{
                    this._changePureVideoLowModelShaderCanvasSize({
                        width: this._videoElement.height,
                        height: this._videoElement.width,
                        fov: e.fov
                    })
                });
                this._lowModelShader.setTexture("texture_video", this._videoTexture);
                this._lowModelShader.setFloat("isYUV", 0);
            }
        });

        E(this, "_changePureVideoLowModelShaderCanvasSize", e=>{
            var lowModelShader;
            const fov = e.fov || 50
            const width = e.width || 720
            const height = e.height || 1280
            const focus = width / (2 * Math.tan(Math.PI * fov / 360));
            (lowModelShader = this._lowModelShader) == null || lowModelShader.setVector3("focal_width_height", new BABYLON.Vector3(focus,width,height))
        }
        );

        E(this, "updateRawYUVData", (stream,width,height,fov=-1)=>{
            var o, a;
            fov == -1 && (fov = this.yuvInfo.fov);
            if (this._isUpdateYUV == !0) {
                const s = {
                    width: width,
                    height: height,
                    fov: fov
                }
                const videosResOriArrayIndex = this._videoRawYUVTexArray.findId(width, height)
                const currentVideoId = this._videoRawYUVTexArray.getCurrentVideoTexId();

                if(currentVideoId < 0 || videosResOriArrayIndex != currentVideoId || fov != this.yuvInfo.fov)
                {
                    this.yuvInfo.width = width;
                    this.yuvInfo.height = height;
                    this.yuvInfo.fov = fov;
                    this._videoRawYUVTexArray.setCurrentVideoTexId(videosResOriArrayIndex);
                    this._changeVideoRes(videosResOriArrayIndex);
                    this._changePureVideoLowModelShaderCanvasSize(s);
                    this._scenemanager.cameraComponent.cameraFovChange(s);
                    this._scenemanager.yuvInfo = s;
                }

                let VideoTexture = this._videoRawYUVTexArray.getVideoYUVTex(videosResOriArrayIndex)
                if(VideoTexture != null){
                    VideoTexture.update(stream)
                    VideoTexture.updateSamplingMode(BABYLON.Texture.BILINEAR_SAMPLINGMODE)
                }
                //(o = this._videoRawYUVTexArray.getVideoYUVTex(videosResOriArrayIndex)) == null || o.update(stream),
                //(a = this._videoRawYUVTexArray.getVideoYUVTex(videosResOriArrayIndex)) == null || a.updateSamplingMode(BABYLON.Texture.BILINEAR_SAMPLINGMODE)
            }
        }
        );
        E(this, "_changeVideoRes", e=>{
            this._lowModelShader.setTexture("texture_video", this._videoRawYUVTexArray.getVideoYUVTex(e))
        }
        );

        E(this, "initDynamicData", (dynamicRange,width,height)=>new Promise((resolve,reject)=>{
            this.setDynamicSize(dynamicRange).then(a=>{
                if (a) {
                    for (let s = 0; s < dynamicRange; ++s)
                        (l=>{
                            this.initDynamicTexture(l, width, height),
                            this.initDynamicShaders(l).then(()=>{
                                this._updatePanoShaderInput(l)
                            }
                            )
                        }
                        )(s);
                    resolve(!0)
                } else
                    reject(new XMaterialError(`[Engine] DynamicRoomSize (${dynamicRange}) is too small`))
            }
            )
        }
        ).catch(n=>logger.error(`[Engine] ${n}`)));
        
        E(this, "_initDefaultShader", ()=>{
            this._defaultShader == null && (this._defaultShader = new BABYLON.GridMaterial("GridShader",this.scene),
            this._defaultShader.gridRatio = 50,
            this._defaultShader.lineColor = new BABYLON.Color3(0,0,.5),
            this._defaultShader.majorUnitFrequency = 1,
            this._defaultShader.mainColor = new BABYLON.Color3(.6,.6,.6),
            this._defaultShader.backFaceCulling = !1)
        }
        );
        E(this, "_initPureVideoShader", ()=>{
            if (this._lowModelShader == null) {
                const e = new BABYLON.ShaderMaterial("PureVideoShader",this.scene,{
                    vertexSource: pureVideoVertex,
                    fragmentSource: pureVideoFragment
                },{
                    attributes: ["uv", "position", "world0", "world1", "world2", "world3"],
                    uniforms: ["view", "projection", "worldViewProjection", "world"],
                    defines: ["#define SHADOWFULLFLOAT"]
                });
                e.setTexture("shadowSampler", null),
                e.setMatrix("lightSpaceMatrix", null),
                e.setFloat("haveShadowLight", 0),
                e.setTexture("texture_video", null),
                e.setFloat("isYUV", this._inputYUV420 ? 1 : 0),
                e.setFloat("fireworkLight", 0),
                e.setVector3("fireworkLightPosition", new BABYLON.Vector3(0,0,0)),
                e.setVector3("focal_width_height", new BABYLON.Vector3(772.022491,720,1280)),
                e.backFaceCulling = !1,
                this._lowModelShader = e
            }
        }
        );
        E(this, "setDynamicSize", e=>new Promise((t,r)=>{
            e >= 1 && e <= 100 ? (this._dynamic_size = e,
            t(!0)) : (this._dynamic_size = 1,
            t(!1))
        }
        ));
        E(this, "_isInDynamicRange", e=>e < this._dynamic_size && e >= 0);
        E(this, "initDynamicTexture", (e,t,r)=>{
            this._isInDynamicRange(e) && (this._dynamic_textures[e] != null && (this._dynamic_textures[e].dispose(),
            this._dynamic_textures[e] = null),
            this._dynamic_textures[e] = new BABYLON.RawTexture(null,t,r * 1.5,BABYLON.Engine.TEXTUREFORMAT_LUMINANCE,this.scene,!1,!0,BABYLON.Texture.NEAREST_SAMPLINGMODE,BABYLON.Engine.TEXTURETYPE_UNSIGNED_BYTE),
            this._dynamic_textures[e].name = "Pano_Dynamic_" + e + "_" + Date.now())
        }
        );
        E(this, "initDynamicShaders", e=>(logger.info("[Engine] Material init dynamic shader."),
        new Promise((t,r)=>{
            this._dynamic_shaders[e] != null && this._dynamic_shaders[e].dispose();
            const n = new BABYLON.ShaderMaterial("Pano_Shader_" + e,this.scene,{
                vertexSource: panoVertex,
                fragmentSource: panoFragment
            },{
                attributes: ["uv", "position", "world0", "world1", "world2", "world3"],
                uniforms: ["view", "projection", "worldViewProjection", "world"],
                defines: ["#define SHADOWFULLFLOAT"]
            });
            n.setTexture("texture_pano", null),
            n.setVector3("centre_pose", new BABYLON.Vector3(0,0,0)),
            n.setFloat("isYUV", this._inputPanoYUV420 ? 1 : 0),
            n.setTexture("shadowSampler", null),
            n.setMatrix("lightSpaceMatrix", null),
            n.setFloat("haveShadowLight", 0),
            n.setFloat("fireworkLight", 0),
            n.setVector3("fireworkLightPosition", new BABYLON.Vector3(0,0,0)),
            n.backFaceCulling = !1,
            this._dynamic_shaders[e] = n,
            t(!0)
        }
        )));
        this._scenemanager = e,
        this.scene = e.Scene,
        this.engine = this.scene.getEngine(),
        this.shaderMode = 1,
        this._dynamic_textures = [],
        this._dynamic_shaders = [],
        this._dynamic_babylonpose = [],
        this._videoRawYUVTexArray = new XVideoRawYUV(this.scene,t.videoResOriArray),
        this.shaderMode = t.shaderMode,
        t.yuvInfo != null && (this.yuvInfo = t.yuvInfo),
        t.panoInfo != null && this.setPanoInfo(t.panoInfo)
    }
    stopYUVUpdate() {
        this._isUpdateYUV = !1
    }
    allowYUVUpdate() {
        this._isUpdateYUV = !0
    }
    setPanoInfo(e) {
        this._panoInfo = e
    }
    _prepareRender(focal_width_height) {
        if(focal_width_height){
            this._initPureVideoContent(focal_width_height)
            this._updatePureVideoShaderInput()
        }
    }
    getPureVideoShader() {
        return this._lowModelShader
    }
    getDefaultShader() {
        return this._defaultShader
    }
    updatePanoPartYUV(e, t, r) {
        const n = t.subarray(0, r.width * r.height)
          , o = t.subarray(r.width * r.height, r.width * r.height * 1.25)
          , a = t.subarray(r.width * r.height * 1.25)
          , s = this._panoInfo.width
          , l = this._panoInfo.height;
        if (this._dynamic_textures[e] != null) {
            const u = this._dynamic_textures[e].getInternalTexture();
            if (u != null && u != null) {
                const c = this.engine._getTextureTarget(u);
                this.engine._bindTextureDirectly(c, u, !0),
                this.engine.updateTextureData(u, n, r.startX, l * 1.5 - r.startY - r.height, r.width, r.height),
                this.engine.updateTextureData(u, o, r.startX * .5, (l - r.startY - r.height) * .5, r.width * .5 - 1, r.height * .5 - 1),
                this.engine.updateTextureData(u, a, r.startX * .5 + s * .5, (l - r.startY - r.height) * .5, r.width * .5, r.height * .5),
                this.engine._bindTextureDirectly(c, null)
            }
        }
    }
    changePanoImg(e, t) {
        if (logger.info(`[Engine] changePanoImg, id=${e}, pose=${t.pose.position.x},${t.pose.position.y},${t.pose.position.z}`),
        !this._isInDynamicRange(e))
            return logger.error(`[Engine] ${e} is bigger than dynamic size set in PanoInfo`),
            Promise.reject(new XMaterialError(`[Engine] ${e} is bigger than dynamic size set in PanoInfo`));
        const r = ue4Position2Xverse(t.pose.position);
        return r && (this._dynamic_babylonpose[e] = {
            position: r
        }),
        new Promise((n,o)=>{
            try {
                typeof t.data == "string" ? (this.setPanoYUV420(!1),
                this._dynamic_textures[e].updateURL(t.data, null, ()=>{
                    this._dynamic_textures[e].updateSamplingMode(BABYLON.Texture.NEAREST_SAMPLINGMODE)
                }
                )) : (this.isPanoYUV420() == !1 && this.initDynamicTexture(e, this._panoInfo.width, this._panoInfo.height),
                this.setPanoYUV420(!0),
                this._dynamic_textures[e].update(t.data),
                this._dynamic_textures[e].updateSamplingMode(BABYLON.Texture.NEAREST_SAMPLINGMODE)),
                n(this)
            } catch (a) {
                o(new XMaterialError(`[Engine] ChangePanoImg Error! ${a}`))
            }
        }
        ).then(n=>(t.fov != null && this._scenemanager.cameraComponent.changeCameraFov(t.fov * Math.PI / 180),
        this._dynamic_shaders[e].setFloat("isYUV", this._inputPanoYUV420 ? 1 : 0),
        this._dynamic_shaders[e].setTexture("texture_pano", this._dynamic_textures[e]),
        this._dynamic_shaders[e].setVector3("centre_pose", this._dynamic_babylonpose[e].position),
        !0))
    }
    setYUV420(e) {
        this._inputYUV420 = e
    }
    isYUV420() {
        return this._inputYUV420
    }
    setPanoYUV420(e) {
        this._inputPanoYUV420 = e
    }
    isPanoYUV420() {
        return this._inputPanoYUV420
    }
    getDynamicShader(e) {
        return this._dynamic_shaders[e]
    }
    _updatePureVideoShaderInput() {
        /*
        var e, t, r, n, o, a, s, l, u, c;

        if(this.scene.getLightByName("AvatarLight")){
            (e = this._lowModelShader) == null || e.setFloat("haveShadowLight", 1);

            n = this._lowModelShader
            if(n != null){
                t = this.scene.getLightByName("AvatarLight")
                if(t == null){
                    r = void 0
                }
                else{
                    r = t.getShadowGenerator()
                }

                if(r == null){
                    n.setTexture("shadowSampler",void 0)
                }
                else{
                    n.setTexture("shadowSampler",r.getShadowMapForRendering())
                }
            }

            //(n = this._lowModelShader) == null || n.setTexture("shadowSampler", (r = (t = this.scene.getLightByName("AvatarLight")) == null ? void 0 : t.getShadowGenerator()) == null ? void 0 : r.getShadowMapForRendering());

            s = this._lowModelShader
            if(s != null){
                o = this.scene.getLightByName("AvatarLight")
                if(o == null){
                    a = void 0
                }
                else{
                    a = o.getShadowGenerator()
                }

                if(a == null){
                    s.setMatrix("lightSpaceMatrix",void 0)
                }
                else{
                    s.setMatrix("lightSpaceMatrix",a.getTransformMatrix())
                }
            }
            //(s = this._lowModelShader) == null || s.setMatrix("lightSpaceMatrix", (a = (o = this.scene.getLightByName("AvatarLight")) == null ? void 0 : o.getShadowGenerator()) == null ? void 0 : a.getTransformMatrix())
        }
        else{
            (l = this._lowModelShader) == null || l.setTexture("shadowSampler", this._videoTexture);
            (u = this._lowModelShader) == null || u.setMatrix("lightSpaceMatrix", new Matrix);
            (c = this._lowModelShader) == null || c.setFloat("haveShadowLight", 0);
        }
        */
        
        let lowModelShader = this._lowModelShader
        if(lowModelShader!= null){
            if(this.scene.getLightByName("AvatarLight")){
                lowModelShader.setFloat("haveShadowLight", 1);

                let avatarLight = this.scene.getLightByName("AvatarLight")
                let shadow = void 0
                if(avatarLight != null){
                    shadow = avatarLight.getShadowGenerator()
                }

                if(shadow == null){
                    lowModelShader.setTexture("shadowSampler",void 0)
                    lowModelShader.setMatrix("lightSpaceMatrix",void 0)
                }
                else{
                    lowModelShader.setTexture("shadowSampler",shadow.getShadowMapForRendering())
                    lowModelShader.setMatrix("lightSpaceMatrix",shadow.getTransformMatrix())
                }
            }
            else{
                lowModelShader.setTexture("shadowSampler", this._videoTexture);
                lowModelShader.setMatrix("lightSpaceMatrix", new Matrix);
                lowModelShader.setFloat("haveShadowLight", 0);
            }
        }

        if (this.scene.getLightByName("fireworkLight"))
        {
            this.scene.registerBeforeRender(()=>{
                
                this._lowModelShader.setFloat("fireworkLight", this.scene.getLightByName("fireworkLight").getScaledIntensity());
                var fireworkLight = this.scene.getLightByName("fireworkLight")
                if(fireworkLight == null){
                    this._lowModelShader.setVector3("fireworkLightPosition",  void 0);
                }
                else{
                    this._lowModelShader.setVector3("fireworkLightPosition",  fireworkLight.position);
                }
                //this._lowModelShader.setVector3("fireworkLightPosition", (h = this.scene.getLightByName("fireworkLight")) == null ? void 0 : h.position);
            });
        }
        else {
            const pointLight = new BABYLON.PointLight("fireworkLight",new BABYLON.Vector3(0,0,0),this.scene);
            pointLight.intensity = 0
        }
    }
    _updatePanoShaderInput(e) {
        var t, r, n, o, a, s, l, u, c, h;
        if (this._isInDynamicRange(e))
            if (this.scene.getLightByName("AvatarLight") ? ((t = this._dynamic_shaders[e]) == null || t.setFloat("haveShadowLight", 1),
            (o = this._dynamic_shaders[e]) == null || o.setTexture("shadowSampler", (n = (r = this.scene.getLightByName("AvatarLight")) == null ? void 0 : r.getShadowGenerator()) == null ? void 0 : n.getShadowMapForRendering()),
            (l = this._dynamic_shaders[e]) == null || l.setMatrix("lightSpaceMatrix", (s = (a = this.scene.getLightByName("AvatarLight")) == null ? void 0 : a.getShadowGenerator()) == null ? void 0 : s.getTransformMatrix())) : ((u = this._dynamic_shaders[e]) == null || u.setTexture("shadowSampler", null),
            (c = this._dynamic_shaders[e]) == null || c.setMatrix("lightSpaceMatrix", new Matrix),
            (h = this._dynamic_shaders[e]) == null || h.setFloat("haveShadowLight", 0)),
            this.scene.getLightByName("fireworkLight"))
                this.scene.registerBeforeRender(()=>{
                    var f;
                    this._dynamic_shaders[e].setFloat("fireworkLight", this.scene.getLightByName("fireworkLight").getScaledIntensity()),
                    this._dynamic_shaders[e].setVector3("fireworkLightPosition", (f = this.scene.getLightByName("fireworkLight")) == null ? void 0 : f.position)
                }
                );
            else {
                const f = new BABYLON.PointLight("fireworkLight",new BABYLON.Vector3(0,0,0),this.scene);
                f.intensity = 0
            }
    }
}