import XVideoRawYUV from "./XVideoRawYUV" import Logger from "./Logger.js" const planeWidth = 1728 const planeHeight = 1080 window.generateRandomArray = ()=>{ var rnd=[]; for(let i=0;i< planeWidth*planeHeight;i++) { rnd[i] = Math.floor(Math.random()*255); } return rnd; } window.updateTexture = (yuv)=> { let Y = yuv.subarray(0, planeWidth*planeHeight); window._videoRawYTexture.update(Y) window._videoRawYTexture.updateSamplingMode(BABYLON.Texture.BILINEAR_SAMPLINGMODE) Y = undefined; } const yuv = new Uint8Array(window.generateRandomArray()) const renderCount = 100; let sub = 0; const logger = new Logger('XMaterial') export default class XMaterialComponent { constructor(e, t) { E(this, "scene"); E(this, "engine"); E(this, "yuvInfo"); E(this, "shaderMode"); E(this, "_panoInfo"); E(this, "_dynamic_size"); E(this, "_dynamic_babylonpose"); E(this, "_dynamic_textures"); E(this, "_dynamic_shaders"); E(this, "_scenemanager"); E(this, "_videoTexture"); E(this, "_videoElement"); E(this, "_lowModelShader"); E(this, "_defaultShader"); E(this, "_inputYUV420", !0); E(this, "_inputPanoYUV420", !0); E(this, "_videoRawYUVTexArray"); E(this, "_isUpdateYUV", !0); E(this, "initMaterial", async()=>new Promise((resolve,t)=>{ this._initDefaultShader(); if(this.shaderMode == 2){ this.initDynamicData(this._panoInfo.dynamicRange, this._panoInfo.width, this._panoInfo.height).then(()=>{ this._initPureVideoShader(); this._prepareRender(this.yuvInfo); }) } else if(this.shaderMode == 1){ this._initPureVideoShader(); this._prepareRender(this.yuvInfo); } // else if(this.shaderMode == 0){ // resolve(!0) // } resolve(!0) // this.shaderMode == 2 ? this.initDynamicData(this._panoInfo.dynamicRange, this._panoInfo.width, this._panoInfo.height).then(()=>{ // this._initPureVideoShader(), // this._prepareRender(this.yuvInfo) // } // ) : this.shaderMode == 1 ? (this._initPureVideoShader(), // this._prepareRender(this.yuvInfo)) : this.shaderMode == 0, // resolve(!0) })); E(this, "_initPureVideoContent", focal_width_height=>{ if(this._inputYUV420){ if(this._videoRawYUVTexArray.getVideoYUVTex(0) != null){ this._lowModelShader.setTexture("texture_video", this._videoRawYUVTexArray.getVideoYUVTex(0)); this._lowModelShader.setFloat("isYUV", 1); BABYLON.Texture.WhenAllReady([this._videoRawYUVTexArray.getVideoYUVTex(0)], ()=>{ this._changePureVideoLowModelShaderCanvasSize(focal_width_height) }) } } // else{ // this._videoElement = e.videoElement; // this._videoTexture || (this._videoTexture = new VideoTexture("InterVideoTexture",this._videoElement,this.scene,!0,!1)); // BABYLON.Texture.WhenAllReady([this._videoTexture], ()=>{ // this._changePureVideoLowModelShaderCanvasSize({ // width: this._videoElement.height, // height: this._videoElement.width, // fov: e.fov // }) // }); // this._lowModelShader.setTexture("texture_video", this._videoTexture); // this._lowModelShader.setFloat("isYUV", 0); // } }); E(this, "_changePureVideoLowModelShaderCanvasSize", e=>{ var lowModelShader; const fov = e.fov || 50 const width = e.width || 720 const height = e.height || 1280 const focus = width / (2 * Math.tan(Math.PI * fov / 360)); (lowModelShader = this._lowModelShader) == null || lowModelShader.setVector3("focal_width_height", new BABYLON.Vector3(focus,width,height)) } ); E(this, "updateRawYUVData", (stream,width,height,fov=-1)=>{ var o, a; fov == -1 && (fov = this.yuvInfo.fov); if (this._isUpdateYUV == !0) { const s = { width: width, height: height, fov: fov } const videosResOriArrayIndex = this._videoRawYUVTexArray.findId(width, height) const currentVideoId = this._videoRawYUVTexArray.getCurrentVideoTexId(); if(currentVideoId < 0 || videosResOriArrayIndex != currentVideoId || fov != this.yuvInfo.fov) { this.yuvInfo.width = width; this.yuvInfo.height = height; this.yuvInfo.fov = fov; this._videoRawYUVTexArray.setCurrentVideoTexId(videosResOriArrayIndex); this._changeVideoRes(videosResOriArrayIndex); this._changePureVideoLowModelShaderCanvasSize(s); this._scenemanager.cameraComponent.cameraFovChange(s); this._scenemanager.yuvInfo = s; } let VideoTexture = this._videoRawYUVTexArray.getVideoYUVTex(videosResOriArrayIndex) if(VideoTexture != null){ VideoTexture.update(stream) VideoTexture.updateSamplingMode(BABYLON.Texture.BILINEAR_SAMPLINGMODE) } //(o = this._videoRawYUVTexArray.getVideoYUVTex(videosResOriArrayIndex)) == null || o.update(stream), //(a = this._videoRawYUVTexArray.getVideoYUVTex(videosResOriArrayIndex)) == null || a.updateSamplingMode(BABYLON.Texture.BILINEAR_SAMPLINGMODE) if(window.testPlane != null){ // if(window.testPlane.material == null){ BABYLON.Effect.ShadersStore['customVertexShader'] = houseShader.vertex // ` // precision highp float; // attribute vec2 uv; // attribute vec3 position; // varying vec2 vUV; // uniform mat4 view; // uniform mat4 projection; // uniform mat4 world; // uniform mat4 worldViewProjection; // void main() // { // vUV = uv; // gl_Position = projection * view * world * vec4(position , 1.0); // } // `; BABYLON.Effect.ShadersStore["customFragmentShader"]= houseShader.fragment // ` // precision highp float; // varying vec2 vUV; // uniform sampler2D chrominanceYTexture; // void main() // { // vec2 uv = vUV; // vec3 color = vec3(0,0,0); // color = texture2D(chrominanceYTexture, uv).rgb; // // const mat4 YUV2RGB = mat4 // // ( // // 1.1643828125, 0, 1.59602734375, -.87078515625, // // 1.1643828125, -.39176171875, -.81296875, .52959375, // // 1.1643828125, 2.017234375, 0, -1.081390625, // // 0, 0, 0, 1 // // ); // // vec4 result = vec4( // // texture2D( chrominanceYTexture, vec2( uv.x, uv.y * 0.666666 + 0.333333 ) ).x, // // texture2D( chrominanceYTexture, vec2( uv.x * 0.5, uv.y * 0.333333 ) ).x, // // texture2D( chrominanceYTexture, vec2( 0.5 + uv.x * 0.5, uv.y * 0.333333 ) ).x, // // 1 // // ) * YUV2RGB; // // color = clamp(result.rgb, 0.0, 1.0); // gl_FragColor = vec4( color, 1.0); // } // `; window.testPlane.material = new BABYLON.ShaderMaterial( 'customShader', this.scene, { vertex: "custom", fragment: "custom", }, { attributes: ['uv', 'position'], uniforms: houseShader.uniforms //['view', 'projection', 'worldViewProjection', 'world'], }, ) window._videoRawYTexture = BABYLON.RawTexture.CreateLuminanceTexture( null, planeWidth, planeHeight, this.scene, false, true, ) // window.testPlane.material.setTexture('chrominanceYTexture', window._videoRawYTexture) window.testPlane.material.setTexture('texture_video', window._videoRawYTexture) window.testPlane.material.setFloat('isYUV', 1) window.testPlane.material.setVector3('focal_width_height', new BABYLON.Vector3(772.022491, planeWidth, planeHeight / 1.5)) // window.testPlane.material = new BABYLON.StandardMaterial("xsttest",this.scene); // window.testPlane.material.diffuseTexture = window._videoRawYTexture window.testPlane.material.backFaceCulling = false;//Allways show the front and the back of an element // window.updateTexture( new Uint8Array(window.generateRandomArray()) ); ++sub if(sub< renderCount) { window._videoRawYTexture.update(stream) window._videoRawYTexture.updateSamplingMode(BABYLON.Texture.BILINEAR_SAMPLINGMODE) } // this.scene.meshes.forEach(mesh => { // mesh.material != window.testPlane.material && (mesh.material = window.testPlane.material) // }) } // } } } ); E(this, "_changeVideoRes", e=>{ this._lowModelShader.setTexture("texture_video", this._videoRawYUVTexArray.getVideoYUVTex(e)) } ); E(this, "initDynamicData", (dynamicRange,width,height)=>new Promise((resolve,reject)=>{ this.setDynamicSize(dynamicRange).then(a=>{ if (a) { for (let s = 0; s < dynamicRange; ++s) (l=>{ this.initDynamicTexture(l, width, height), this.initDynamicShaders(l).then(()=>{ this._updatePanoShaderInput(l) } ) } )(s); resolve(!0) } else reject(new XMaterialError(`[Engine] DynamicRoomSize (${dynamicRange}) is too small`)) } ) } ).catch(n=>logger.error(`[Engine] ${n}`))); E(this, "_initDefaultShader", ()=>{ this._defaultShader == null && (this._defaultShader = new BABYLON.GridMaterial("GridShader",this.scene), this._defaultShader.gridRatio = 50, this._defaultShader.lineColor = new BABYLON.Color3(0,0,.5), this._defaultShader.majorUnitFrequency = 1, this._defaultShader.mainColor = new BABYLON.Color3(.6,.6,.6), this._defaultShader.backFaceCulling = !1) } ); E(this, "_initPureVideoShader", ()=>{ if (this._lowModelShader == null) { const e = new BABYLON.ShaderMaterial("PureVideoShader",this.scene,{ vertexSource: pureVideoVertex, fragmentSource: pureVideoFragment },{ attributes: ["uv", "position", "world0", "world1", "world2", "world3"], uniforms: ["view", "projection", "worldViewProjection", "world"], defines: ["#define SHADOWFULLFLOAT"] }); e.setTexture("shadowSampler", null), e.setMatrix("lightSpaceMatrix", null), e.setFloat("haveShadowLight", 0), e.setTexture("texture_video", null), e.setFloat("isYUV", this._inputYUV420 ? 1 : 0), e.setFloat("fireworkLight", 0), e.setVector3("fireworkLightPosition", new BABYLON.Vector3(0,0,0)), e.setVector3("focal_width_height", new BABYLON.Vector3(772.022491,720,1280)), e.backFaceCulling = !1, this._lowModelShader = e } } ); E(this, "setDynamicSize", e=>new Promise((t,r)=>{ e >= 1 && e <= 100 ? (this._dynamic_size = e, t(!0)) : (this._dynamic_size = 1, t(!1)) } )); E(this, "_isInDynamicRange", e=>e < this._dynamic_size && e >= 0); E(this, "initDynamicTexture", (e,t,r)=>{ this._isInDynamicRange(e) && (this._dynamic_textures[e] != null && (this._dynamic_textures[e].dispose(), this._dynamic_textures[e] = null), this._dynamic_textures[e] = new BABYLON.RawTexture(null,t,r * 1.5,BABYLON.Engine.TEXTUREFORMAT_LUMINANCE,this.scene,!1,!0,BABYLON.Texture.NEAREST_SAMPLINGMODE,BABYLON.Engine.TEXTURETYPE_UNSIGNED_BYTE), this._dynamic_textures[e].name = "Pano_Dynamic_" + e + "_" + Date.now()) } ); E(this, "initDynamicShaders", e=>(logger.info("[Engine] Material init dynamic shader."), new Promise((t,r)=>{ this._dynamic_shaders[e] != null && this._dynamic_shaders[e].dispose(); const n = new BABYLON.ShaderMaterial("Pano_Shader_" + e,this.scene,{ vertexSource: panoVertex, fragmentSource: panoFragment },{ attributes: ["uv", "position", "world0", "world1", "world2", "world3"], uniforms: ["view", "projection", "worldViewProjection", "world"], defines: ["#define SHADOWFULLFLOAT"] }); n.setTexture("texture_pano", null), n.setVector3("centre_pose", new BABYLON.Vector3(0,0,0)), n.setFloat("isYUV", this._inputPanoYUV420 ? 1 : 0), n.setTexture("shadowSampler", null), n.setMatrix("lightSpaceMatrix", null), n.setFloat("haveShadowLight", 0), n.setFloat("fireworkLight", 0), n.setVector3("fireworkLightPosition", new BABYLON.Vector3(0,0,0)), n.backFaceCulling = !1, this._dynamic_shaders[e] = n, t(!0) } ))); this._scenemanager = e, this.scene = e.Scene, this.engine = this.scene.getEngine(), this.shaderMode = 1, this._dynamic_textures = [], this._dynamic_shaders = [], this._dynamic_babylonpose = [], this._videoRawYUVTexArray = new XVideoRawYUV(this.scene,t.videoResOriArray), this.shaderMode = t.shaderMode, t.yuvInfo != null && (this.yuvInfo = t.yuvInfo), t.panoInfo != null && this.setPanoInfo(t.panoInfo) window.testPlane = null this.addPlane() } //调试 addPlane(){ //window.testPlane = BABYLON.Mesh.CreatePlane("xstplane", {width: 6, height: 6}, this.scene); //window.testPlane = BABYLON.Mesh.CreatePlane("xstplane", "6", this.scene); window.testPlane = BABYLON.MeshBuilder.CreateGround("ground", {width: 6, height: 6}, this.scene); window.testPlane.rotation.x = Math.PI/2 window.testPlane.position.y = 3; window.testPlane.position.z = -20; window.testPlane.position.x = 15; } stopYUVUpdate() { this._isUpdateYUV = !1 } allowYUVUpdate() { this._isUpdateYUV = !0 } setPanoInfo(e) { this._panoInfo = e } _prepareRender(focal_width_height) { if(focal_width_height){ this._initPureVideoContent(focal_width_height) this._updatePureVideoShaderInput() } } getPureVideoShader() { return this._lowModelShader } getDefaultShader() { return this._defaultShader } updatePanoPartYUV(e, t, r) { const n = t.subarray(0, r.width * r.height) , o = t.subarray(r.width * r.height, r.width * r.height * 1.25) , a = t.subarray(r.width * r.height * 1.25) , s = this._panoInfo.width , l = this._panoInfo.height; if (this._dynamic_textures[e] != null) { const u = this._dynamic_textures[e].getInternalTexture(); if (u != null && u != null) { const c = this.engine._getTextureTarget(u); this.engine._bindTextureDirectly(c, u, !0), this.engine.updateTextureData(u, n, r.startX, l * 1.5 - r.startY - r.height, r.width, r.height), this.engine.updateTextureData(u, o, r.startX * .5, (l - r.startY - r.height) * .5, r.width * .5 - 1, r.height * .5 - 1), this.engine.updateTextureData(u, a, r.startX * .5 + s * .5, (l - r.startY - r.height) * .5, r.width * .5, r.height * .5), this.engine._bindTextureDirectly(c, null) } } } changePanoImg(e, t) { if (logger.info(`[Engine] changePanoImg, id=${e}, pose=${t.pose.position.x},${t.pose.position.y},${t.pose.position.z}`), !this._isInDynamicRange(e)) return logger.error(`[Engine] ${e} is bigger than dynamic size set in PanoInfo`), Promise.reject(new XMaterialError(`[Engine] ${e} is bigger than dynamic size set in PanoInfo`)); const r = ue4Position2Xverse(t.pose.position); return r && (this._dynamic_babylonpose[e] = { position: r }), new Promise((n,o)=>{ try { typeof t.data == "string" ? (this.setPanoYUV420(!1), this._dynamic_textures[e].updateURL(t.data, null, ()=>{ this._dynamic_textures[e].updateSamplingMode(BABYLON.Texture.NEAREST_SAMPLINGMODE) } )) : (this.isPanoYUV420() == !1 && this.initDynamicTexture(e, this._panoInfo.width, this._panoInfo.height), this.setPanoYUV420(!0), this._dynamic_textures[e].update(t.data), this._dynamic_textures[e].updateSamplingMode(BABYLON.Texture.NEAREST_SAMPLINGMODE)), n(this) } catch (a) { o(new XMaterialError(`[Engine] ChangePanoImg Error! ${a}`)) } } ).then(n=>(t.fov != null && this._scenemanager.cameraComponent.changeCameraFov(t.fov * Math.PI / 180), this._dynamic_shaders[e].setFloat("isYUV", this._inputPanoYUV420 ? 1 : 0), this._dynamic_shaders[e].setTexture("texture_pano", this._dynamic_textures[e]), this._dynamic_shaders[e].setVector3("centre_pose", this._dynamic_babylonpose[e].position), !0)) } setYUV420(e) { this._inputYUV420 = e } isYUV420() { return this._inputYUV420 } setPanoYUV420(e) { this._inputPanoYUV420 = e } isPanoYUV420() { return this._inputPanoYUV420 } getDynamicShader(e) { return this._dynamic_shaders[e] } _updatePureVideoShaderInput() { /* var e, t, r, n, o, a, s, l, u, c; if(this.scene.getLightByName("AvatarLight")){ (e = this._lowModelShader) == null || e.setFloat("haveShadowLight", 1); n = this._lowModelShader if(n != null){ t = this.scene.getLightByName("AvatarLight") if(t == null){ r = void 0 } else{ r = t.getShadowGenerator() } if(r == null){ n.setTexture("shadowSampler",void 0) } else{ n.setTexture("shadowSampler",r.getShadowMapForRendering()) } } //(n = this._lowModelShader) == null || n.setTexture("shadowSampler", (r = (t = this.scene.getLightByName("AvatarLight")) == null ? void 0 : t.getShadowGenerator()) == null ? void 0 : r.getShadowMapForRendering()); s = this._lowModelShader if(s != null){ o = this.scene.getLightByName("AvatarLight") if(o == null){ a = void 0 } else{ a = o.getShadowGenerator() } if(a == null){ s.setMatrix("lightSpaceMatrix",void 0) } else{ s.setMatrix("lightSpaceMatrix",a.getTransformMatrix()) } } //(s = this._lowModelShader) == null || s.setMatrix("lightSpaceMatrix", (a = (o = this.scene.getLightByName("AvatarLight")) == null ? void 0 : o.getShadowGenerator()) == null ? void 0 : a.getTransformMatrix()) } else{ (l = this._lowModelShader) == null || l.setTexture("shadowSampler", this._videoTexture); (u = this._lowModelShader) == null || u.setMatrix("lightSpaceMatrix", new Matrix); (c = this._lowModelShader) == null || c.setFloat("haveShadowLight", 0); } */ let lowModelShader = this._lowModelShader if(lowModelShader!= null){ if(this.scene.getLightByName("AvatarLight")){ lowModelShader.setFloat("haveShadowLight", 1); let avatarLight = this.scene.getLightByName("AvatarLight") let shadow = void 0 if(avatarLight != null){ shadow = avatarLight.getShadowGenerator() } if(shadow == null){ lowModelShader.setTexture("shadowSampler",void 0) lowModelShader.setMatrix("lightSpaceMatrix",void 0) } else{ lowModelShader.setTexture("shadowSampler",shadow.getShadowMapForRendering()) lowModelShader.setMatrix("lightSpaceMatrix",shadow.getTransformMatrix()) } } else{ lowModelShader.setTexture("shadowSampler", this._videoTexture); lowModelShader.setMatrix("lightSpaceMatrix", new Matrix); lowModelShader.setFloat("haveShadowLight", 0); } } if (this.scene.getLightByName("fireworkLight")) { this.scene.registerBeforeRender(()=>{ this._lowModelShader.setFloat("fireworkLight", this.scene.getLightByName("fireworkLight").getScaledIntensity()); var fireworkLight = this.scene.getLightByName("fireworkLight") if(fireworkLight == null){ this._lowModelShader.setVector3("fireworkLightPosition", void 0); } else{ this._lowModelShader.setVector3("fireworkLightPosition", fireworkLight.position); } //this._lowModelShader.setVector3("fireworkLightPosition", (h = this.scene.getLightByName("fireworkLight")) == null ? void 0 : h.position); }); } else { const pointLight = new BABYLON.PointLight("fireworkLight",new BABYLON.Vector3(0,0,0),this.scene); pointLight.intensity = 0 } } _updatePanoShaderInput(e) { var t, r, n, o, a, s, l, u, c, h; if (this._isInDynamicRange(e)) if (this.scene.getLightByName("AvatarLight") ? ((t = this._dynamic_shaders[e]) == null || t.setFloat("haveShadowLight", 1), (o = this._dynamic_shaders[e]) == null || o.setTexture("shadowSampler", (n = (r = this.scene.getLightByName("AvatarLight")) == null ? void 0 : r.getShadowGenerator()) == null ? void 0 : n.getShadowMapForRendering()), (l = this._dynamic_shaders[e]) == null || l.setMatrix("lightSpaceMatrix", (s = (a = this.scene.getLightByName("AvatarLight")) == null ? void 0 : a.getShadowGenerator()) == null ? void 0 : s.getTransformMatrix())) : ((u = this._dynamic_shaders[e]) == null || u.setTexture("shadowSampler", null), (c = this._dynamic_shaders[e]) == null || c.setMatrix("lightSpaceMatrix", new Matrix), (h = this._dynamic_shaders[e]) == null || h.setFloat("haveShadowLight", 0)), this.scene.getLightByName("fireworkLight")) this.scene.registerBeforeRender(()=>{ var f; this._dynamic_shaders[e].setFloat("fireworkLight", this.scene.getLightByName("fireworkLight").getScaledIntensity()), this._dynamic_shaders[e].setVector3("fireworkLightPosition", (f = this.scene.getLightByName("fireworkLight")) == null ? void 0 : f.position) } ); else { const f = new BABYLON.PointLight("fireworkLight",new BABYLON.Vector3(0,0,0),this.scene); f.intensity = 0 } } } var houseShader = { attributes: ['uv', 'position'], uniforms: ['view', 'projection', 'world', 'isYUV', 'focal_width_height', 'texture_video'], defines: ["#define SHADOWFULLFLOAT", "#define NUM_BONE_INFLUENCERS 0", "#define NUM_MORPH_INFLUENCERS 0"], samplers: ['shadowSampler', 'texture_video'], vertex: ` precision highp float; varying vec3 ModelPos; attribute vec2 uv; attribute vec3 position; uniform mat4 view; uniform mat4 projection; uniform mat4 world; void main() { ModelPos = vec3( view * world * vec4(position , 1.0)); gl_Position = projection * view * world * vec4(position , 1.0); } `, fragment: ` precision highp float; varying vec3 ModelPos; uniform float isYUV; // false: 0, true: 1.0 uniform sampler2D texture_video; uniform vec3 focal_width_height; vec2 SampleTex(vec3 pt3d) { return focal_width_height.x / focal_width_height.yz * pt3d.xy / pt3d.z + 0.5; } void main() { vec3 color = vec3(0,0,0); float shadow = 1.0; vec2 uv = SampleTex( normalize(ModelPos) ); if( isYUV < 0.5 ) { color = texture2D(texture_video, uv).rgb; }else{ const mat4 YUV2RGB = mat4 ( 1.1643828125, 0, 1.59602734375, -.87078515625, 1.1643828125, -.39176171875, -.81296875, .52959375, 1.1643828125, 2.017234375, 0, -1.081390625, 0, 0, 0, 1 ); vec4 result = vec4( texture2D( texture_video, vec2( uv.x, uv.y * 0.666666 + 0.333333 ) ).x, texture2D( texture_video, vec2( uv.x * 0.5, uv.y * 0.333333 ) ).x, texture2D( texture_video, vec2( 0.5 + uv.x * 0.5, uv.y * 0.333333 ) ).x, 1 ) * YUV2RGB; color = clamp(result.rgb, 0.0, 1.0); } if( uv.x < 0.0 || uv.x > 1.0 || uv.y < 0.0 || uv.y > 1.0 ) { color = vec3(0,0,0); } gl_FragColor = vec4(shadow * color * 1.0, 1.0); } ` }