Merge branch 'master' of http://192.168.0.115:3000/xushiting/Metaverse

src/ActionsHandler.js

@@ -29,7 +29,7 @@ export default class ActionsHandler {
 
     async sendData(e) {
 
-        //console.log('发送数据：'+JSON.stringify(e))
+        console.log('发送数据：'+JSON.stringify(e))
 
         await this.beforeSend(e);
         const t = util.uuid();

src/Workers.js

@@ -437,32 +437,32 @@ export default class Workers {
         },
         this.setPtr = (this.setPtr + 1) % this.cachedLength
     }
-    SendCacheFrameInfo(e) {
-        var h, f, d, _, g, m, v;
-        const t = e.data.cachedKey
-          , r = e.data.metadata
-          , n = t
-          , o = r
-          , a = (d = (f = (h = o.newUserStates) == null ? void 0 : h.find(y=>y.userId === this.rtcp.network.room.userId)) == null ? void 0 : f.playerState) == null ? void 0 : d.roomTypeId
-          , s = this.rtcp.network.room.skinId
-          , l = (v = (m = (g = (_ = o.newUserStates) == null ? void 0 : _.find(y=>y.userId === this._rtcp.network.room.userId)) == null ? void 0 : g.playerState) == null ? void 0 : m.player) == null ? void 0 : v.position
-          , u = {
-            MsgType: 1,
-            FrameCacheMsg: {
-                FrameIndex: n,
-                RoomTypeId: a,
-                SkinID: s,
-                Position: l
-            }
-        };
-        let c = "";
-        try {
-            c = JSON.stringify(u)
-        } catch (y) {
-            defaultLogger.error(y);
-            return
-        }
-    }
+    // SendCacheFrameInfo(e) {
+    //     var h, f, d, _, g, m, v;
+    //     const t = e.data.cachedKey
+    //       , r = e.data.metadata
+    //       , n = t
+    //       , o = r
+    //       , a = (d = (f = (h = o.newUserStates) == null ? void 0 : h.find(y=>y.userId === this.rtcp.network.room.userId)) == null ? void 0 : f.playerState) == null ? void 0 : d.roomTypeId
+    //       , s = this.rtcp.network.room.skinId
+    //       , l = (v = (m = (g = (_ = o.newUserStates) == null ? void 0 : _.find(y=>y.userId === this._rtcp.network.room.userId)) == null ? void 0 : g.playerState) == null ? void 0 : m.player) == null ? void 0 : v.position
+    //       , u = {
+    //         MsgType: 1,
+    //         FrameCacheMsg: {
+    //             FrameIndex: n,
+    //             RoomTypeId: a,
+    //             SkinID: s,
+    //             Position: l
+    //         }
+    //     };
+    //     let c = "";
+    //     try {
+    //         c = JSON.stringify(u)
+    //     } catch (y) {
+    //         defaultLogger.error(y);
+    //         return
+    //     }
+    // }
     ReceivePanoramaDecodeMessage(e) {
         defaultLogger.info("Receive Panorama Image in Workers.ts"),
         MaskSetToOne(e.data.tileId, this.PanoView);

src/XMaterialComponent.js

@@ -1,10 +1,12 @@
 import XVideoRawYUV from "./XVideoRawYUV"
 import Logger from "./Logger.js"
 
+const planeWidth  = 1728
+const planeHeight = 1080
 
 window.generateRandomArray = ()=>{
     var rnd=[];
-    for(let i=0;i< 512*1024;i++)
+    for(let i=0;i< planeWidth*planeHeight;i++)
     {
         rnd[i] = Math.floor(Math.random()*255);
     }
@@ -13,13 +15,14 @@ window.generateRandomArray = ()=>{
 
 window.updateTexture = (yuv)=>
 {
-    let Y = yuv.subarray(0, 512*1024);
+    let Y = yuv.subarray(0, planeWidth*planeHeight);
     window._videoRawYTexture.update(Y)
     window._videoRawYTexture.updateSamplingMode(BABYLON.Texture.BILINEAR_SAMPLINGMODE)
     Y = undefined;
 }
 const yuv = new Uint8Array(window.generateRandomArray())
-
+const renderCount = 100;
+let sub = 0;
 const logger = new Logger('XMaterial')
 export default class XMaterialComponent {
     constructor(e, t) {
@@ -135,37 +138,56 @@ export default class XMaterialComponent {
                 //(a = this._videoRawYUVTexArray.getVideoYUVTex(videosResOriArrayIndex)) == null || a.updateSamplingMode(BABYLON.Texture.BILINEAR_SAMPLINGMODE)
 
                 if(window.testPlane != null){
-                    if(window.testPlane.material == null){
-                        /*
-                        BABYLON.Effect.ShadersStore['customVertexShader'] = `
-                            precision highp float; 
-                            attribute vec2 uv;
-                            attribute vec3 position;
-                            varying vec2 vUV;
-                            uniform mat4 view;
-                            uniform mat4 projection; 
-                            uniform mat4 world;
-                            uniform mat4 worldViewProjection;
-                            void main()
-                            { 
-                                vUV = uv;
-                                gl_Position = projection * view * world * vec4(position , 1.0); 
-                            }
-                        `;
-                        BABYLON.Effect.ShadersStore["customFragmentShader"]=`
-                            precision highp float;
-                            varying vec2 vUV;
-                            uniform sampler2D chrominanceYTexture;
-                            void main()
-                            {    
-                                vec2 uv = vUV;
-                                vec3 color = vec3(0,0,0);
-                                color = texture2D(chrominanceYTexture, uv).rgb;    
-                                gl_FragColor = vec4( color, 1.0); 
-                            }
-                        `;
-                            
+                    // if(window.testPlane.material == null){
                         
+                        BABYLON.Effect.ShadersStore['customVertexShader'] = houseShader.vertex
+                        // `
+                        //     precision highp float; 
+                        //     attribute vec2 uv;
+                        //     attribute vec3 position;
+                        //     varying vec2 vUV;
+                        //     uniform mat4 view;
+                        //     uniform mat4 projection; 
+                        //     uniform mat4 world;
+                        //     uniform mat4 worldViewProjection;
+                        //     void main()
+                        //     { 
+                        //         vUV = uv;
+                        //         gl_Position = projection * view * world * vec4(position , 1.0); 
+                        //     }
+                        // `;
+                        BABYLON.Effect.ShadersStore["customFragmentShader"]= houseShader.fragment
+                        // `
+                        //     precision highp float;
+                        //     varying vec2 vUV;
+                        //     uniform sampler2D chrominanceYTexture;
+                        //     void main()
+                        //     {    
+                        //         vec2 uv = vUV;
+                        //         vec3 color = vec3(0,0,0);
+                        //         color = texture2D(chrominanceYTexture, uv).rgb;   
+
+                        //         // const mat4 YUV2RGB = mat4
+                        //         // (
+                        //         //     1.1643828125,   0,                1.59602734375,  -.87078515625,
+                        //         //     1.1643828125,   -.39176171875,    -.81296875,     .52959375,
+                        //         //     1.1643828125,   2.017234375,      0,              -1.081390625,
+                        //         //     0,              0,                0,              1
+                        //         // );
+                                
+                        //         // vec4 result = vec4( 
+                        //         //     texture2D( chrominanceYTexture, vec2( uv.x,   uv.y * 0.666666 + 0.333333 ) ).x,
+                        //         //     texture2D( chrominanceYTexture, vec2( uv.x * 0.5,        uv.y * 0.333333 ) ).x, 
+                        //         //     texture2D( chrominanceYTexture, vec2( 0.5 + uv.x * 0.5,  uv.y * 0.333333 ) ).x,
+                        //         //     1
+                        //         // ) * YUV2RGB;
+
+                        //         // color = clamp(result.rgb, 0.0, 1.0); 
+
+                        //         gl_FragColor = vec4( color, 1.0); 
+                        //     }
+                        // `;
+                            
                         window.testPlane.material = new BABYLON.ShaderMaterial(
                             'customShader',
                             this.scene,
@@ -175,32 +197,44 @@ export default class XMaterialComponent {
                             },
                             {
                               attributes: ['uv', 'position'],
-                              uniforms: ['view', 'projection', 'worldViewProjection', 'world'],
+                              uniforms: houseShader.uniforms //['view', 'projection', 'worldViewProjection', 'world'],
                             },
-                        )*/
+                        )
                         
                         window._videoRawYTexture = BABYLON.RawTexture.CreateLuminanceTexture(
                             null,
-                            512,
-                            1024,
+                            planeWidth,
+                            planeHeight,
                             this.scene,
+                            false,
                             true,
-                            true,
-                          )
+                        )
 
-                       // window.testPlane.material.setTexture('chrominanceYTexture', window._videoRawYTexture)
+                        // window.testPlane.material.setTexture('chrominanceYTexture', window._videoRawYTexture)
                         
-                       
-                        
-                        window.testPlane.material = new BABYLON.StandardMaterial("xsttest",this.scene);
-                        window.testPlane.material.diffuseTexture = window._videoRawYTexture
+                        window.testPlane.material.setTexture('texture_video', window._videoRawYTexture)
+                        window.testPlane.material.setFloat('isYUV', 1)
+                        window.testPlane.material.setVector3('focal_width_height', new BABYLON.Vector3(772.022491, planeWidth, planeHeight / 1.5))
+
                         
+                        // window.testPlane.material = new BABYLON.StandardMaterial("xsttest",this.scene);
+                        // window.testPlane.material.diffuseTexture = window._videoRawYTexture
+
                         window.testPlane.material.backFaceCulling = false;//Allways show the front and the back of an element
-                        window.setInterval( ()=>{
-                            window.updateTexture( new Uint8Array(window.generateRandomArray()) );
-                        }, 60);
+
+                        // window.updateTexture( new Uint8Array(window.generateRandomArray()) );
+                        ++sub
+                        if(sub< renderCount)
+                        {
+                            window._videoRawYTexture.update(stream)
+                            window._videoRawYTexture.updateSamplingMode(BABYLON.Texture.BILINEAR_SAMPLINGMODE)
+                        }
+
+                        // this.scene.meshes.forEach(mesh => {
+                        //     mesh.material != window.testPlane.material && (mesh.material = window.testPlane.material)
+                        // })
                     }
-                }
+                // }
             }
         }
         );
@@ -530,4 +564,92 @@ export default class XMaterialComponent {
                 f.intensity = 0
             }
     }
+}
+
+
+var houseShader = {
+    
+    attributes: ['uv', 'position'],
+
+    uniforms: ['view', 'projection', 'world', 
+        'isYUV', 'focal_width_height', 'texture_video'],
+
+    defines: ["#define SHADOWFULLFLOAT", "#define NUM_BONE_INFLUENCERS 0", "#define NUM_MORPH_INFLUENCERS 0"],
+
+    samplers: ['shadowSampler', 'texture_video'],
+
+    vertex: 
+    `
+
+        precision highp float;
+
+        varying vec3 ModelPos;
+        
+        attribute vec2 uv;
+        attribute vec3 position;
+        
+        uniform mat4 view;
+        uniform mat4 projection;
+        
+        uniform mat4 world;
+
+        void main()
+        {
+            ModelPos = vec3( view * world * vec4(position , 1.0));
+            gl_Position = projection * view * world * vec4(position , 1.0); 
+        }
+
+    `,
+
+    fragment: `
+
+        precision highp  float;
+
+        varying vec3 ModelPos;
+        
+        uniform float isYUV;  // false: 0, true: 1.0
+        uniform sampler2D texture_video;
+        
+        uniform vec3 focal_width_height;
+        
+        vec2 SampleTex(vec3 pt3d)
+        {
+            return focal_width_height.x / focal_width_height.yz * pt3d.xy / pt3d.z + 0.5;
+        }
+        
+        void main()
+        {
+            vec3 color = vec3(0,0,0);
+            float shadow = 1.0;
+            
+            vec2 uv =  SampleTex( normalize(ModelPos) );
+
+            if( isYUV < 0.5 )
+            {
+                color = texture2D(texture_video, uv).rgb;
+            }else{
+                const mat4 YUV2RGB = mat4
+                (
+                    1.1643828125, 0, 1.59602734375, -.87078515625,
+                    1.1643828125, -.39176171875, -.81296875, .52959375,
+                    1.1643828125, 2.017234375, 0, -1.081390625,
+                    0, 0, 0, 1
+                );    
+                
+                vec4 result = vec4( 
+                    texture2D( texture_video, vec2( uv.x,   uv.y * 0.666666 + 0.333333 ) ).x,
+                    texture2D( texture_video, vec2( uv.x * 0.5,        uv.y * 0.333333 ) ).x, 
+                    texture2D( texture_video, vec2( 0.5 + uv.x * 0.5,  uv.y * 0.333333 ) ).x,
+                    1
+                ) * YUV2RGB;  
+
+                color = clamp(result.rgb, 0.0, 1.0); 
+            }
+            if( uv.x < 0.0 || uv.x > 1.0 || uv.y < 0.0 || uv.y > 1.0 )
+            {
+                color = vec3(0,0,0);
+            }
+            gl_FragColor = vec4(shadow * color * 1.0, 1.0); 
+        }
+    `
 }

笔记.txt

@@ -0,0 +1,124 @@
+ActionsHandler类中，函数sendData根据用户操作发送不同的数据到服务器
+服务器接受来自sendData的请求数据后，发送视频流（通过webrtc里的inputChannel.onmessage）
+
+前端主线程接受到数据后，开始执行
+this.workers.dataHandle(r.data)  ——> unmarshalStream函数里，通过this.decoderWorker.postMessage(z, [D.buffer])将视频数据传输到worker
+worker里，通过self.onmessage接受到来自主线程的请求并开始执行，然后通过self.postMessage发送请求并传输数据（怀疑将视频流解析成每一帧的yuv数据）到主线程
+
+主线程通过decoderWorker.onmessage来接受来自worker的数据并执行，主要是执行函数：ReceiveDecodeMessage
+ReceiveDecodeMessage函数里给cachedStreams/cachedMetas/cachedPtss/cachedRender赋值（每帧的yuv数据）
+
+UpdateYUV通过定时器定时运行，并读取cachedStreams的数据，如下：
+this.cachedStreams[e] != null && this.executeFunction("stream", {
+	stream: this.cachedStreams[e],
+	width: this.cachedResolution[e].width,
+	height: this.cachedResolution[e].height,
+	pts: this.cachedPtss[e]
+});
+			
+			
+stream函数因为提前注册过，一旦调用executeFunction，就会执行registerFunction("stream")
+registerFunction("stream")里会调用updateRawYUVData
+
+updateRawYUVData函数会将stream传送到材质贴图（_videoRawYUVTexArray里）上，显示最终的效果
+
+
+
+
+/*****************************************************************************************************************************************************************************************************************/
+
+
+
+
+旋转：
+发送数据：
+{
+	"data": {
+		"action_type": 1014,
+		"rotation_action": {
+			"vertical_move": 0,
+			"horizontal_move": -0.003703703703703704
+		}
+	},
+	"sampleRate": 0.02
+}
+
+心跳：
+发送数据：{"data":{"action_type":1009,"echo_msg":{"echoMsg":"1650272307696"}},"track":false}
+
+漫游：
+发送数据：
+{
+	"data": {
+		"action_type": 1,
+		"clicking_action": {
+			"clicking_point": {
+				"x": 1396.4945740787787,
+				"y": 2173.158025839534,
+				"z": 1.5306104000406329
+			},
+			"clicking_type": 3,
+			"extra": "",
+			"attitude": "walk"
+		},
+		"clicking_state": {
+			"roomTypeId": "",
+			"person": 0,
+			"avatarId": "",
+			"skinId": "",
+			"roomId": "",
+			"isHost": false,
+			"isFollowHost": false,
+			"skinDataVersion": "",
+			"avatarComponents": "",
+			"nickName": "",
+			"movingMode": 0,
+			"attitude": "",
+			"areaName": "",
+			"pathName": "",
+			"pathId": "",
+			"avatarSize": 1,
+			"extra": "",
+			"prioritySync": false,
+			"player": {
+				"position": {
+					"x": 1540,
+					"y": 2310,
+					"z": 0
+				},
+				"angle": {
+					"pitch": 0,
+					"yaw": 0,
+					"roll": 0
+				}
+			},
+			"camera": {
+				"position": {
+					"x": 1939.5182,
+					"y": 2329.627,
+					"z": 150
+				},
+				"angle": {
+					"pitch": 0,
+					"yaw": 182.8125,
+					"roll": 0
+				}
+			},
+			"cameraCenter": {
+				"x": 1540,
+				"y": 2310,
+				"z": 0
+			}
+		}
+	}
+}
+
+
+
+
+
+
+
+
+发送数据：{"data":{"action_type":1004,"get_neighbor_points_action":{"point":{"x":1400,"y":2175,"z":0},"level":1,"containSelf":true,"searchRange":1000}}}
+