Merge pull request #9635 from Popov72/async-serialize

Async serialization: Fix bug with serialization of base64 texture data

src/Engines/Extensions/engine.readTexture.ts

@@ -6,10 +6,13 @@ declare module "../../Engines/thinEngine" {
     export interface ThinEngine {
         /** @hidden */
         _readTexturePixels(texture: InternalTexture, width: number, height: number, faceIndex?: number, level?: number, buffer?: Nullable<ArrayBufferView>, flushRenderer?: boolean): Promise<ArrayBufferView>;
+
+        /** @hidden */
+        _readTexturePixelsSync(texture: InternalTexture, width: number, height: number, faceIndex?: number, level?: number, buffer?: Nullable<ArrayBufferView>, flushRenderer?: boolean): ArrayBufferView;
     }
 }
 
-ThinEngine.prototype._readTexturePixels = function(texture: InternalTexture, width: number, height: number, faceIndex = -1, level = 0, buffer: Nullable<ArrayBufferView> = null, flushRenderer = true): Promise<ArrayBufferView> {
+ThinEngine.prototype._readTexturePixelsSync = function(texture: InternalTexture, width: number, height: number, faceIndex = -1, level = 0, buffer: Nullable<ArrayBufferView> = null, flushRenderer = true): ArrayBufferView {
     let gl = this._gl;
     if (!gl) {
         throw new Error ("Engine does not have gl rendering context.");
@@ -55,5 +58,9 @@ ThinEngine.prototype._readTexturePixels = function(texture: InternalTexture, wid
     gl.readPixels(0, 0, width, height, gl.RGBA, readType, <DataView>buffer);
     gl.bindFramebuffer(gl.FRAMEBUFFER, this._currentFramebuffer);
 
-    return Promise.resolve(buffer);
-};
+    return buffer;
+};
+
+ThinEngine.prototype._readTexturePixels = function(texture: InternalTexture, width: number, height: number, faceIndex = -1, level = 0, buffer: Nullable<ArrayBufferView> = null, flushRenderer = true): Promise<ArrayBufferView> {
+    return Promise.resolve(this._readTexturePixelsSync(texture, width, height, faceIndex, level, buffer, flushRenderer));
+};

src/Engines/engineFeatures.ts

@@ -45,6 +45,9 @@ export interface EngineFeatures {
     /** Indicates that the switch/case construct is supported in shaders */
     supportSwitchCaseInShader: boolean;
 
+    /** Indicates that synchronous texture reading is supported */
+    supportSyncTextureRead: boolean;
+
     /** @hidden */
     _collectUbosUpdatedInFrame: boolean;
 }

src/Engines/nativeEngine.ts

@@ -815,6 +815,7 @@ export class NativeEngine extends Engine {
             supportSSAO2: false,
             supportExtendedTextureFormats: false,
             supportSwitchCaseInShader: false,
+            supportSyncTextureRead: false,
             _collectUbosUpdatedInFrame: false,
         };
 

src/Engines/nullEngine.ts

@@ -160,6 +160,7 @@ export class NullEngine extends Engine {
             supportSSAO2: false,
             supportExtendedTextureFormats: false,
             supportSwitchCaseInShader: false,
+            supportSyncTextureRead: false,
             _collectUbosUpdatedInFrame: false,
         };
 

src/Engines/thinEngine.ts

@@ -1076,6 +1076,7 @@ export class ThinEngine {
             supportSSAO2: this._webGLVersion !== 1,
             supportExtendedTextureFormats: this._webGLVersion !== 1,
             supportSwitchCaseInShader: this._webGLVersion !== 1,
+            supportSyncTextureRead: true,
             _collectUbosUpdatedInFrame: false,
         };
     }

src/Engines/webgpuEngine.ts

@@ -374,6 +374,8 @@ export class WebGPUEngine extends Engine {
 
         this._sharedInit(canvas, !!options.doNotHandleTouchAction, options.audioEngine);
 
+        this._shaderProcessor = this._getShaderProcessor();
+
         // TODO. WEBGPU. Use real way to do it.
         this._canvas.style.transform = "scaleY(-1)";
     }
@@ -554,6 +556,7 @@ export class WebGPUEngine extends Engine {
             supportSSAO2: true,
             supportExtendedTextureFormats: true,
             supportSwitchCaseInShader: true,
+            supportSyncTextureRead: false,
             _collectUbosUpdatedInFrame: true,
         };
     }
@@ -2291,6 +2294,11 @@ export class WebGPUEngine extends Engine {
         return this._textureHelper.readPixels(gpuTextureWrapper.underlyingResource!, 0, 0, width, height, gpuTextureWrapper.format, faceIndex, level, buffer);
     }
 
+    /** @hidden */
+    public _readTexturePixelsSync(texture: InternalTexture, width: number, height: number, faceIndex = -1, level = 0, buffer: Nullable<ArrayBufferView> = null, flushRenderer = true): ArrayBufferView {
+        throw "_readTexturePixelsSync is unsupported in WebGPU!";
+    }
+
     //------------------------------------------------------------------------------
     //                              Render Target Textures
     //------------------------------------------------------------------------------

src/Materials/Textures/baseTexture.ts

@@ -629,6 +629,40 @@ export class BaseTexture extends ThinTexture implements IAnimatable {
     }
 
     /** @hidden */
+    public _readPixelsSync(faceIndex = 0, level = 0, buffer: Nullable<ArrayBufferView> = null, flushRenderer = true): Nullable<ArrayBufferView> {
+        if (!this._texture) {
+            return null;
+        }
+
+        var size = this.getSize();
+        var width = size.width;
+        var height = size.height;
+
+        const engine = this._getEngine();
+        if (!engine) {
+            return null;
+        }
+
+        if (level != 0) {
+            width = width / Math.pow(2, level);
+            height = height / Math.pow(2, level);
+
+            width = Math.round(width);
+            height = Math.round(height);
+        }
+
+        try {
+            if (this._texture.isCube) {
+                return engine._readTexturePixelsSync(this._texture, width, height, faceIndex, level, buffer, flushRenderer);
+            }
+
+            return engine._readTexturePixelsSync(this._texture, width, height, -1, level, buffer, flushRenderer);
+        } catch (e) {
+            return null;
+        }
+    }
+
+    /** @hidden */
     public get _lodTextureHigh(): Nullable<BaseTexture> {
         if (this._texture) {
             return this._texture._lodTextureHigh;

src/Materials/Textures/texture.ts

@@ -681,7 +681,7 @@ export class Texture extends BaseTexture {
             } else if (this.url && StringTools.StartsWith(this.url, "data:") && this._buffer instanceof Uint8Array) {
                 serializationObject.base64String = "data:image/png;base64," + StringTools.EncodeArrayBufferToBase64(this._buffer);
             } else if (Texture.ForceSerializeBuffers) {
-                serializationObject.base64String = CopyTools.GenerateBase64StringFromTexture(this); // TODO WEBGPU serialize should turn asynchronous as GenerateBase64StringFromTexture now returns a promise...
+                serializationObject.base64String = !this._engine || this._engine._features.supportSyncTextureRead ? CopyTools.GenerateBase64StringFromTexture(this) : CopyTools.GenerateBase64StringFromTextureAsync(this);
             }
         }
 

src/Misc/copyTools.ts

@@ -1,3 +1,4 @@
+import { ISize } from "../Maths/math.size";
 import { Nullable } from "../types";
 
 declare type BaseTexture = import("../Materials/Textures/baseTexture").BaseTexture;
@@ -7,25 +8,13 @@ declare type BaseTexture = import("../Materials/Textures/baseTexture").BaseTextu
  */
 export class CopyTools {
     /**
-     * Reads the pixels stored in the webgl texture and returns them as a base64 string
-     * @param texture defines the texture to read pixels from
-     * @param faceIndex defines the face of the texture to read (in case of cube texture)
-     * @param level defines the LOD level of the texture to read (in case of Mip Maps)
+     * Transform some pixel data to a base64 string
+     * @param pixels defines the pixel data to transform to base64
+     * @param size defines the width and height of the (texture) data
+     * @param invertY true if the data must be inverted for the Y coordinate during the conversion
      * @returns The base64 encoded string or null
      */
-    public static async GenerateBase64StringFromTexture(texture: BaseTexture, faceIndex = 0, level = 0): Promise<Nullable<string>> {
-
-        var internalTexture = texture.getInternalTexture();
-        if (!internalTexture) {
-            return null;
-        }
-
-        var pixels = await texture.readPixels(faceIndex, level);
-        if (!pixels) {
-            return null;
-        }
-
-        var size = texture.getSize();
+    public static GenerateBase64StringFromPixelData(pixels: ArrayBufferView, size: ISize, invertY = false): Nullable<string> {
         var width = size.width;
         var height = size.height;
 
@@ -60,7 +49,7 @@ export class CopyTools {
         castData.set(pixels);
         ctx.putImageData(imageData, 0, 0);
 
-        if (internalTexture.invertY) {
+        if (invertY) {
             var canvas2 = document.createElement('canvas');
             canvas2.width = width;
             canvas2.height = height;
@@ -79,4 +68,46 @@ export class CopyTools {
 
         return canvas.toDataURL('image/png');
     }
+
+    /**
+     * Reads the pixels stored in the webgl texture and returns them as a base64 string
+     * @param texture defines the texture to read pixels from
+     * @param faceIndex defines the face of the texture to read (in case of cube texture)
+     * @param level defines the LOD level of the texture to read (in case of Mip Maps)
+     * @returns The base64 encoded string or null
+     */
+    public static GenerateBase64StringFromTexture(texture: BaseTexture, faceIndex = 0, level = 0): Nullable<string> {
+        var internalTexture = texture.getInternalTexture();
+        if (!internalTexture) {
+            return null;
+        }
+
+        var pixels = texture._readPixelsSync(faceIndex, level);
+        if (!pixels) {
+            return null;
+        }
+
+        return CopyTools.GenerateBase64StringFromPixelData(pixels, texture.getSize(), internalTexture.invertY);
+    }
+
+    /**
+     * Reads the pixels stored in the webgl texture and returns them as a base64 string
+     * @param texture defines the texture to read pixels from
+     * @param faceIndex defines the face of the texture to read (in case of cube texture)
+     * @param level defines the LOD level of the texture to read (in case of Mip Maps)
+     * @returns The base64 encoded string or null wrapped in a promise
+     */
+    public static async GenerateBase64StringFromTextureAsync(texture: BaseTexture, faceIndex = 0, level = 0): Promise<Nullable<string>> {
+        var internalTexture = texture.getInternalTexture();
+        if (!internalTexture) {
+            return null;
+        }
+
+        var pixels = await texture.readPixels(faceIndex, level);
+        if (!pixels) {
+            return null;
+        }
+
+        return CopyTools.GenerateBase64StringFromPixelData(pixels, texture.getSize(), internalTexture.invertY);
+    }
 }

src/Misc/sceneRecorder.ts

@@ -12,6 +12,7 @@ import { ParticleSystem } from '../Particles/particleSystem';
 import { MorphTargetManager } from '../Morph/morphTargetManager';
 import { ShadowGenerator } from '../Lights/Shadows/shadowGenerator';
 import { PostProcess } from '../PostProcesses/postProcess';
+import { Texture } from "../Materials/Textures/texture";
 
 /**
  * Class used to record delta files between 2 scene states
@@ -32,13 +33,16 @@ export class SceneRecorder {
 
     /**
      * Get the delta between current state and original state
-     * @returns a string containing the delta
+     * @returns a any containing the delta
      */
-    public getDelta() {
+    public getDelta(): any {
         if (!this._trackedScene) {
             return null;
         }
 
+        const currentForceSerializeBuffers = Texture.ForceSerializeBuffers;
+        Texture.ForceSerializeBuffers = false;
+
         let newJSON = SceneSerializer.Serialize(this._trackedScene);
         let deltaJSON: any = {};
 
@@ -46,6 +50,8 @@ export class SceneRecorder {
             this._compareCollections(node, this._savedJSON[node], newJSON[node], deltaJSON);
         }
 
+        Texture.ForceSerializeBuffers = currentForceSerializeBuffers;
+
         return deltaJSON;
     }
 

src/Misc/sceneSerializer.ts

@@ -6,6 +6,7 @@ import { Material } from "../Materials/material";
 import { Scene } from "../scene";
 import { Light } from "../Lights/light";
 import { SerializationHelper } from "./decorators";
+import { Texture } from "../Materials/Textures/texture";
 
 var serializedGeometries: Geometry[] = [];
 var serializeGeometry = (geometry: Geometry, serializationGeometries: any): any => {
@@ -110,12 +111,22 @@ export class SceneSerializer {
 
     /**
      * Serialize a scene into a JSON compatible object
+     * Note that if the current engine does not support synchronous texture reading (like WebGPU), you should use SerializeAsync instead
+     * as else you may not retrieve the proper base64 encoded texture data (when using the Texture.ForceSerializeBuffers flag)
      * @param scene defines the scene to serialize
      * @returns a JSON compatible object
      */
     public static Serialize(scene: Scene): any {
+        return SceneSerializer._Serialize(scene);
+    }
+
+    private static _Serialize(scene: Scene, checkSyncReadSupported = true): any {
         var serializationObject: any = {};
 
+        if (checkSyncReadSupported && !scene.getEngine()._features.supportSyncTextureRead && Texture.ForceSerializeBuffers) {
+            console.warn("The serialization object may not contain the proper base64 encoded texture data! You should use the SerializeAsync method instead.");
+        }
+
         SceneSerializer.ClearCache();
 
         // Scene
@@ -316,6 +327,45 @@ export class SceneSerializer {
     }
 
     /**
+     * Serialize a scene into a JSON compatible object
+     * @param scene defines the scene to serialize
+     * @returns a JSON promise compatible object
+     */
+    public static SerializeAsync(scene: Scene): Promise<any> {
+        const serializationObject = SceneSerializer._Serialize(scene, false);
+
+        const promises: Array<Promise<any>> = [];
+
+        this._CollectPromises(serializationObject, promises);
+
+        return Promise.all(promises).then(() => serializationObject);
+    }
+
+    private static _CollectPromises(obj: any, promises: Array<Promise<any>>): void  {
+        if (Array.isArray(obj)) {
+            for (let i = 0; i < obj.length; ++i) {
+                const o = obj[i];
+                if (o instanceof Promise) {
+                    promises.push(o.then((res: any) => obj[i] = res));
+                } else if (o instanceof Object || Array.isArray(o)) {
+                    this._CollectPromises(o, promises);
+                }
+            }
+        } else if (obj instanceof Object) {
+            for (const name in obj) {
+                if (obj.hasOwnProperty(name)) {
+                    const o = obj[name];
+                    if (o instanceof Promise) {
+                        promises.push(o.then((res: any) => obj[name] = res));
+                    } else if (o instanceof Object || Array.isArray(o)) {
+                        this._CollectPromises(o, promises);
+                    }
+                }
+            }
+        }
+    }
+
+    /**
      * Serialize a mesh into a JSON compatible object
      * @param toSerialize defines the mesh to serialize
      * @param withParents defines if parents must be serialized as well