import { Nullable } from "../types"; import { Tools } from "./tools"; import { Vector3 } from "../Maths/math.vector"; import { Scalar } from "../Maths/math.scalar"; import { SphericalPolynomial } from "../Maths/sphericalPolynomial"; import { InternalTexture, InternalTextureSource } from "../Materials/Textures/internalTexture"; import { BaseTexture } from "../Materials/Textures/baseTexture"; import { Constants } from "../Engines/constants"; import { Scene } from "../scene"; import { PostProcess } from "../PostProcesses/postProcess"; import { Logger } from "../Misc/logger"; import "../Engines/Extensions/engine.renderTargetCube"; import "../Engines/Extensions/engine.readTexture"; import "../Materials/Textures/baseTexture.polynomial"; import "../Shaders/rgbdEncode.fragment"; import "../Shaders/rgbdDecode.fragment"; import { Engine } from '../Engines/engine'; import { ThinEngine } from '../Engines/thinEngine'; import { RGBDTextureTools } from './rgbdTextureTools'; /** * Raw texture data and descriptor sufficient for WebGL texture upload */ export interface EnvironmentTextureInfo { /** * Version of the environment map */ version: number; /** * Width of image */ width: number; /** * Irradiance information stored in the file. */ irradiance: any; /** * Specular information stored in the file. */ specular: any; } /** * Defines One Image in the file. It requires only the position in the file * as well as the length. */ interface BufferImageData { /** * Length of the image data. */ length: number; /** * Position of the data from the null terminator delimiting the end of the JSON. */ position: number; } /** * Defines the specular data enclosed in the file. * This corresponds to the version 1 of the data. */ export interface EnvironmentTextureSpecularInfoV1 { /** * Defines where the specular Payload is located. It is a runtime value only not stored in the file. */ specularDataPosition?: number; /** * This contains all the images data needed to reconstruct the cubemap. */ mipmaps: Array; /** * Defines the scale applied to environment texture. This manages the range of LOD level used for IBL according to the roughness. */ lodGenerationScale: number; } /** * Defines the required storage to save the environment irradiance information. */ interface EnvironmentTextureIrradianceInfoV1 { x: Array; y: Array; z: Array; xx: Array; yy: Array; zz: Array; yz: Array; zx: Array; xy: Array; } /** * Sets of helpers addressing the serialization and deserialization of environment texture * stored in a BabylonJS env file. * Those files are usually stored as .env files. */ export class EnvironmentTextureTools { /** * Magic number identifying the env file. */ private static _MagicBytes = [0x86, 0x16, 0x87, 0x96, 0xf6, 0xd6, 0x96, 0x36]; /** * Gets the environment info from an env file. * @param data The array buffer containing the .env bytes. * @returns the environment file info (the json header) if successfully parsed. */ public static GetEnvInfo(data: ArrayBufferView): Nullable { let dataView = new DataView(data.buffer, data.byteOffset, data.byteLength); let pos = 0; for (let i = 0; i < EnvironmentTextureTools._MagicBytes.length; i++) { if (dataView.getUint8(pos++) !== EnvironmentTextureTools._MagicBytes[i]) { Logger.Error('Not a babylon environment map'); return null; } } // Read json manifest - collect characters up to null terminator let manifestString = ''; let charCode = 0x00; while ((charCode = dataView.getUint8(pos++))) { manifestString += String.fromCharCode(charCode); } let manifest: EnvironmentTextureInfo = JSON.parse(manifestString); if (manifest.specular) { // Extend the header with the position of the payload. manifest.specular.specularDataPosition = pos; // Fallback to 0.8 exactly if lodGenerationScale is not defined for backward compatibility. manifest.specular.lodGenerationScale = manifest.specular.lodGenerationScale || 0.8; } return manifest; } /** * Creates an environment texture from a loaded cube texture. * @param texture defines the cube texture to convert in env file * @return a promise containing the environment data if succesfull. */ public static async CreateEnvTextureAsync(texture: BaseTexture): Promise { let internalTexture = texture.getInternalTexture(); if (!internalTexture) { return Promise.reject("The cube texture is invalid."); } let engine = internalTexture.getEngine() as Engine; if (texture.textureType === Constants.TEXTURETYPE_UNSIGNED_INT) { return Promise.reject("The cube texture should allow HDR (Full Float or Half Float)."); } let textureType = Constants.TEXTURETYPE_FLOAT; if (!engine.getCaps().textureFloatRender) { textureType = Constants.TEXTURETYPE_HALF_FLOAT; if (!engine.getCaps().textureHalfFloatRender) { return Promise.reject("Env texture can only be created when the browser supports half float or full float rendering."); } } let cubeWidth = internalTexture.width; let hostingScene = new Scene(engine); let specularTextures: { [key: number]: ArrayBuffer } = {}; // As we are going to readPixels the faces of the cube, make sure the drawing/update commands for the cube texture are fully sent to the GPU in case it is drawn for the first time in this very frame! engine.flushFramebuffer(); // Read and collect all mipmaps data from the cube. let mipmapsCount = Scalar.ILog2(internalTexture.width); for (let i = 0; i <= mipmapsCount; i++) { let faceWidth = Math.pow(2, mipmapsCount - i); // All faces of the cube. for (let face = 0; face < 6; face++) { let faceData = await texture.readPixels(face, i); let tempTexture = engine.createRawTexture(faceData, faceWidth, faceWidth, Constants.TEXTUREFORMAT_RGBA, false, true, Constants.TEXTURE_NEAREST_SAMPLINGMODE, null, textureType); await RGBDTextureTools.EncodeTextureToRGBD(tempTexture, hostingScene, textureType); engine.flushFramebuffer(); // make sure the tempTexture has its data up to date when reading them just below const rgbdEncodedData = await engine._readTexturePixels(tempTexture, faceWidth, faceWidth); const pngEncodedata = await Tools.DumpDataAsync(faceWidth, faceWidth, rgbdEncodedData, "image/png", undefined, false, true); specularTextures[i * 6 + face] = pngEncodedata as ArrayBuffer; tempTexture.dispose(); } } // We can delete the hosting scene keeping track of all the creation objects hostingScene.dispose(); // Creates the json header for the env texture let info: EnvironmentTextureInfo = { version: 1, width: cubeWidth, irradiance: this._CreateEnvTextureIrradiance(texture), specular: { mipmaps: [], lodGenerationScale: texture.lodGenerationScale } }; // Sets the specular image data information let position = 0; for (let i = 0; i <= mipmapsCount; i++) { for (let face = 0; face < 6; face++) { let byteLength = specularTextures[i * 6 + face].byteLength; info.specular.mipmaps.push({ length: byteLength, position: position }); position += byteLength; } } // Encode the JSON as an array buffer let infoString = JSON.stringify(info); let infoBuffer = new ArrayBuffer(infoString.length + 1); let infoView = new Uint8Array(infoBuffer); // Limited to ascii subset matching unicode. for (let i = 0, strLen = infoString.length; i < strLen; i++) { infoView[i] = infoString.charCodeAt(i); } // Ends up with a null terminator for easier parsing infoView[infoString.length] = 0x00; // Computes the final required size and creates the storage let totalSize = EnvironmentTextureTools._MagicBytes.length + position + infoBuffer.byteLength; let finalBuffer = new ArrayBuffer(totalSize); let finalBufferView = new Uint8Array(finalBuffer); let dataView = new DataView(finalBuffer); // Copy the magic bytes identifying the file in let pos = 0; for (let i = 0; i < EnvironmentTextureTools._MagicBytes.length; i++) { dataView.setUint8(pos++, EnvironmentTextureTools._MagicBytes[i]); } // Add the json info finalBufferView.set(new Uint8Array(infoBuffer), pos); pos += infoBuffer.byteLength; // Finally inserts the texture data for (let i = 0; i <= mipmapsCount; i++) { for (let face = 0; face < 6; face++) { let dataBuffer = specularTextures[i * 6 + face]; finalBufferView.set(new Uint8Array(dataBuffer), pos); pos += dataBuffer.byteLength; } } // Voila return finalBuffer; } /** * Creates a JSON representation of the spherical data. * @param texture defines the texture containing the polynomials * @return the JSON representation of the spherical info */ private static _CreateEnvTextureIrradiance(texture: BaseTexture): Nullable { let polynmials = texture.sphericalPolynomial; if (polynmials == null) { return null; } return { x: [polynmials.x.x, polynmials.x.y, polynmials.x.z], y: [polynmials.y.x, polynmials.y.y, polynmials.y.z], z: [polynmials.z.x, polynmials.z.y, polynmials.z.z], xx: [polynmials.xx.x, polynmials.xx.y, polynmials.xx.z], yy: [polynmials.yy.x, polynmials.yy.y, polynmials.yy.z], zz: [polynmials.zz.x, polynmials.zz.y, polynmials.zz.z], yz: [polynmials.yz.x, polynmials.yz.y, polynmials.yz.z], zx: [polynmials.zx.x, polynmials.zx.y, polynmials.zx.z], xy: [polynmials.xy.x, polynmials.xy.y, polynmials.xy.z] } as any; } /** * Creates the ArrayBufferViews used for initializing environment texture image data. * @param data the image data * @param info parameters that determine what views will be created for accessing the underlying buffer * @return the views described by info providing access to the underlying buffer */ public static CreateImageDataArrayBufferViews(data: ArrayBufferView, info: EnvironmentTextureInfo): Array> { if (info.version !== 1) { throw new Error(`Unsupported babylon environment map version "${info.version}"`); } const specularInfo = info.specular as EnvironmentTextureSpecularInfoV1; // Double checks the enclosed info let mipmapsCount = Scalar.Log2(info.width); mipmapsCount = Math.round(mipmapsCount) + 1; if (specularInfo.mipmaps.length !== 6 * mipmapsCount) { throw new Error(`Unsupported specular mipmaps number "${specularInfo.mipmaps.length}"`); } const imageData = new Array>(mipmapsCount); for (let i = 0; i < mipmapsCount; i++) { imageData[i] = new Array(6); for (let face = 0; face < 6; face++) { const imageInfo = specularInfo.mipmaps[i * 6 + face]; imageData[i][face] = new Uint8Array(data.buffer, data.byteOffset + specularInfo.specularDataPosition! + imageInfo.position, imageInfo.length); } } return imageData; } /** * Uploads the texture info contained in the env file to the GPU. * @param texture defines the internal texture to upload to * @param data defines the data to load * @param info defines the texture info retrieved through the GetEnvInfo method * @returns a promise */ public static UploadEnvLevelsAsync(texture: InternalTexture, data: ArrayBufferView, info: EnvironmentTextureInfo): Promise { if (info.version !== 1) { throw new Error(`Unsupported babylon environment map version "${info.version}"`); } const specularInfo = info.specular as EnvironmentTextureSpecularInfoV1; if (!specularInfo) { // Nothing else parsed so far return Promise.resolve(); } texture._lodGenerationScale = specularInfo.lodGenerationScale; const imageData = EnvironmentTextureTools.CreateImageDataArrayBufferViews(data, info); return EnvironmentTextureTools.UploadLevelsAsync(texture, imageData); } private static _OnImageReadyAsync(image: HTMLImageElement | ImageBitmap, engine: Engine, expandTexture: boolean, rgbdPostProcess: Nullable, url: string, face: number, i: number, generateNonLODTextures: boolean, lodTextures: Nullable<{ [lod: number]: BaseTexture }>, cubeRtt: Nullable, texture: InternalTexture ): Promise { return new Promise((resolve, reject) => { if (expandTexture) { let tempTexture = engine.createTexture(null, true, true, null, Constants.TEXTURE_NEAREST_SAMPLINGMODE, null, (message) => { reject(message); }, image); rgbdPostProcess!.getEffect().executeWhenCompiled(() => { // Uncompress the data to a RTT rgbdPostProcess!.onApply = (effect) => { effect._bindTexture("textureSampler", tempTexture); effect.setFloat2("scale", 1, 1); }; engine.scenes[0].postProcessManager.directRender([rgbdPostProcess!], cubeRtt, true, face, i); // Cleanup engine.restoreDefaultFramebuffer(); tempTexture.dispose(); URL.revokeObjectURL(url); resolve(); }); } else { engine._uploadImageToTexture(texture, image, face, i); // Upload the face to the non lod texture support if (generateNonLODTextures) { let lodTexture = lodTextures![i]; if (lodTexture) { engine._uploadImageToTexture(lodTexture._texture!, image, face, 0); } } resolve(); } } ); } /** * Uploads the levels of image data to the GPU. * @param texture defines the internal texture to upload to * @param imageData defines the array buffer views of image data [mipmap][face] * @returns a promise */ public static UploadLevelsAsync(texture: InternalTexture, imageData: ArrayBufferView[][]): Promise { if (!Tools.IsExponentOfTwo(texture.width)) { throw new Error("Texture size must be a power of two"); } const mipmapsCount = Scalar.ILog2(texture.width) + 1; // Gets everything ready. let engine = texture.getEngine() as Engine; let expandTexture = false; let generateNonLODTextures = false; let rgbdPostProcess: Nullable = null; let cubeRtt: Nullable = null; let lodTextures: Nullable<{ [lod: number]: BaseTexture }> = null; let caps = engine.getCaps(); texture.format = Constants.TEXTUREFORMAT_RGBA; texture.type = Constants.TEXTURETYPE_UNSIGNED_INT; texture.generateMipMaps = true; texture._cachedAnisotropicFilteringLevel = null; engine.updateTextureSamplingMode(Constants.TEXTURE_TRILINEAR_SAMPLINGMODE, texture); // Add extra process if texture lod is not supported if (!caps.textureLOD) { expandTexture = false; generateNonLODTextures = true; lodTextures = {}; } // in webgl 1 there are no ways to either render or copy lod level information for float textures. else if (!ThinEngine.Features.supportRenderAndCopyToLodForFloatTextures) { expandTexture = false; } // If half float available we can uncompress the texture else if (caps.textureHalfFloatRender && caps.textureHalfFloatLinearFiltering) { expandTexture = true; texture.type = Constants.TEXTURETYPE_HALF_FLOAT; } // If full float available we can uncompress the texture else if (caps.textureFloatRender && caps.textureFloatLinearFiltering) { expandTexture = true; texture.type = Constants.TEXTURETYPE_FLOAT; } // Expand the texture if possible if (expandTexture) { // Simply run through the decode PP rgbdPostProcess = new PostProcess("rgbdDecode", "rgbdDecode", null, null, 1, null, Constants.TEXTURE_TRILINEAR_SAMPLINGMODE, engine, false, undefined, texture.type, undefined, null, false); texture._isRGBD = false; texture.invertY = false; cubeRtt = engine.createRenderTargetCubeTexture(texture.width, { generateDepthBuffer: false, generateMipMaps: true, generateStencilBuffer: false, samplingMode: Constants.TEXTURE_TRILINEAR_SAMPLINGMODE, type: texture.type, format: Constants.TEXTUREFORMAT_RGBA }); } else { texture._isRGBD = true; texture.invertY = true; // In case of missing support, applies the same patch than DDS files. if (generateNonLODTextures) { let mipSlices = 3; let scale = texture._lodGenerationScale; let offset = texture._lodGenerationOffset; for (let i = 0; i < mipSlices; i++) { //compute LOD from even spacing in smoothness (matching shader calculation) let smoothness = i / (mipSlices - 1); let roughness = 1 - smoothness; let minLODIndex = offset; // roughness = 0 let maxLODIndex = (mipmapsCount - 1) * scale + offset; // roughness = 1 (mipmaps start from 0) let lodIndex = minLODIndex + (maxLODIndex - minLODIndex) * roughness; let mipmapIndex = Math.round(Math.min(Math.max(lodIndex, 0), maxLODIndex)); let glTextureFromLod = new InternalTexture(engine, InternalTextureSource.Temp); glTextureFromLod.isCube = true; glTextureFromLod.invertY = true; glTextureFromLod.generateMipMaps = false; engine.updateTextureSamplingMode(Constants.TEXTURE_LINEAR_LINEAR, glTextureFromLod); // Wrap in a base texture for easy binding. let lodTexture = new BaseTexture(null); lodTexture.isCube = true; lodTexture._texture = glTextureFromLod; lodTextures![mipmapIndex] = lodTexture; switch (i) { case 0: texture._lodTextureLow = lodTexture; break; case 1: texture._lodTextureMid = lodTexture; break; case 2: texture._lodTextureHigh = lodTexture; break; } } } } let promises: Promise[] = []; // All mipmaps up to provided number of images for (let i = 0; i < imageData.length; i++) { // All faces for (let face = 0; face < 6; face++) { // Constructs an image element from image data let bytes = imageData[i][face]; let blob = new Blob([bytes], { type: 'image/png' }); let url = URL.createObjectURL(blob); let promise: Promise; if (typeof Image === "undefined" || ThinEngine.Features.forceBitmapOverHTMLImageElement) { promise = createImageBitmap(blob, { premultiplyAlpha: "none" }).then((img) => { return this._OnImageReadyAsync(img, engine, expandTexture, rgbdPostProcess, url, face, i, generateNonLODTextures, lodTextures, cubeRtt, texture); }); } else { let image = new Image(); image.src = url; // Enqueue promise to upload to the texture. promise = new Promise((resolve, reject) => { image.onload = () => { this._OnImageReadyAsync(image, engine, expandTexture, rgbdPostProcess, url, face, i, generateNonLODTextures, lodTextures, cubeRtt, texture) .then(() => resolve()) .catch((reason) => { reject(reason); }); }; image.onerror = (error) => { reject(error); }; }); } promises.push(promise); } } // Fill remaining mipmaps with black textures. if (imageData.length < mipmapsCount) { let data: ArrayBufferView; const size = Math.pow(2, mipmapsCount - 1 - imageData.length); const dataLength = size * size * 4; switch (texture.type) { case Constants.TEXTURETYPE_UNSIGNED_INT: { data = new Uint8Array(dataLength); break; } case Constants.TEXTURETYPE_HALF_FLOAT: { data = new Uint16Array(dataLength); break; } case Constants.TEXTURETYPE_FLOAT: { data = new Float32Array(dataLength); break; } } for (let i = imageData.length; i < mipmapsCount; i++) { for (let face = 0; face < 6; face++) { engine._uploadArrayBufferViewToTexture(texture, data!, face, i); } } } // Once all done, finishes the cleanup and return return Promise.all(promises).then(() => { // Release temp RTT. if (cubeRtt) { engine._releaseFramebufferObjects(cubeRtt); engine._releaseTexture(texture); cubeRtt._swapAndDie(texture); } // Release temp Post Process. if (rgbdPostProcess) { rgbdPostProcess.dispose(); } // Flag internal texture as ready in case they are in use. if (generateNonLODTextures) { if (texture._lodTextureHigh && texture._lodTextureHigh._texture) { texture._lodTextureHigh._texture.isReady = true; } if (texture._lodTextureMid && texture._lodTextureMid._texture) { texture._lodTextureMid._texture.isReady = true; } if (texture._lodTextureLow && texture._lodTextureLow._texture) { texture._lodTextureLow._texture.isReady = true; } } }); } /** * Uploads spherical polynomials information to the texture. * @param texture defines the texture we are trying to upload the information to * @param info defines the environment texture info retrieved through the GetEnvInfo method */ public static UploadEnvSpherical(texture: InternalTexture, info: EnvironmentTextureInfo): void { if (info.version !== 1) { Logger.Warn('Unsupported babylon environment map version "' + info.version + '"'); } let irradianceInfo = info.irradiance as EnvironmentTextureIrradianceInfoV1; if (!irradianceInfo) { return; } const sp = new SphericalPolynomial(); Vector3.FromArrayToRef(irradianceInfo.x, 0, sp.x); Vector3.FromArrayToRef(irradianceInfo.y, 0, sp.y); Vector3.FromArrayToRef(irradianceInfo.z, 0, sp.z); Vector3.FromArrayToRef(irradianceInfo.xx, 0, sp.xx); Vector3.FromArrayToRef(irradianceInfo.yy, 0, sp.yy); Vector3.FromArrayToRef(irradianceInfo.zz, 0, sp.zz); Vector3.FromArrayToRef(irradianceInfo.yz, 0, sp.yz); Vector3.FromArrayToRef(irradianceInfo.zx, 0, sp.zx); Vector3.FromArrayToRef(irradianceInfo.xy, 0, sp.xy); texture._sphericalPolynomial = sp; } /** @hidden */ public static _UpdateRGBDAsync(internalTexture: InternalTexture, data: ArrayBufferView[][], sphericalPolynomial: Nullable, lodScale: number, lodOffset: number): Promise { internalTexture._source = InternalTextureSource.CubeRawRGBD; internalTexture._bufferViewArrayArray = data; internalTexture._lodGenerationScale = lodScale; internalTexture._lodGenerationOffset = lodOffset; internalTexture._sphericalPolynomial = sphericalPolynomial; return EnvironmentTextureTools.UploadLevelsAsync(internalTexture, data).then(() => { internalTexture.isReady = true; }); } } // References the dependencies. InternalTexture._UpdateRGBDAsync = EnvironmentTextureTools._UpdateRGBDAsync;