Babylon.js/src/Cameras/XR/webXRCamera.ts

import { Vector3, Matrix, Quaternion } from "../../Maths/math.vector";
import { Scene } from "../../scene";
import { Camera } from "../../Cameras/camera";
import { FreeCamera } from "../../Cameras/freeCamera";
import { TargetCamera } from "../../Cameras/targetCamera";
import { WebXRSessionManager } from "./webXRSessionManager";
import { Viewport } from '../../Maths/math.viewport';

/**
 * WebXR Camera which holds the views for the xrSession
 * @see https://doc.babylonjs.com/how_to/webxr
 */
export class WebXRCamera extends FreeCamera {

    /**
     * Is the camera in debug mode. Used when using an emulator
     */
    public debugMode = false;

    private _firstFrame = false;
    private _referencedPosition: Vector3 = new Vector3();
    private _referenceQuaternion: Quaternion = Quaternion.Identity();
    private _xrInvPositionCache: Vector3 = new Vector3();
    private _xrInvQuaternionCache = Quaternion.Identity();

    /**
     * Creates a new webXRCamera, this should only be set at the camera after it has been updated by the xrSessionManager
     * @param name the name of the camera
     * @param scene the scene to add the camera to
     */
    constructor(name: string, scene: Scene, private _xrSessionManager: WebXRSessionManager) {
        super(name, Vector3.Zero(), scene);

        // Initial camera configuration
        this.minZ = 0.1;
        this.rotationQuaternion = new Quaternion();
        this.cameraRigMode = Camera.RIG_MODE_CUSTOM;
        this.updateUpVectorFromRotation = true;
        this._updateNumberOfRigCameras(1);

        this._xrSessionManager.onXRSessionInit.add(() => {
            this._referencedPosition.copyFromFloats(0, 0, 0);
            this._referenceQuaternion.copyFromFloats(0, 0, 0, 1);
            // first frame - camera's y position should be 0 for the correct offset
            this._firstFrame = true;
        });

        // Check transformation changes on each frame. Callback is added to be first so that the transformation will be
        // applied to the rest of the elements using the referenceSpace object
        this._xrSessionManager.onXRFrameObservable.add((frame) => {
            if (!this._firstFrame) {
                this._updateReferenceSpace();
            }
            this._updateFromXRSession();
        }, undefined, true);
    }

    private _updateNumberOfRigCameras(viewCount = 1) {
        while (this.rigCameras.length < viewCount) {
            var newCamera = new TargetCamera("view: " + this.rigCameras.length, Vector3.Zero(), this.getScene());
            newCamera.minZ = 0.1;
            newCamera.rotationQuaternion = new Quaternion();
            newCamera.updateUpVectorFromRotation = true;
            this.rigCameras.push(newCamera);
        }
        while (this.rigCameras.length > viewCount) {
            var removedCamera = this.rigCameras.pop();
            if (removedCamera) {
                removedCamera.dispose();
            }
        }
    }

    /** @hidden */
    public _updateForDualEyeDebugging(/*pupilDistance = 0.01*/) {
        // Create initial camera rigs
        this._updateNumberOfRigCameras(2);
        this.rigCameras[0].viewport = new Viewport(0, 0, 0.5, 1.0);
        // this.rigCameras[0].position.x = -pupilDistance / 2;
        this.rigCameras[0].outputRenderTarget = null;
        this.rigCameras[1].viewport = new Viewport(0.5, 0, 0.5, 1.0);
        // this.rigCameras[1].position.x = pupilDistance / 2;
        this.rigCameras[1].outputRenderTarget = null;
    }

    private _updateReferenceSpace(): boolean {
        // were position & rotation updated OUTSIDE of the xr update loop
        if (!this.position.equals(this._referencedPosition) || !this.rotationQuaternion.equals(this._referenceQuaternion)) {
            this.position.subtractToRef(this._referencedPosition, this._referencedPosition);
            this._referenceQuaternion.conjugateInPlace();
            this._referenceQuaternion.multiplyToRef(this.rotationQuaternion, this._referenceQuaternion);
            this._updateReferenceSpaceOffset(this._referencedPosition, this._referenceQuaternion.normalize());
            return true;
        }
        return false;
    }

    private _updateReferenceSpaceOffset(positionOffset: Vector3, rotationOffset?: Quaternion, ignoreHeight: boolean = false) {
        if (!this._xrSessionManager.referenceSpace || !this._xrSessionManager.currentFrame) {
            return;
        }
        // Compute the origin offset based on player position/orientation.
        this._xrInvPositionCache.copyFrom(positionOffset);
        if (rotationOffset) {
            this._xrInvQuaternionCache.copyFrom(rotationOffset);
        } else {
            this._xrInvQuaternionCache.copyFromFloats(0, 0, 0, 1);
        }

        // right handed system
        if (!this._scene.useRightHandedSystem) {
            this._xrInvPositionCache.z *= -1;
            this._xrInvQuaternionCache.z *= -1;
            this._xrInvQuaternionCache.w *= -1;
        }

        this._xrInvPositionCache.negateInPlace();
        this._xrInvQuaternionCache.conjugateInPlace();
        // transform point according to rotation with pivot
        this._xrInvPositionCache.rotateByQuaternionToRef(this._xrInvQuaternionCache, this._xrInvPositionCache);
        if (ignoreHeight) {
            this._xrInvPositionCache.y = 0;
        }
        const transform = new XRRigidTransform(
            { ...this._xrInvPositionCache },
            { ...this._xrInvQuaternionCache });
        // Update offset reference to use a new originOffset with the teleported
        // player position and orientation.
        // This new offset needs to be applied to the base ref space.
        const referenceSpace = this._xrSessionManager.referenceSpace.getOffsetReferenceSpace(transform);

        const pose = this._xrSessionManager.currentFrame && this._xrSessionManager.currentFrame.getViewerPose(referenceSpace);

        if (pose) {
            const pos = new Vector3();
            pos.copyFrom(<any>(pose.transform.position));
            if (!this._scene.useRightHandedSystem) {
                pos.z *= -1;
            }
            this.position.subtractToRef(pos, pos);
            if (!this._scene.useRightHandedSystem) {
                pos.z *= -1;
            }
            pos.negateInPlace();

            const transform2 = new XRRigidTransform(
                { ...pos });
            // Update offset reference to use a new originOffset with the teleported
            // player position and orientation.
            // This new offset needs to be applied to the base ref space.
            this._xrSessionManager.referenceSpace = referenceSpace.getOffsetReferenceSpace(transform2);
        }
    }

    private _updateFromXRSession() {

        const pose = this._xrSessionManager.currentFrame && this._xrSessionManager.currentFrame.getViewerPose(this._xrSessionManager.referenceSpace);

        if (!pose) {
            return;
        }

        if (pose.transform) {
            this._referencedPosition.copyFrom(<any>(pose.transform.position));
            this._referenceQuaternion.copyFrom(<any>(pose.transform.orientation));
            if (!this._scene.useRightHandedSystem) {
                this._referencedPosition.z *= -1;
                this._referenceQuaternion.z *= -1;
                this._referenceQuaternion.w *= -1;
            }

            if (this._firstFrame) {
                this._firstFrame = false;
                // we have the XR reference, now use this to find the offset to get the camera to be
                // in the right position

                // set the height to correlate to the current height
                this.position.y += this._referencedPosition.y;
                // avoid using the head rotation on the first frame.
                this._referenceQuaternion.copyFromFloats(0, 0, 0, 1);
                // update the reference space so that the position will be correct

                return this.update();

            }

            this.rotationQuaternion.copyFrom(this._referenceQuaternion);
            this.position.copyFrom(this._referencedPosition);
        }

        // Update camera rigs
        if (this.rigCameras.length !== pose.views.length) {
            this._updateNumberOfRigCameras(pose.views.length);
        }

        pose.views.forEach((view: any, i: number) => {
            const currentRig = <TargetCamera>this.rigCameras[i];
            // update right and left, where applicable
            if (!currentRig.isLeftCamera && !currentRig.isRightCamera) {
                if (view.eye === 'right') {
                    currentRig._isRightCamera = true;
                } else if (view.eye === 'left') {
                    currentRig._isLeftCamera = true;
                }
            }
            // Update view/projection matrix
            if (view.transform.position) {
                currentRig.position.copyFrom(view.transform.position);
                currentRig.rotationQuaternion.copyFrom(view.transform.orientation);
                if (!this._scene.useRightHandedSystem) {
                    currentRig.position.z *= -1;
                    currentRig.rotationQuaternion.z *= -1;
                    currentRig.rotationQuaternion.w *= -1;
                }
            } else {
                Matrix.FromFloat32ArrayToRefScaled(view.transform.matrix, 0, 1, currentRig._computedViewMatrix);
                if (!this._scene.useRightHandedSystem) {
                    currentRig._computedViewMatrix.toggleModelMatrixHandInPlace();
                }
            }
            Matrix.FromFloat32ArrayToRefScaled(view.projectionMatrix, 0, 1, currentRig._projectionMatrix);

            if (!this._scene.useRightHandedSystem) {
                currentRig._projectionMatrix.toggleProjectionMatrixHandInPlace();
            }

            // Update viewport
            if (this._xrSessionManager.session.renderState.baseLayer) {
                var viewport = this._xrSessionManager.session.renderState.baseLayer.getViewport(view);
                var width = this._xrSessionManager.session.renderState.baseLayer.framebufferWidth;
                var height = this._xrSessionManager.session.renderState.baseLayer.framebufferHeight;
                currentRig.viewport.width = viewport.width / width;
                currentRig.viewport.height = viewport.height / height;
                currentRig.viewport.x = viewport.x / width;
                currentRig.viewport.y = viewport.y / height;
            }
            if (this.debugMode) {
                this._updateForDualEyeDebugging();
            }

            // Set cameras to render to the session's render target
            currentRig.outputRenderTarget = this._xrSessionManager.getRenderTargetTextureForEye(view.eye);
        });
    }
}