import { Vector3, Matrix, Quaternion } from "../../Maths/math.vector"; import { Scene } from "../../scene"; import { Camera } from "../../Cameras/camera"; import { FreeCamera } from "../../Cameras/freeCamera"; import { TargetCamera } from "../../Cameras/targetCamera"; import { WebXRSessionManager } from "./webXRSessionManager"; import { Viewport } from '../../Maths/math.viewport'; /** * WebXR Camera which holds the views for the xrSession * @see https://doc.babylonjs.com/how_to/webxr */ export class WebXRCamera extends FreeCamera { /** * Is the camera in debug mode. Used when using an emulator */ public debugMode = false; private _firstFrame = false; private _referencedPosition: Vector3 = new Vector3(); private _referenceQuaternion: Quaternion = Quaternion.Identity(); private _xrInvPositionCache: Vector3 = new Vector3(); private _xrInvQuaternionCache = Quaternion.Identity(); /** * Creates a new webXRCamera, this should only be set at the camera after it has been updated by the xrSessionManager * @param name the name of the camera * @param scene the scene to add the camera to */ constructor(name: string, scene: Scene, private _xrSessionManager: WebXRSessionManager) { super(name, Vector3.Zero(), scene); // Initial camera configuration this.minZ = 0.1; this.rotationQuaternion = new Quaternion(); this.cameraRigMode = Camera.RIG_MODE_CUSTOM; this.updateUpVectorFromRotation = true; this._updateNumberOfRigCameras(1); this._xrSessionManager.onXRSessionInit.add(() => { this._referencedPosition.copyFromFloats(0, 0, 0); this._referenceQuaternion.copyFromFloats(0, 0, 0, 1); // first frame - camera's y position should be 0 for the correct offset this._firstFrame = true; }); // Check transformation changes on each frame. Callback is added to be first so that the transformation will be // applied to the rest of the elements using the referenceSpace object this._xrSessionManager.onXRFrameObservable.add((frame) => { if (!this._firstFrame) { this._updateReferenceSpace(); } this._updateFromXRSession(); }, undefined, true); } private _updateNumberOfRigCameras(viewCount = 1) { while (this.rigCameras.length < viewCount) { var newCamera = new TargetCamera("view: " + this.rigCameras.length, Vector3.Zero(), this.getScene()); newCamera.minZ = 0.1; newCamera.rotationQuaternion = new Quaternion(); newCamera.updateUpVectorFromRotation = true; this.rigCameras.push(newCamera); } while (this.rigCameras.length > viewCount) { var removedCamera = this.rigCameras.pop(); if (removedCamera) { removedCamera.dispose(); } } } /** @hidden */ public _updateForDualEyeDebugging(/*pupilDistance = 0.01*/) { // Create initial camera rigs this._updateNumberOfRigCameras(2); this.rigCameras[0].viewport = new Viewport(0, 0, 0.5, 1.0); // this.rigCameras[0].position.x = -pupilDistance / 2; this.rigCameras[0].outputRenderTarget = null; this.rigCameras[1].viewport = new Viewport(0.5, 0, 0.5, 1.0); // this.rigCameras[1].position.x = pupilDistance / 2; this.rigCameras[1].outputRenderTarget = null; } private _updateReferenceSpace(): boolean { // were position & rotation updated OUTSIDE of the xr update loop if (!this.position.equals(this._referencedPosition) || !this.rotationQuaternion.equals(this._referenceQuaternion)) { this.position.subtractToRef(this._referencedPosition, this._referencedPosition); this._referenceQuaternion.conjugateInPlace(); this._referenceQuaternion.multiplyToRef(this.rotationQuaternion, this._referenceQuaternion); this._updateReferenceSpaceOffset(this._referencedPosition, this._referenceQuaternion.normalize()); return true; } return false; } private _updateReferenceSpaceOffset(positionOffset: Vector3, rotationOffset?: Quaternion, ignoreHeight: boolean = false) { if (!this._xrSessionManager.referenceSpace || !this._xrSessionManager.currentFrame) { return; } // Compute the origin offset based on player position/orientation. this._xrInvPositionCache.copyFrom(positionOffset); if (rotationOffset) { this._xrInvQuaternionCache.copyFrom(rotationOffset); } else { this._xrInvQuaternionCache.copyFromFloats(0, 0, 0, 1); } // right handed system if (!this._scene.useRightHandedSystem) { this._xrInvPositionCache.z *= -1; this._xrInvQuaternionCache.z *= -1; this._xrInvQuaternionCache.w *= -1; } this._xrInvPositionCache.negateInPlace(); this._xrInvQuaternionCache.conjugateInPlace(); // transform point according to rotation with pivot this._xrInvPositionCache.rotateByQuaternionToRef(this._xrInvQuaternionCache, this._xrInvPositionCache); if (ignoreHeight) { this._xrInvPositionCache.y = 0; } const transform = new XRRigidTransform( { ...this._xrInvPositionCache }, { ...this._xrInvQuaternionCache }); // Update offset reference to use a new originOffset with the teleported // player position and orientation. // This new offset needs to be applied to the base ref space. const referenceSpace = this._xrSessionManager.referenceSpace.getOffsetReferenceSpace(transform); const pose = this._xrSessionManager.currentFrame && this._xrSessionManager.currentFrame.getViewerPose(referenceSpace); if (pose) { const pos = new Vector3(); pos.copyFrom((pose.transform.position)); if (!this._scene.useRightHandedSystem) { pos.z *= -1; } this.position.subtractToRef(pos, pos); if (!this._scene.useRightHandedSystem) { pos.z *= -1; } pos.negateInPlace(); const transform2 = new XRRigidTransform( { ...pos }); // Update offset reference to use a new originOffset with the teleported // player position and orientation. // This new offset needs to be applied to the base ref space. this._xrSessionManager.referenceSpace = referenceSpace.getOffsetReferenceSpace(transform2); } } private _updateFromXRSession() { const pose = this._xrSessionManager.currentFrame && this._xrSessionManager.currentFrame.getViewerPose(this._xrSessionManager.referenceSpace); if (!pose) { return; } if (pose.transform) { this._referencedPosition.copyFrom((pose.transform.position)); this._referenceQuaternion.copyFrom((pose.transform.orientation)); if (!this._scene.useRightHandedSystem) { this._referencedPosition.z *= -1; this._referenceQuaternion.z *= -1; this._referenceQuaternion.w *= -1; } if (this._firstFrame) { this._firstFrame = false; // we have the XR reference, now use this to find the offset to get the camera to be // in the right position // set the height to correlate to the current height this.position.y += this._referencedPosition.y; // avoid using the head rotation on the first frame. this._referenceQuaternion.copyFromFloats(0, 0, 0, 1); // update the reference space so that the position will be correct return this.update(); } this.rotationQuaternion.copyFrom(this._referenceQuaternion); this.position.copyFrom(this._referencedPosition); } // Update camera rigs if (this.rigCameras.length !== pose.views.length) { this._updateNumberOfRigCameras(pose.views.length); } pose.views.forEach((view: any, i: number) => { const currentRig = this.rigCameras[i]; // update right and left, where applicable if (!currentRig.isLeftCamera && !currentRig.isRightCamera) { if (view.eye === 'right') { currentRig._isRightCamera = true; } else if (view.eye === 'left') { currentRig._isLeftCamera = true; } } // Update view/projection matrix if (view.transform.position) { currentRig.position.copyFrom(view.transform.position); currentRig.rotationQuaternion.copyFrom(view.transform.orientation); if (!this._scene.useRightHandedSystem) { currentRig.position.z *= -1; currentRig.rotationQuaternion.z *= -1; currentRig.rotationQuaternion.w *= -1; } } else { Matrix.FromFloat32ArrayToRefScaled(view.transform.matrix, 0, 1, currentRig._computedViewMatrix); if (!this._scene.useRightHandedSystem) { currentRig._computedViewMatrix.toggleModelMatrixHandInPlace(); } } Matrix.FromFloat32ArrayToRefScaled(view.projectionMatrix, 0, 1, currentRig._projectionMatrix); if (!this._scene.useRightHandedSystem) { currentRig._projectionMatrix.toggleProjectionMatrixHandInPlace(); } // Update viewport if (this._xrSessionManager.session.renderState.baseLayer) { var viewport = this._xrSessionManager.session.renderState.baseLayer.getViewport(view); var width = this._xrSessionManager.session.renderState.baseLayer.framebufferWidth; var height = this._xrSessionManager.session.renderState.baseLayer.framebufferHeight; currentRig.viewport.width = viewport.width / width; currentRig.viewport.height = viewport.height / height; currentRig.viewport.x = viewport.x / width; currentRig.viewport.y = viewport.y / height; } if (this.debugMode) { this._updateForDualEyeDebugging(); } // Set cameras to render to the session's render target currentRig.outputRenderTarget = this._xrSessionManager.getRenderTargetTextureForEye(view.eye); }); } }