123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443 |
- <!DOCTYPE html>
- <html lang="en">
- <head>
- <title>three.js vr - layers</title>
- <meta charset="utf-8">
- <meta name="viewport" content="width=device-width, initial-scale=1.0, user-scalable=no">
- <link type="text/css" rel="stylesheet" href="main.css">
- </head>
- <body>
- <div id="info">
- <a href="https://threejs.org" target="_blank" rel="noopener">three.js</a> media and projection layers<br/>
- (Oculus Browser 16.1+)
- <p> This example demonstrates the use of <a href="https://www.w3.org/TR/webxrlayers-1/">WebXR Layers</a> to render high quality text and video.
- For static content such as text, using layers increases the usable resolution of the content by avoiding the extra resampling pass that occurs during normal VR rendering.
- For dynamic content such as video, using layers also improves performance by only copying data when new frames are available. </p>
- <br />
- <p><i>See the comments in the code for more information.</i></p>
- </div>
- <script type="importmap">
- {
- "imports": {
- "three": "../build/three.module.js",
- "three/addons/": "./jsm/"
- }
- }
- </script>
- <script type="module">
- import * as THREE from 'three';
- import { GUI } from 'three/addons/libs/lil-gui.module.min.js';
- import { HTMLMesh } from 'three/addons/interactive/HTMLMesh.js';
- import { InteractiveGroup } from 'three/addons/interactive/InteractiveGroup.js';
- import { VRButton } from 'three/addons/webxr/VRButton.js';
- import { XRControllerModelFactory } from 'three/addons/webxr/XRControllerModelFactory.js';
- import { XRHandModelFactory } from 'three/addons/webxr/XRHandModelFactory.js';
- let camera, scene, renderer;
- let video;
- // Four eye charts are rendered to demonstrate the differences in text quality.
- // The two charts on the top are rendered into XRQuadLayers ( substantially more legible ).
- // The two charts on the bottom are rendered to the eye buffer.
- //
- // The two charts on the left are rendered without mipmaps and have aliasing artifacts.
- // The two charts on the right are with mipmaps an don't twinkle but are blurrier.
- // To maximize text legibility, it's important to choose a texture size optimized for the
- // distance of the text. (This example intentionally uses incorrectly large textures to
- // demonstrate this issue.) If the optimal text size can't be determined beforehand, then
- // mipmaps are required to avoid aliasing.
- //
- // The background of the scene is an equirectangular layer. It uses an XRMediaBinding to
- // render the contents of a video element into the scene. This example uses a low resolution
- // video to avoid large files, but using media layers allows video that is higher resolution than normal rendering.
- let snellenTexture;
- let quadLayerPlain;
- let quadLayerMips;
- let guiLayer;
- let guiMesh;
- let errorMesh;
- // Set via GUI.
- const parameters = {
- eyeChartDistanceFt: 20,
- };
- // Data shared between the THREE.Meshes on the bottom and WebXR Layers on the top for the eye
- // charts. See https://en.wikipedia.org/wiki/Snellen_chart for details about the math.
- //
- // The image was designed so that each 2x2px block on the 20/20 line subtends 1 minute of
- // arc. That is
- // tan(1/60 deg) * 6.1m * 160px/142mm = 2px
- // per block on line 8.
- //
- // This fidelity is beyond any modern consumer headset since it would require ~60px/deg of
- // resolution. The Quest has ~16ppd and the Quest 2 has ~20ppd so only lines 3 or 4 will be
- // legible when using layers. Without layers, you lose ~sqrt(2) in resolution due to the
- // extra resampling.
- const snellenConfig = {
- // The texture is a power of two so that mipmaps can be generated.
- textureSizePx: 512,
- // This is the valid part of the image.
- widthPx: 320,
- heightPx: 450,
- x: 0,
- y: 1.5,
- z: - 6.1, // 20/20 vision @ 20ft = 6.1m
- // This is the size of mesh and the visible part of the quad layer.
- widthMeters: .268, // 320px image * (142mm/160px scale factor)
- heightMeters: .382 // 450px image * (142mm/160px scale factor)
- };
- snellenConfig.cropX = snellenConfig.widthPx / snellenConfig.textureSizePx;
- snellenConfig.cropY = snellenConfig.heightPx / snellenConfig.textureSizePx;
- // The quad layer is a [-1, 1] quad but only a part of it has image data. Scale the layer so
- // that the part with image data is the same size as the mesh.
- snellenConfig.quadWidth = .5 * snellenConfig.widthMeters / snellenConfig.cropX;
- snellenConfig.quadHeight = .5 * snellenConfig.heightMeters / snellenConfig.cropY;
- init();
- function init() {
- scene = new THREE.Scene();
- camera = new THREE.PerspectiveCamera( 50, window.innerWidth / window.innerHeight, 0.1, 10 );
- camera.position.set( 0, 1.6, 3 );
- const hemLight = new THREE.HemisphereLight( 0x808080, 0x606060, 3 );
- const light = new THREE.DirectionalLight( 0xffffff, 3 );
- scene.add( hemLight, light );
- //
- renderer = new THREE.WebGLRenderer( { antialias: false } );
- renderer.setPixelRatio( window.devicePixelRatio );
- renderer.setSize( window.innerWidth, window.innerHeight );
- renderer.setAnimationLoop( animate );
- renderer.setClearAlpha( 1 );
- renderer.setClearColor( new THREE.Color( 0 ), 0 );
- renderer.xr.enabled = true;
- document.body.appendChild( renderer.domElement );
- document.body.appendChild( VRButton.createButton( renderer ) );
- // controllers
- const lineGeometry = new THREE.BufferGeometry().setFromPoints( [
- new THREE.Vector3( 0, 0, 0 ),
- new THREE.Vector3( 0, 0, - 10 )
- ] );
- const line = new THREE.Line( lineGeometry, new THREE.LineBasicMaterial( { color: 0x5555ff } ) );
- // The invisible dummyMesh quads and the guiMesh need to be rendered before the controller lines so that they
- // leave a hole in the depth buffer that the lines can intersect.
- line.renderOrder = 1;
- const controllerModelFactory = new XRControllerModelFactory();
- const handModelFactory = new XRHandModelFactory().setPath( './models/fbx/' );
- //
- const controllers = [
- renderer.xr.getController( 0 ),
- renderer.xr.getController( 1 )
- ];
- controllers.forEach( ( controller, i ) => {
- const controllerGrip = renderer.xr.getControllerGrip( i );
- controllerGrip.add( controllerModelFactory.createControllerModel( controllerGrip ) );
- scene.add( controllerGrip );
- const hand = renderer.xr.getHand( i );
- hand.add( handModelFactory.createHandModel( hand ) );
- controller.add( line.clone() );
- scene.add( controller, controllerGrip, hand );
- } );
- // Eye charts
- const eyeCharts = new THREE.Group();
- eyeCharts.position.z = snellenConfig.z;
- scene.add( eyeCharts );
- snellenTexture = new THREE.TextureLoader().load( 'textures/snellen.png' );
- snellenTexture.repeat.x = snellenConfig.cropX;
- snellenTexture.repeat.y = snellenConfig.cropY;
- snellenTexture.generateMipmaps = false;
- snellenTexture.minFilter = THREE.LinearFilter;
- const snellenMeshPlain = new THREE.Mesh(
- new THREE.PlaneGeometry( snellenConfig.widthMeters, snellenConfig.heightMeters ),
- new THREE.MeshBasicMaterial( { map: snellenTexture } ) );
- snellenMeshPlain.position.x = snellenConfig.x - snellenConfig.widthMeters;
- snellenMeshPlain.position.y = snellenConfig.y - snellenConfig.heightMeters;
- eyeCharts.add( snellenMeshPlain );
- snellenTexture = new THREE.TextureLoader().load( 'textures/snellen.png' );
- snellenTexture.repeat.x = snellenConfig.cropX;
- snellenTexture.repeat.y = snellenConfig.cropY;
- const snellenMeshMipMap = new THREE.Mesh(
- new THREE.PlaneGeometry( snellenConfig.widthMeters, snellenConfig.heightMeters ),
- new THREE.MeshBasicMaterial( { map: snellenTexture } ) );
- snellenMeshMipMap.position.x = snellenConfig.x + snellenConfig.widthMeters;
- snellenMeshMipMap.position.y = snellenConfig.y - snellenConfig.heightMeters;
- eyeCharts.add( snellenMeshMipMap );
- // The layers don't participate depth testing between each other. Since the projection
- // layer is rendered last, any 3D object will incorrecly overlap layers. To avoid this,
- // invisible quads can be placed into the scene to participate in depth testing when the
- // projection layer is rendered.
- const dummyMeshLeft = new THREE.Mesh(
- new THREE.PlaneGeometry( snellenConfig.widthMeters, snellenConfig.heightMeters ),
- new THREE.MeshBasicMaterial( { colorWrite: false } ) );
- dummyMeshLeft.position.x = snellenConfig.x - snellenConfig.widthMeters;
- dummyMeshLeft.position.y = snellenConfig.y + snellenConfig.heightMeters;
- eyeCharts.add( dummyMeshLeft );
- const dummyMeshRight = dummyMeshLeft.clone( true );
- dummyMeshRight.position.x = snellenConfig.x + snellenConfig.widthMeters;
- eyeCharts.add( dummyMeshRight );
- // The GUI is rendered into an invisible HTMLMesh and the backing canvas's data is copied
- // into a layer as required. Hit testing and interaction is done using standard HTMLMesh
- // behavior, but since the layer is in the same place as the invisible mesh, the user
- // thinks they're directly interacting with the layer.
- const gui = new GUI( { width: 300 } );
- gui.add( parameters, 'eyeChartDistanceFt', 1.0, 20.0 ).onChange( onChange );
- gui.domElement.style.visibility = 'hidden';
- function onChange() {
- eyeCharts.position.z = - parameters.eyeChartDistanceFt * 0.3048;
- snellenConfig.z = eyeCharts.position.z;
- if ( quadLayerPlain ) {
- quadLayerPlain.transform = new XRRigidTransform( {
- x: snellenConfig.x - snellenConfig.widthMeters,
- y: snellenConfig.y + snellenConfig.heightMeters,
- z: eyeCharts.position.z
- } );
- }
- if ( quadLayerMips ) {
- quadLayerMips.transform = new XRRigidTransform( {
- x: snellenConfig.x + snellenConfig.widthMeters,
- y: snellenConfig.y + snellenConfig.heightMeters,
- z: eyeCharts.position.z
- } );
- }
- guiLayer.needsUpdate = true;
- }
- const group = new InteractiveGroup();
- group.listenToXRControllerEvents( controllers[ 0 ] );
- group.listenToXRControllerEvents( controllers[ 1 ] );
- scene.add( group );
- guiMesh = new HTMLMesh( gui.domElement );
- guiMesh.position.set( 1.0, 1.5, - 1.0 );
- guiMesh.rotation.y = - Math.PI / 4;
- guiMesh.scale.setScalar( 2 );
- guiMesh.material.colorWrite = false;
- guiMesh.material.transparent = false;
- group.add( guiMesh );
- // Error message if layer initialization fails.
- const errorCanvas = document.createElement( 'canvas' );
- errorCanvas.width = 400;
- errorCanvas.height = 40;
- const errorContext = errorCanvas.getContext( '2d' );
- errorContext.fillStyle = '#FF0000';
- errorContext.fillRect( 0, 0, errorCanvas.width, errorCanvas.height );
- errorContext.fillStyle = '#000000';
- errorContext.font = '28px sans-serif';
- errorContext.fillText( 'ERROR: Layers not initialized!', 10, 30 );
- errorMesh = new THREE.Mesh(
- new THREE.PlaneGeometry( 1, .1 ),
- new THREE.MeshBasicMaterial( { map: new THREE.CanvasTexture( errorCanvas ) } )
- );
- errorMesh.position.z = - 1;
- errorMesh.position.y = 1.5;
- errorMesh.visible = false;
- scene.add( errorMesh );
- window.addEventListener( 'resize', onWindowResize, false );
- video = document.createElement( 'video' );
- video.loop = true;
- video.src = 'textures/MaryOculus.webm';
- }
- function onWindowResize() {
- camera.aspect = window.innerWidth / window.innerHeight;
- camera.updateProjectionMatrix();
- renderer.setSize( window.innerWidth, window.innerHeight );
- }
- //
- function animate( t, frame ) {
- const xr = renderer.xr;
- const session = xr.getSession();
- const gl = renderer.getContext();
- // Init layers once in immersive mode and video is ready.
- if ( session && session.renderState.layers === undefined ) {
- errorMesh.visible = true;
- }
- if (
- session &&
- session.renderState.layers !== undefined &&
- session.hasMediaLayer === undefined &&
- video.readyState >= 2
- ) {
- session.hasMediaLayer = true;
- session.requestReferenceSpace( 'local-floor' ).then( ( refSpace ) => {
- // Create Quad layers for Snellen chart.
- const glBinding = xr.getBinding();
- const quadLayerConfig = {
- width: snellenConfig.quadWidth,
- height: snellenConfig.quadHeight,
- viewPixelWidth: snellenConfig.textureSizePx,
- viewPixelHeight: snellenConfig.textureSizePx,
- isStatic: true,
- space: refSpace,
- layout: 'mono',
- transform: new XRRigidTransform( {
- x: snellenConfig.x - snellenConfig.widthMeters,
- y: snellenConfig.y + snellenConfig.heightMeters,
- z: snellenConfig.z
- } )
- };
- quadLayerPlain = glBinding.createQuadLayer( quadLayerConfig );
- quadLayerConfig.mipLevels = 3;
- quadLayerConfig.transform = new XRRigidTransform( {
- x: snellenConfig.x + snellenConfig.widthMeters,
- y: snellenConfig.y + snellenConfig.heightMeters,
- z: snellenConfig.z
- } );
- quadLayerMips = glBinding.createQuadLayer( quadLayerConfig );
- // Create GUI layer.
- guiLayer = glBinding.createQuadLayer( {
- width: guiMesh.geometry.parameters.width,
- height: guiMesh.geometry.parameters.height,
- viewPixelWidth: guiMesh.material.map.image.width,
- viewPixelHeight: guiMesh.material.map.image.height,
- space: refSpace,
- transform: new XRRigidTransform( guiMesh.position, guiMesh.quaternion )
- } );
- // Create background EQR video layer.
- const mediaBinding = new XRMediaBinding( session );
- const equirectLayer = mediaBinding.createEquirectLayer(
- video,
- {
- space: refSpace,
- layout: 'stereo-left-right',
- // Rotate by 45 deg to avoid stereo conflict with the 3D geometry.
- transform: new XRRigidTransform(
- {},
- { x: 0, y: .28, z: 0, w: .96 }
- )
- }
- );
- errorMesh.visible = false;
- session.updateRenderState( {
- layers: [
- equirectLayer,
- quadLayerPlain,
- quadLayerMips,
- guiLayer,
- session.renderState.layers[ 0 ]
- ]
- } );
- video.play();
- } );
- }
- // Copy image to layers as required.
- // needsRedraw is set on creation or if the underlying GL resources of a layer are lost.
- if ( session && quadLayerPlain && quadLayerPlain.needsRedraw ) {
- const glayer = xr.getBinding().getSubImage( quadLayerPlain, frame );
- renderer.state.bindTexture( gl.TEXTURE_2D, glayer.colorTexture );
- gl.pixelStorei( gl.UNPACK_FLIP_Y_WEBGL, true );
- gl.texSubImage2D( gl.TEXTURE_2D, 0,
- ( snellenConfig.textureSizePx - snellenConfig.widthPx ) / 2,
- ( snellenConfig.textureSizePx - snellenConfig.heightPx ) / 2,
- snellenConfig.widthPx, snellenConfig.heightPx,
- gl.RGBA, gl.UNSIGNED_BYTE, snellenTexture.image );
- }
- // Same as above but also gl.generateMipmap.
- if ( session && quadLayerMips && quadLayerMips.needsRedraw ) {
- const glayer = xr.getBinding().getSubImage( quadLayerMips, frame );
- renderer.state.bindTexture( gl.TEXTURE_2D, glayer.colorTexture );
- gl.pixelStorei( gl.UNPACK_FLIP_Y_WEBGL, true );
- gl.texSubImage2D( gl.TEXTURE_2D, 0,
- ( snellenConfig.textureSizePx - snellenConfig.widthPx ) / 2,
- ( snellenConfig.textureSizePx - snellenConfig.heightPx ) / 2,
- snellenConfig.widthPx, snellenConfig.heightPx,
- gl.RGBA, gl.UNSIGNED_BYTE, snellenTexture.image );
- gl.generateMipmap( gl.TEXTURE_2D );
- }
- // Same as above, but guiLayer.needsUpdate is set when the user interacts with the GUI.
- if ( session && guiLayer && ( guiLayer.needsRedraw || guiLayer.needsUpdate ) ) {
- const glayer = xr.getBinding().getSubImage( guiLayer, frame );
- renderer.state.bindTexture( gl.TEXTURE_2D, glayer.colorTexture );
- gl.pixelStorei( gl.UNPACK_FLIP_Y_WEBGL, true );
- const canvas = guiMesh.material.map.image;
- gl.texSubImage2D( gl.TEXTURE_2D, 0, 0, 0, canvas.width, canvas.height, gl.RGBA, gl.UNSIGNED_BYTE, canvas );
- guiLayer.needsUpdate = false;
- }
- renderer.render( scene, camera );
- }
- </script>
- </body>
- </html>
|