From dad53f7f5d77aca6f1aa78597652ca762f5b1c29 Mon Sep 17 00:00:00 2001 From: Mario Buikhuizen Date: Mon, 1 Nov 2021 13:33:26 +0100 Subject: [PATCH 01/13] WIP: AR support --- js/package-lock.json | 14 ++- js/package.json | 2 +- js/src/ar.js | 203 +++++++++++++++++++++++++++++++++++++++++++ js/src/figure.ts | 72 +++++++++++---- notebooks/AR.ipynb | 148 +++++++++++++++++++++++++++++++ 5 files changed, 420 insertions(+), 19 deletions(-) create mode 100644 js/src/ar.js create mode 100644 notebooks/AR.ipynb diff --git a/js/package-lock.json b/js/package-lock.json index 6d717128..f2923ce0 100644 --- a/js/package-lock.json +++ b/js/package-lock.json @@ -7061,6 +7061,14 @@ "jupyter-dataserializers": "^2.2.0", "three": "^0.97.0", "underscore": "^1.8.3" + }, + "dependencies": { + "three": { + "version": "0.97.0", + "resolved": "https://registry.npmjs.org/three/-/three-0.97.0.tgz", + "integrity": "sha512-ctZF79O1R2aMIDnz9cV5GUIONyFnYvfQKg4+EAwEVaEr1mgy99rglstH6hhRdIThu3SOa4Ns5da/Ee5fTbWc9A==", + "dev": true + } } }, "karma": { @@ -11090,9 +11098,9 @@ "dev": true }, "three": { - "version": "0.97.0", - "resolved": "https://registry.npmjs.org/three/-/three-0.97.0.tgz", - "integrity": "sha512-ctZF79O1R2aMIDnz9cV5GUIONyFnYvfQKg4+EAwEVaEr1mgy99rglstH6hhRdIThu3SOa4Ns5da/Ee5fTbWc9A==" + "version": "0.112.1", + "resolved": "https://registry.npmjs.org/three/-/three-0.112.1.tgz", + "integrity": "sha512-8I0O74hiYtKl3LgDNcPJbBGOlpekbcJ6fJnImmW3mFdeUFJ2H9Y3/UuUSW2sBdjrIlCM0gvOkaTEFlofO900TQ==" }, "three-text2d": { "version": "0.4.1", diff --git a/js/package.json b/js/package.json index 615c2092..ae03eae9 100644 --- a/js/package.json +++ b/js/package.json @@ -91,7 +91,7 @@ "ndarray-pack": "^1.2.1", "screenfull": "^3.3.1", "style-loader": "^0.18.2", - "three": "^0.97.0", + "three": "^0.112.1", "three-text2d": "~0.4.1", "tslint": "^5.20.0", "underscore": "^1.8.3", diff --git a/js/src/ar.js b/js/src/ar.js new file mode 100644 index 00000000..e5f2e517 --- /dev/null +++ b/js/src/ar.js @@ -0,0 +1,203 @@ +import { WebXRManager } from 'three' + +export function createButton( renderer, onStart, onEnd, sessionInit = {} ) { + + const button = document.createElement( 'button' ); + console.log('__renderer:', renderer) + + function showStartAR( /*device*/ ) { + + if ( sessionInit.domOverlay === undefined ) { + + var overlay = document.createElement( 'div' ); + overlay.id = 'aroverlay'; + overlay.style.display = 'none'; + document.body.appendChild( overlay ); + // document.document.getElementById('bliep').appendChild( overlay ); + + var svg = document.createElementNS( 'http://www.w3.org/2000/svg', 'svg' ); + svg.setAttribute( 'width', 38 ); + svg.setAttribute( 'height', 38 ); + svg.style.position = 'absolute'; + svg.style.right = '20px'; + svg.style.top = '20px'; + svg.addEventListener( 'click', function () { + onEnd(); + console.log('end'); + currentSession.end(); + + } ); + overlay.appendChild( svg ); + + var path = document.createElementNS( 'http://www.w3.org/2000/svg', 'path' ); + path.setAttribute( 'd', 'M 12,12 L 28,28 M 28,12 12,28' ); + path.setAttribute( 'stroke', '#fff' ); + path.setAttribute( 'stroke-width', 2 ); + svg.appendChild( path ); + + if ( sessionInit.optionalFeatures === undefined ) { + + sessionInit.optionalFeatures = []; + + } + + sessionInit.optionalFeatures.push( 'dom-overlay' ); + sessionInit.domOverlay = { root: overlay }; + + } + + // + + let currentSession = null; + + async function onSessionStarted( session ) { + session.addEventListener( 'end', onSessionEnded ); + console.log('rr', renderer); + console.log('start session', renderer.xr) + renderer.xr.setReferenceSpaceType( 'local' ); + console.log('b4 set session'); + await renderer.xr.setSession( session ); + renderer.setSize( window.innerWidth, window.innerHeight ); + console.log('after set session'); + button.textContent = 'STOP AR'; + sessionInit.domOverlay.root.style.display = ''; + sessionInit.domOverlay.root.style.display = ''; + + currentSession = session; + + } + + function onSessionEnded( /*event*/ ) { + + currentSession.removeEventListener( 'end', onSessionEnded ); + + button.textContent = 'START AR'; + sessionInit.domOverlay.root.style.display = 'none'; + + currentSession = null; + + } + + // + + button.style.display = ''; + button.id = 'arbtn'; + + button.style.cursor = 'pointer'; + button.style.left = 'calc(50% - 50px)'; + button.style.width = '100px'; + + button.textContent = 'START AR'; + + button.onmouseenter = function () { + + button.style.opacity = '1.0'; + + }; + + button.onmouseleave = function () { + + button.style.opacity = '0.5'; + + }; + + button.onclick = function () { + console.log('current', currentSession); + if ( currentSession === null ) { + console.log('requesting session') + onStart(); + navigator.xr.requestSession( 'immersive-ar', sessionInit ).then( onSessionStarted ); + + } else { + + currentSession.end(); + + } + + }; + + } + + function disableButton() { + + button.style.display = ''; + + button.style.cursor = 'auto'; + button.style.left = 'calc(50% - 75px)'; + button.style.width = '150px'; + + button.onmouseenter = null; + button.onmouseleave = null; + + button.onclick = null; + + } + + function showARNotSupported() { + console.log('ar not supported'); + disableButton(); + + button.textContent = 'AR NOT SUPPORTED'; + + } + + function stylizeElement( element ) { + + element.style.position = 'absolute'; + element.style.bottom = '20px'; + element.style.padding = '12px 6px'; + element.style.border = '1px solid #fff'; + element.style.borderRadius = '4px'; + element.style.background = 'rgba(0,0,0,0.1)'; + element.style.color = '#fff'; + element.style.font = 'normal 13px sans-serif'; + element.style.textAlign = 'center'; + element.style.opacity = '0.5'; + element.style.outline = 'none'; + element.style.zIndex = '999'; + + } + console.log('xr:', 'xr' in navigator); + if ( 'xr' in navigator ) { + + button.id = 'ARButton'; + // button.style.display = 'none'; + + stylizeElement( button ); + + navigator.xr.isSessionSupported( 'immersive-ar' ).then( function ( supported ) { + console.log('supported!:', supported) + console.log('context', renderer.getContext()); + supported ? showStartAR() : showARNotSupported(); + + } ).catch( showARNotSupported ); + + return button; + + } else { + + const message = document.createElement( 'a' ); + + if ( window.isSecureContext === false ) { + + message.href = document.location.href.replace( /^http:/, 'https:' ); + message.innerHTML = 'WEBXR NEEDS HTTPS'; // TODO Improve message + + } else { + + message.href = 'https://immersiveweb.dev/'; + message.innerHTML = 'WEBXR NOT AVAILABLE'; + + } + + message.style.left = 'calc(50% - 90px)'; + message.style.width = '180px'; + message.style.textDecoration = 'none'; + + stylizeElement( message ); + + return message; + + } + +} diff --git a/js/src/figure.ts b/js/src/figure.ts index e5a4765f..8a548c00 100644 --- a/js/src/figure.ts +++ b/js/src/figure.ts @@ -32,6 +32,7 @@ import "./three/OrbitControls.js"; import "./three/StereoEffect.js"; import "./three/THREEx.FullScreen.js"; import "./three/TrackballControls.js"; +import {createButton} from "./ar"; const shaders = { "screen-fragment": (require("raw-loader!../glsl/screen-fragment.glsl") as any).default, @@ -260,6 +261,7 @@ class FigureView extends widgets.DOMWidgetView { id_pass_target: THREE.WebGLRenderTarget; lastId: number; _wantsPopup: boolean; + controller: any = null; // rendered to get the front coordinate front_box_mesh: THREE.Mesh; @@ -711,9 +713,41 @@ class FigureView extends widgets.DOMWidgetView { // the threejs animation system looks at the parent of the camera and sends rerender msg'es // this.shared_scene.add(this.camera); + const geometry = new THREE.BoxGeometry(0.1, 0.1, 0.1); + const material = new THREE.MeshBasicMaterial( { color: 0x00ff00 } ); + const cube = new THREE.Mesh( geometry, material ); + cube.position.set( 0, 0, -3 ); + this.scene.add( cube ); + + // this.renderer.render( this.scene, this.camera ); + + const onSelect = () => { + console.log('controller world:', this.controller.matrixWorld); + cube.position.set( 0, 0, -3 ).applyMatrix4( this.controller.matrixWorld ) + cube.quaternion.setFromRotationMatrix( this.controller.matrixWorld ); + this.rootObject.position.set( 0, 0, -3 ).applyMatrix4( this.controller.matrixWorld ); + this.rootObject.quaternion.setFromRotationMatrix( this.controller.matrixWorld ); + // update_box(); + } + + this.controller = this.renderer.xr.getController( 0 ); + this.controller.addEventListener( 'select', onSelect ); + this.scene.add( this.controller ); + + this.renderer.setAnimationLoop( () => { + cube.rotation.x += 0.01; + cube.rotation.y += 0.01; + // this.renderer.render( this.scene, this.camera ); + this._real_update(); + }); + // this.scene_scatter = new THREE.Scene(); this.scene_opaque = new THREE.Scene(); + this.rootObject.add(this.wire_box); + this.rootObject.add(this.axes); + this.rootObject.add( this.controller ); + this.scene_opaque.add(this.wire_box); this.scene_opaque.add(this.axes); @@ -729,7 +763,8 @@ class FigureView extends widgets.DOMWidgetView { this.scene_opaque.position.copy(box_position); this.rootObject.scale.copy(box_scale); - this.rootObject.position.copy(box_position); + // this.rootObject.position.copy(box_position); + this.rootObject.position.set( 0, 0, -3 ) this.update(); } this.model.on("change:box_center change:box_size", update_box); @@ -909,7 +944,7 @@ class FigureView extends widgets.DOMWidgetView { sync_controls_external(); }); - window.addEventListener("deviceorientation", this.on_orientationchange.bind(this), false); + // window.addEventListener("deviceorientation", this.on_orientationchange.bind(this), false); const render_size = this.getRenderSize(); @@ -1137,6 +1172,13 @@ class FigureView extends widgets.DOMWidgetView { this.renderer.domElement.onmouseleave = () => { this.hover = false; }; + + this.renderer.xr.enabled = true; + document.getElementById('bliep').appendChild(createButton(this.renderer, + () => {this.model.set("render_continuous", true); this.model.set("camera_control", "xr")}, + () => this.model.set("render_continuous", false) + /*{domOverlay: document.getElementById('bliep')}*/)); + } camera_initial(camera_initial: any) { throw new Error("Method not implemented."); @@ -2357,7 +2399,7 @@ class FigureView extends widgets.DOMWidgetView { (this.front_box_mesh.material as THREE.ShaderMaterial).side = THREE.FrontSide; this.renderer.setRenderTarget(this.volume_front_target); this.renderer.clear(true, true, true); - this.renderer.render(this.scene, camera, this.volume_front_target); + this.renderer.render(this.scene, camera); this.front_box_mesh.visible = false; @@ -2375,7 +2417,7 @@ class FigureView extends widgets.DOMWidgetView { this.renderer.setRenderTarget(this.volume_back_target); this.renderer.clear(true, true, true); setVisible({volumes: true}); - this.renderer.render(this.scene, camera, this.volume_back_target); + this.renderer.render(this.scene, camera); this.renderer.state.buffers.depth.setClear(1); // Color and depth render pass for volume rendering @@ -2383,18 +2425,18 @@ class FigureView extends widgets.DOMWidgetView { this.renderer.setRenderTarget(this.geometry_depth_target); this.renderer.clear(true, true, true); setVisible({volumes: false}); - this.renderer.render(this.scene, camera, this.geometry_depth_target); - this.renderer.render(this.scene_opaque, camera, this.geometry_depth_target); + this.renderer.render(this.scene, camera); + this.renderer.render(this.scene_opaque, camera); this.renderer.autoClear = true; } // Normal color pass of geometry for final screen pass this.renderer.autoClear = false; - this.renderer.setRenderTarget(this.color_pass_target); + this.renderer.setRenderTarget(null); this.renderer.clear(true, true, true); setVisible({volumes: false}); - this.renderer.render(this.scene, camera, this.color_pass_target); - this.renderer.render(this.scene_opaque, camera, this.color_pass_target); + this.renderer.render(this.scene, camera); + this.renderer.render(this.scene_opaque, camera); this.renderer.autoClear = true; if (has_volumes) { @@ -2410,7 +2452,7 @@ class FigureView extends widgets.DOMWidgetView { this.renderer.setRenderTarget(this.color_pass_target); this.renderer.clear(false, true, false); setVisible({volumes: true}); - this.renderer.render(this.scene, camera, this.color_pass_target); + this.renderer.render(this.scene, camera); this.renderer.autoClear = true; this.renderer.context.colorMask(true, true, true, true); @@ -2425,7 +2467,7 @@ class FigureView extends widgets.DOMWidgetView { // threejs does not want to be called with all three false // this.renderer.clear(false, false, false); setVisible({volumes: true}); - this.renderer.render(this.scene, camera, this.color_pass_target); + this.renderer.render(this.scene, camera); this.renderer.autoClear = true; } @@ -2445,7 +2487,7 @@ class FigureView extends widgets.DOMWidgetView { this.renderer.setRenderTarget(this.coordinate_target); this.renderer.clear(true, true, true); setVisible({volumes: false}); - this.renderer.render(this.scene, camera, this.coordinate_target); + this.renderer.render(this.scene, camera); this.renderer.autoClear = true; // id pass @@ -2465,7 +2507,7 @@ class FigureView extends widgets.DOMWidgetView { this.renderer.setRenderTarget(this.id_pass_target); this.renderer.clear(true, true, true); setVisible({volumes: false}); - this.renderer.render(this.scene, camera, this.id_pass_target); + this.renderer.render(this.scene, camera); this.renderer.autoClear = true; // now we render the weighted coordinate for the volumetric data @@ -2483,7 +2525,7 @@ class FigureView extends widgets.DOMWidgetView { this.renderer.setRenderTarget(this.color_pass_target); this.renderer.clear(false, true, false); setVisible({volumes: true}); - this.renderer.render(this.scene, camera, this.color_pass_target); + this.renderer.render(this.scene, camera); this.renderer.autoClear = true; this.renderer.context.colorMask(true, true, true, true); @@ -2498,7 +2540,7 @@ class FigureView extends widgets.DOMWidgetView { // threejs does not want to be called with all three false // this.renderer.clear(false, false, false); setVisible({volumes: true}); - this.renderer.render(this.scene, camera, this.coordinate_target); + this.renderer.render(this.scene, camera); this.renderer.autoClear = true; } diff --git a/notebooks/AR.ipynb b/notebooks/AR.ipynb new file mode 100644 index 00000000..1a07b9e7 --- /dev/null +++ b/notebooks/AR.ipynb @@ -0,0 +1,148 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "import ipyvolume as ipv" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "\n" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "%%html\n", + "" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [ + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "b1a949f842f948a9bdd7a9c6e3741ecd", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "HTML(value='
')" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "import ipywidgets\n", + "ipywidgets.HTML(value='
')" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": { + "scrolled": false + }, + "outputs": [ + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "ae46d541feee4bcbaebdd10b040a4971", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "Container(figure=Figure(box_center=[0.5, 0.5, 0.5], box_size=[1.0, 1.0, 1.0], camera=PerspectiveCamera(fov=45.…" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "43334914f9084518bcf24bf211de1ca3", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "Mesh(description='Mesh 0', description_color='red', line_material=ShaderMaterial(), material=ShaderMaterial(si…" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "ipv.figure()\n", + "ipv.examples.klein_bottle()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.9.6" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} From cac0166fc45dacb7f158ff736fc91bf567d91233 Mon Sep 17 00:00:00 2001 From: Mario Buikhuizen Date: Mon, 1 Nov 2021 13:56:12 +0100 Subject: [PATCH 02/13] add pythreejs dependecy with updated threejs version --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index b5d6f6b3..04c290c9 100644 --- a/setup.py +++ b/setup.py @@ -92,7 +92,7 @@ def read(fname): 'requests', 'ipyvuetify', 'ipyvue>=1.7.0', - 'pythreejs>=2.0.0', + 'pythreejs @ git+https://github.com/mariobuikhuizen/pythreejs@feat_upgrade_for_xr', 'matplotlib' ], license='MIT', From 17c1bd5973b52c3d6a060ee60bce3125aa941bf6 Mon Sep 17 00:00:00 2001 From: Mario Buikhuizen Date: Mon, 1 Nov 2021 15:29:24 +0100 Subject: [PATCH 03/13] fix: no image --- js/src/figure.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/js/src/figure.ts b/js/src/figure.ts index 8a548c00..ea5dbad2 100644 --- a/js/src/figure.ts +++ b/js/src/figure.ts @@ -2432,7 +2432,7 @@ class FigureView extends widgets.DOMWidgetView { // Normal color pass of geometry for final screen pass this.renderer.autoClear = false; - this.renderer.setRenderTarget(null); + this.renderer.setRenderTarget(this.color_pass_target); this.renderer.clear(true, true, true); setVisible({volumes: false}); this.renderer.render(this.scene, camera); From 41d684d990c23515a79dfb1ae86fc2a71d260807 Mon Sep 17 00:00:00 2001 From: Mario Buikhuizen Date: Mon, 1 Nov 2021 15:48:13 +0100 Subject: [PATCH 04/13] fix: don't use render_continuous --- js/src/figure.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/js/src/figure.ts b/js/src/figure.ts index ea5dbad2..2d1fd2df 100644 --- a/js/src/figure.ts +++ b/js/src/figure.ts @@ -1175,7 +1175,7 @@ class FigureView extends widgets.DOMWidgetView { this.renderer.xr.enabled = true; document.getElementById('bliep').appendChild(createButton(this.renderer, - () => {this.model.set("render_continuous", true); this.model.set("camera_control", "xr")}, + () => {/*this.model.set("render_continuous", true);*/ this.model.set("camera_control", "xr")}, () => this.model.set("render_continuous", false) /*{domOverlay: document.getElementById('bliep')}*/)); From 3057275e4bed20a5c60ba8c51b3e39e2ee212b25 Mon Sep 17 00:00:00 2001 From: "Maarten A. Breddels" Date: Tue, 2 Nov 2021 14:02:31 +0100 Subject: [PATCH 05/13] fix: better directional light defaults --- ipyvolume/pylab.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ipyvolume/pylab.py b/ipyvolume/pylab.py index ad447f1b..64f31a11 100644 --- a/ipyvolume/pylab.py +++ b/ipyvolume/pylab.py @@ -1803,11 +1803,11 @@ def light_hemisphere( def light_directional( light_color=default_color_selected, intensity=1, - position=[10, 10, 10], + position=[2, 2, 2], target=[0, 0, 0], near=0.1, - far=100, - shadow_camera_orthographic_size=10, + far=5, + shadow_camera_orthographic_size=3, cast_shadow=True): """Create a new Directional Light From 29e6dccc50fbc2f0a06660506e82d4b9a6c3f53b Mon Sep 17 00:00:00 2001 From: "Maarten A. Breddels" Date: Tue, 2 Nov 2021 14:04:04 +0100 Subject: [PATCH 06/13] feat: background-opacity in style, used for ar style --- ipyvolume/styles.py | 9 +++++ js/data/style.json | 88 ++++++++++++++++++++++----------------------- js/src/figure.ts | 2 +- 3 files changed, 53 insertions(+), 46 deletions(-) diff --git a/ipyvolume/styles.py b/ipyvolume/styles.py index c199b821..1279300d 100644 --- a/ipyvolume/styles.py +++ b/ipyvolume/styles.py @@ -3,6 +3,7 @@ Possible properies * background-color + * background-opacity * axes * color * visible @@ -31,6 +32,7 @@ styles = {} _defaults = { + 'background-opacity': 1.0, 'axes': {'visible': True, 'label': {'color': 'black'}, 'ticklabel': {'color': 'black'}}, 'box': {'visible': True}, } @@ -73,6 +75,13 @@ def create(name, properties): minimal = {'box': {'visible': False}, 'axes': {'visible': False}} nobox = create("nobox", {'box': {'visible': False}, 'axes': {'visible': True}}) +_ar = { + 'background-color': '#000001', # for some reason we cannot set it to black!?! + 'background-opacity': 0, +} +utils.dict_deep_update(_ar, minimal) +ar = create("ar", _ar) + if __name__ == "__main__": source = __file__ dest = os.path.join(os.path.dirname(source), "../js/data/style.json") diff --git a/js/data/style.json b/js/data/style.json index ef0364c7..eb5630e0 100644 --- a/js/data/style.json +++ b/js/data/style.json @@ -1,4 +1,22 @@ { + "light": { + "background-color": "white", + "axes": { + "color": "black" + } + }, + "dark": { + "background-color": "#000001", + "axes": { + "color": "white", + "label": { + "color": "white" + }, + "ticklabel": { + "color": "white" + } + } + }, "demo": { "background-color": "white", "box": { @@ -6,73 +24,53 @@ "visible": true }, "axes": { - "z": { - "color": "#00f", - "ticklabel": { - "color": "#0f0" - }, - "label": { - "color": "#f00" - } - }, - "ticklabel": { - "color": "black" - }, - "label": { - "color": "black" - }, + "color": "black", + "visible": true, "x": { "color": "#f00", - "ticklabel": { - "color": "#00f" - }, "label": { "color": "#0f0" + }, + "ticklabel": { + "color": "#00f" } }, "y": { "color": "#0f0", - "ticklabel": { - "color": "#f00" - }, "label": { "color": "#00f" + }, + "ticklabel": { + "color": "#f00" } }, - "color": "black", - "visible": true + "z": { + "color": "#00f", + "label": { + "color": "#f00" + }, + "ticklabel": { + "color": "#0f0" + } + } } }, - "dark": { - "background-color": "black", + "nobox": { "box": { - "visible": true + "visible": false }, "axes": { - "color": "white", - "ticklabel": { - "color": "white" - }, - "visible": true, - "label": { - "color": "white" - } + "visible": true } }, - "light": { - "background-color": "white", + "ar": { + "background-color": "#000001", + "background-opacity": 0, "box": { - "visible": true + "visible": false }, "axes": { - "color": "black", - "ticklabel": { - "color": "black" - }, - "visible": true, - "label": { - "color": "black" - } + "visible": false } } } \ No newline at end of file diff --git a/js/src/figure.ts b/js/src/figure.ts index 2d1fd2df..9894a87a 100644 --- a/js/src/figure.ts +++ b/js/src/figure.ts @@ -2141,7 +2141,7 @@ class FigureView extends widgets.DOMWidgetView { scatter_view.uniforms.aspect.value = this.model.get('box_size'); } - this.renderer.setClearColor(this.get_style_color("background-color")); + this.renderer.setClearColor(this.get_style_color("background-color"), this.get_style("background-opacity")); this.x_axis.visible = this.get_style("axes.x.visible axes.visible"); this.y_axis.visible = this.get_style("axes.y.visible axes.visible"); this.z_axis.visible = this.get_style("axes.z.visible axes.visible"); From 7bd0d430294057f2db9dad20c4a2608eb452a90f Mon Sep 17 00:00:00 2001 From: "Maarten A. Breddels" Date: Tue, 2 Nov 2021 14:08:17 +0100 Subject: [PATCH 07/13] use hittest to place box in scene --- js/src/figure.ts | 76 +++++++++++++++++++++++++++++++++--------------- 1 file changed, 53 insertions(+), 23 deletions(-) diff --git a/js/src/figure.ts b/js/src/figure.ts index 9894a87a..2f504c10 100644 --- a/js/src/figure.ts +++ b/js/src/figure.ts @@ -268,6 +268,7 @@ class FigureView extends widgets.DOMWidgetView { front_box_geo: THREE.BoxBufferGeometry; front_box_material: THREE.ShaderMaterial; slice_icon: ToolIcon; + arRenderLoop: (time: any, frame: any) => void; readPixel(x, y) { return this.readPixelFrom(this.screen_texture, x, y); @@ -713,33 +714,63 @@ class FigureView extends widgets.DOMWidgetView { // the threejs animation system looks at the parent of the camera and sends rerender msg'es // this.shared_scene.add(this.camera); - const geometry = new THREE.BoxGeometry(0.1, 0.1, 0.1); - const material = new THREE.MeshBasicMaterial( { color: 0x00ff00 } ); - const cube = new THREE.Mesh( geometry, material ); - cube.position.set( 0, 0, -3 ); - this.scene.add( cube ); - - // this.renderer.render( this.scene, this.camera ); - const onSelect = () => { - console.log('controller world:', this.controller.matrixWorld); - cube.position.set( 0, 0, -3 ).applyMatrix4( this.controller.matrixWorld ) - cube.quaternion.setFromRotationMatrix( this.controller.matrixWorld ); - this.rootObject.position.set( 0, 0, -3 ).applyMatrix4( this.controller.matrixWorld ); - this.rootObject.quaternion.setFromRotationMatrix( this.controller.matrixWorld ); - // update_box(); + if(lastXrTransform) { + // this.rootObject.matrixAutoUpdate = false; + + // this ignores the box_center/scale, but does rotation + // this.rootObject.matrix.fromArray(lastXrTransform.matrix); + // const matrixTrans = new THREE.Matrix4(); + // matrixTrans.makeTranslation(0.5, 0.5, 0.5); + // this.rootObject.matrix.multiply(matrixTrans); + + const pos = lastXrTransform.position; + this.model.set('box_center', [pos.x + 0.5, pos.y + 0.5, pos.z + 0.5]); + this.model.save_changes(); + } } this.controller = this.renderer.xr.getController( 0 ); this.controller.addEventListener( 'select', onSelect ); this.scene.add( this.controller ); - - this.renderer.setAnimationLoop( () => { - cube.rotation.x += 0.01; - cube.rotation.y += 0.01; - // this.renderer.render( this.scene, this.camera ); + let hitTestSourceRequested = false; + let hitTestSource = null; + let lastXrTransform = null; + + this.arRenderLoop = (time, frame) => { + + if (frame) { + // based on https://threejs.org/examples/webxr_ar_hittest.html + const renderer = this.renderer; + const referenceSpace = renderer.xr.getReferenceSpace(); + const session = renderer.xr.getSession(); + + if (hitTestSourceRequested === false) { + session.requestReferenceSpace( 'viewer' ).then( function ( referenceSpace ) { + session.requestHitTestSource( { space: referenceSpace } ).then( function ( source ) { + hitTestSource = source; + } ); + }); + session.addEventListener( 'end', function () { + hitTestSourceRequested = false; + hitTestSource = null; + } ); + hitTestSourceRequested = true; + } + if (hitTestSource) { + const hitTestResults = frame.getHitTestResults( hitTestSource ); + if(hitTestResults.length) { + const hit = hitTestResults[ 0 ]; + const pose = hit.getPose( referenceSpace ) + lastXrTransform = pose.transform; + } + } + } this._real_update(); - }); + } + + // TODO: should only be set in AR mode + this.renderer.setAnimationLoop(this.arRenderLoop); // this.scene_scatter = new THREE.Scene(); this.scene_opaque = new THREE.Scene(); @@ -763,8 +794,7 @@ class FigureView extends widgets.DOMWidgetView { this.scene_opaque.position.copy(box_position); this.rootObject.scale.copy(box_scale); - // this.rootObject.position.copy(box_position); - this.rootObject.position.set( 0, 0, -3 ) + this.rootObject.position.copy(box_position); this.update(); } this.model.on("change:box_center change:box_size", update_box); @@ -1175,7 +1205,7 @@ class FigureView extends widgets.DOMWidgetView { this.renderer.xr.enabled = true; document.getElementById('bliep').appendChild(createButton(this.renderer, - () => {/*this.model.set("render_continuous", true);*/ this.model.set("camera_control", "xr")}, + () => {this.model.set("camera_control", "xr")}, () => this.model.set("render_continuous", false) /*{domOverlay: document.getElementById('bliep')}*/)); From 336c62346d681ce8e7a54cff4e46a3f71e40b98f Mon Sep 17 00:00:00 2001 From: "Maarten A. Breddels" Date: Tue, 2 Nov 2021 14:10:09 +0100 Subject: [PATCH 08/13] render to screen in AR mode and ignore update calls --- js/src/figure.ts | 58 +++++++++++++++++++++++++++++------------------- 1 file changed, 35 insertions(+), 23 deletions(-) diff --git a/js/src/figure.ts b/js/src/figure.ts index 2f504c10..fd39737a 100644 --- a/js/src/figure.ts +++ b/js/src/figure.ts @@ -1210,6 +1210,11 @@ class FigureView extends widgets.DOMWidgetView { /*{domOverlay: document.getElementById('bliep')}*/)); } + + get in_ar_mode(): boolean { + return this.model.get("camera_control") == "xr"; + } + camera_initial(camera_initial: any) { throw new Error("Method not implemented."); } @@ -2146,10 +2151,14 @@ class FigureView extends widgets.DOMWidgetView { } update() { - // requestAnimationFrame stacks, so make sure multiple update calls only lead to 1 _real_update call - if (!this._update_requested) { - this._update_requested = true; - requestAnimationFrame(this._real_update.bind(this)); + if(this.in_ar_mode) { + // xr will render continious (via XR api), so ignore + } else { + // requestAnimationFrame stacks, so make sure multiple update calls only lead to 1 _real_update call + if (!this._update_requested) { + this._update_requested = true; + requestAnimationFrame(this._real_update.bind(this)); + } } } @@ -2462,7 +2471,8 @@ class FigureView extends widgets.DOMWidgetView { // Normal color pass of geometry for final screen pass this.renderer.autoClear = false; - this.renderer.setRenderTarget(this.color_pass_target); + // in AR mode we directly need to render to the screen + this.renderer.setRenderTarget(this.in_ar_mode ? null : this.color_pass_target); this.renderer.clear(true, true, true); setVisible({volumes: false}); this.renderer.render(this.scene, camera); @@ -2585,24 +2595,26 @@ class FigureView extends widgets.DOMWidgetView { }); } - if(this.model.get("show") == "Shadow") { - this._render_shadow(); - } else { - // render to screen - this.screen_texture = { - render: this.color_pass_target, - front: this.volume_front_target, - back: this.volume_back_target, - // Geometry_back: this.geometry_depth_target, - coordinate: this.coordinate_target, - id: this.id_pass_target, - }[this.model.get("show")]; - // TODO: remove any - this.screen_material.uniforms.tex.value = (this.screen_texture as any).texture; - - this.renderer.setRenderTarget(null); - this.renderer.clear(true, true, true); - this.renderer.render(this.screen_scene, this.screen_camera); + if(!this.in_ar_mode) { + if(this.model.get("show") == "Shadow") { + this._render_shadow(); + } else { + // render to screen + this.screen_texture = { + render: this.color_pass_target, + front: this.volume_front_target, + back: this.volume_back_target, + // Geometry_back: this.geometry_depth_target, + coordinate: this.coordinate_target, + id: this.id_pass_target, + }[this.model.get("show")]; + // TODO: remove any + this.screen_material.uniforms.tex.value = (this.screen_texture as any).texture; + + this.renderer.setRenderTarget(null); + this.renderer.clear(true, true, true); + this.renderer.render(this.screen_scene, this.screen_camera); + } } restoreVisible(); } From 1f35631fdf44288eb42de5649d10536535620763 Mon Sep 17 00:00:00 2001 From: "Maarten A. Breddels" Date: Tue, 2 Nov 2021 14:10:40 +0100 Subject: [PATCH 09/13] fix: add lights to rootObject such that they track the AR placement --- js/src/figure.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/js/src/figure.ts b/js/src/figure.ts index fd39737a..d575ab93 100644 --- a/js/src/figure.ts +++ b/js/src/figure.ts @@ -2011,7 +2011,7 @@ class FigureView extends widgets.DOMWidgetView { // Remove previous lights this.model.previous('lights').forEach(light_model => { const light = light_model.obj; - this.scene.remove(light); + this.rootObject.remove(light); }); } @@ -2029,11 +2029,11 @@ class FigureView extends widgets.DOMWidgetView { this.update(); } if(light.target) { - this.scene.add(light.target) + this.rootObject.add(light.target) } light_model.on("change", on_light_change); light_model.on("childchange", on_light_change); - this.scene.add(light); + this.rootObject.add(light); }); // if we update the lights, we need to force all materials to update/ // see https://stackoverflow.com/questions/16879378/adding-and-removing-three-js-lights-at-run-time From 9ea176d165a507cfb4f06ffc4f2a1aa8d78394e2 Mon Sep 17 00:00:00 2001 From: Mario Buikhuizen Date: Thu, 4 Nov 2021 12:22:42 +0100 Subject: [PATCH 10/13] refactor: use template for button and untangle ar code --- ipyvolume/ui.py | 1 + ipyvolume/vue/container.vue | 25 +++- ipyvolume/widgets.py | 2 + js/src/ar.js | 277 ++++++++++++------------------------ js/src/figure.ts | 132 ++++++++--------- notebooks/AR.ipynb | 110 +------------- 6 files changed, 182 insertions(+), 365 deletions(-) diff --git a/ipyvolume/ui.py b/ipyvolume/ui.py index 0ab6799e..87ea7332 100644 --- a/ipyvolume/ui.py +++ b/ipyvolume/ui.py @@ -25,6 +25,7 @@ class Container(v.VuetifyTemplate): children = traitlets.List().tag(sync=True, **widgets.widget_serialization) models = traitlets.Any({'figure': {}}).tag(sync=True) panels = traitlets.List(traitlets.CInt(), default_value=[0, 1, 2]).tag(sync=True) + ar_supported = traitlets.Bool(False).tag(sync=True) class Popup(v.VuetifyTemplate): diff --git a/ipyvolume/vue/container.vue b/ipyvolume/vue/container.vue index f31f517f..175fbc4f 100644 --- a/ipyvolume/vue/container.vue +++ b/ipyvolume/vue/container.vue @@ -29,6 +29,12 @@ +
+ + mdi-glasses + start ar + +
@@ -44,13 +50,17 @@