diff --git a/README.md b/README.md index bf51008..1a76f57 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,24 @@ # WebXR Layers Sample + + +This repository is an amalgamation of WebXR samples created using the [WebXR API](https://www.w3.org/TR/webxrlayers-1/#intro), with [Three.js](https://threejs.org/) as the primary 3D rendering library as part of the [MLH Fellowship](https://fellowship.mlh.io/). + +## Getting Started +- [Project Setup - Local Machine ](./docs/Project-Setup.md) : Since WebXR only works over secure HTTPS connections, we would require a few initial configurations such as an OpenSSL certificate to get the project up and running on the local machime + +- [Project Setup - Oculus Browser](./docs/Oculus-Dev-Setup.md): Detailed steps leading upto a complete developer and debugging setup within the Oculus Browser + +## Samples Overview +This [application](https://webxr-layers.netlify.app/) contains samples that belong to two broad categories: + +- [Creaing and Interacting with Video Layers in an XR Environment]() +- Hand Gestures and Interactions with Three.js Objects + + +## Contributors +| MLH Fellowship Cohort | Contributors | +| ------------- | ------------- | +| [Spring 2021](https://developers.facebook.com/blog/post/2021/03/31/facebook-open-source-introduces-mlh-fellowship-class-spring-2021/) | [Soham Parekh](https://github.com/und3fined-v01d), [Teoh Zhixiang](https://github.com/zhixiangteoh) | +| Summer 2021 | [Schezeen Fazulbhoy](https://github.com/schezfaz), [Awan Shrestha ](https://github.com/awanshrestha) | -[Documentation](https://github.com/und3fined-v01d/webxr-layers/tree/docs) | [Application](https://webxr-layers.netlify.app) diff --git a/apps/changeColor.js b/apps/changeColor.js new file mode 100644 index 0000000..ddde360 --- /dev/null +++ b/apps/changeColor.js @@ -0,0 +1,210 @@ + +import * as THREE from 'three'; +import { VRButton } from 'three/examples/jsm/webxr/VRButton'; +import { BoxLineGeometry } from 'three/examples/jsm/geometries/BoxLineGeometry'; +import { OrbitControls } from 'three/examples/jsm/controls/OrbitControls'; +import { XRControllerModelFactory } from 'three/examples/jsm/webxr/XRControllerModelFactory.js'; +import { XRHandModelFactory } from 'three/examples/jsm/webxr/XRHandModelFactory.js'; + +const tmpVector1 = new THREE.Vector3(); +const tmpVector2 = new THREE.Vector3(); + +let App = class App { + constructor() { + const container = document.createElement('div'); + document.body.appendChild(container); + + let controllerRight, controllerLeft; + let controllerRightGrip, controllerLeftGrip; + + this.drawFlag = false; + + /*Creating Camera, Scene and Renderer */ + this.camera = this.createCamera(); + this.scene = this.createScene(); + this.renderer = this.createRenderer(); + container.appendChild(this.renderer.domElement); + + this.controls = new OrbitControls(this.camera, container); + this.controls.target.set(0, 1.6, 0); + this.controls.update(); + + /*Initialising controllerModelFactory and handModelFactory from Three.js */ + const controllerModelFactory = new XRControllerModelFactory(); + const handModelFactory = new XRHandModelFactory().setPath( + "/hand-models" + );; + + /* Setting up Right Hand from POV */ + controllerRight = this.renderer.xr.getController(0); + this.scene.add(controllerRight); + controllerRightGrip = this.renderer.xr.getControllerGrip(0); + controllerRightGrip.add(controllerModelFactory.createControllerModel(controllerRightGrip)); + this.scene.add(controllerRightGrip); + + this.rightHand = this.renderer.xr.getHand(0); + this.rightHand.add(handModelFactory.createHandModel(this.rightHand)); + this.scene.add(this.rightHand); + + /* Setting up Left Hand from POV */ + controllerLeft = this.renderer.xr.getController(1); + this.scene.add(controllerLeft); + controllerLeftGrip = this.renderer.xr.getControllerGrip(1); + controllerLeftGrip.add(controllerModelFactory.createControllerModel(controllerLeftGrip)); + this.scene.add(controllerLeftGrip); + + this.leftHand = this.renderer.xr.getHand(1); + this.leftHand.add(handModelFactory.createHandModel(this.leftHand)); + this.scene.add(this.leftHand); + + this.session; + this.renderer.xr.addEventListener("sessionstart", (event) => { + this.session = this.renderer.xr.getSession(); + + }); + this.renderer.xr.addEventListener("sessionend", (event) => { + this.session = null; + }); + + this.addLight(); + + const geometry = new THREE.SphereBufferGeometry(0.4, 30, 30); + const material = new THREE.MeshStandardMaterial({ color: 0xfdffbf }); + + this.sphere = new THREE.Mesh(geometry, material); + this.sphere.position.set(0, 0.6, -0.45); + + this.scene.add(this.sphere) + + const room = new THREE.LineSegments( + new BoxLineGeometry(6, 6, 6, 10, 10, 10), + new THREE.LineBasicMaterial({ + color: "#151515", + }) + ); + + room.geometry.translate(0, 3, 0); + + this.scene.add(room); + + this.setupVR(); + + this.renderer.setAnimationLoop(this.render.bind(this)); + window.addEventListener('resize', this.resize.bind(this)); + + } + + getHandVisibilityStatus() { + // if (this.session) { + // for (let i = 0; i < this.session.inputSources.length; i++) { + // if (this.session.inputSources[i].hand) { + // let wrist = this.session.inputSources[i].hand.get("wrist"); + // if (!wrist) { + // return; + // } + // // let wristPose = getJointPose(wrist, this.renderer.referenceSpace); + // } + // } + // } + if (this.session) { + for (const inputSource of this.session.inputSources) { + if (inputSource.hand) { + let name = inputSource.handedness; + let theHand; + if (name === 'right'){ + theHand = this.rightHand; + let indexTip = theHand.joints['index-finger-tip']; + this.checkTouch(indexTip, name) + } else { + theHand = this.leftHand; + let indexTip = theHand.joints['index-finger-tip']; + this.checkTouch(indexTip, name) + } + } + } + } + } + checkTouch( indexTip, hand) { + // let diffX = Math.abs(indexTip.position.x - thumbTip.position.x) + // let diffY = Math.abs(indexTip.position.y - thumbTip.position.y) + // let diffZ = Math.abs(indexTip.position.z - thumbTip.position.z) + // if (diffX!=0 || diffY!=0 || diffZ!=0){// When hands are not seen, the diffs initialize at zeroes. Stops once hands are seen. + // if (diffX < 0.02 && diffY < 0.02 && diffZ < 0.02) { + // console.log(diffX, diffY, diffZ) + // console.log(hand + "PINCHING") + // let geometryPinch = new THREE.SphereBufferGeometry(0.01, 30, 30); + // let materialPinch = new THREE.MeshStandardMaterial({ color: 0xfdffbf }); + + // let spherePinch = new THREE.Mesh(geometryPinch, materialPinch); + // spherePinch.position.set(indexTip.position.x, indexTip.position.y, indexTip.position.z); + // this.scene.add(spherePinch); + // } + // } + + const distance = indexTip.getWorldPosition(tmpVector1).distanceTo(this.sphere.getWorldPosition(tmpVector2)); + if (distance < this.sphere.geometry.boundingSphere.radius * this.sphere.scale.x) { + + console.log('Touch') + if(hand == 'right'){ + this.sphere.material.color.setHex( 0x00ffaa ); + } + if(hand == 'left'){ + this.sphere.material.color.setHex( 0xb983de ); + } + + + } + + } + + resize() { + this.camera.aspect = window.innerWidth / window.innerHeight; + this.camera.updateProjectionMatrix(); + this.renderer.setSize(window.innerWidth, window.innerHeight); + } + render() { + this.renderer.render(this.scene, this.camera); + this.getHandVisibilityStatus(); + } + + createCamera() { + const camera = new THREE.PerspectiveCamera( + 50, + window.innerWidth / window.innerHeight, + 0.1, + 100 + ); + + camera.position.set(0, 1.6, 3); + return camera; + } + + createScene() { + const scene = new THREE.Scene(); + scene.background = new THREE.Color(0x808080); + return scene; + } + + createRenderer() { + const renderer = new THREE.WebGLRenderer({ antialias: true }); + renderer.setPixelRatio(window.devicePixelRatio); + renderer.setSize(window.innerWidth, window.innerHeight); + renderer.outputEncoding = THREE.sRGBEncoding; + renderer.xr.enabled = true; + return renderer; + } + + addLight() { + const ambient = new THREE.HemisphereLight(0x606060, 0x404040); + this.scene.add(ambient); + const light = new THREE.DirectionalLight(0xffffff); + light.position.set(1, 1, 1).normalize(); + this.scene.add(light); + } + setupVR() { + this.renderer.xr.enabled = true; + document.body.appendChild(VRButton.createButton(this.renderer)); + } +} + +export default App; diff --git a/apps/controller-interaction.js b/apps/controller-interactions/controller-interaction.js similarity index 97% rename from apps/controller-interaction.js rename to apps/controller-interactions/controller-interaction.js index ed166c3..2621adf 100644 --- a/apps/controller-interaction.js +++ b/apps/controller-interactions/controller-interaction.js @@ -4,8 +4,8 @@ import { BoxLineGeometry } from "three/examples/jsm/geometries/BoxLineGeometry"; import { OrbitControls } from "three/examples/jsm/controls/OrbitControls"; import { XRControllerModelFactory } from "three/examples/jsm/webxr/XRControllerModelFactory"; -import { WebGLRenderer } from "../util/WebGLRenderer"; -import { VRButton } from "../util/webxr/VRButton"; +import { WebGLRenderer } from "../../util/WebGLRenderer"; +import { VRButton } from "../../util/webxr/VRButton"; class App { constructor() { diff --git a/apps/hand-drawing.js b/apps/hand-drawing.js new file mode 100644 index 0000000..f67691c --- /dev/null +++ b/apps/hand-drawing.js @@ -0,0 +1,202 @@ + +import * as THREE from 'three'; +import { VRButton } from 'three/examples/jsm/webxr/VRButton'; +import { BoxLineGeometry } from 'three/examples/jsm/geometries/BoxLineGeometry'; +import { OrbitControls } from 'three/examples/jsm/controls/OrbitControls'; +import { XRControllerModelFactory } from 'three/examples/jsm/webxr/XRControllerModelFactory.js'; +import { XRHandModelFactory } from 'three/examples/jsm/webxr/XRHandModelFactory.js'; + +let App = class App { + constructor() { + const container = document.createElement('div'); + document.body.appendChild(container); + + let controllerRight, controllerLeft; + let controllerRightGrip, controllerLeftGrip; + + this.drawFlag = false; + + /*Creating Camera, Scene and Renderer */ + this.camera = this.createCamera(); + this.scene = this.createScene(); + this.renderer = this.createRenderer(); + container.appendChild(this.renderer.domElement); + + this.controls = new OrbitControls(this.camera, container); + this.controls.target.set(0, 1.6, 0); + this.controls.update(); + + /*Initialising controllerModelFactory and handModelFactory from Three.js */ + const controllerModelFactory = new XRControllerModelFactory(); + const handModelFactory = new XRHandModelFactory().setPath( + "..hkjhk/hand-models" + );; + + /* Setting up Right Hand from POV */ + controllerRight = this.renderer.xr.getController(0); + this.scene.add(controllerRight); + controllerRightGrip = this.renderer.xr.getControllerGrip(0); + controllerRightGrip.add(controllerModelFactory.createControllerModel(controllerRightGrip)); + this.scene.add(controllerRightGrip); + + this.rightHand = this.renderer.xr.getHand(0); + this.rightHand.add(handModelFactory.createHandModel(this.rightHand)); + this.scene.add(this.rightHand); + + /* Setting up Left Hand from POV */ + controllerLeft = this.renderer.xr.getController(1); + this.scene.add(controllerLeft); + controllerLeftGrip = this.renderer.xr.getControllerGrip(1); + controllerLeftGrip.add(controllerModelFactory.createControllerModel(controllerLeftGrip)); + this.scene.add(controllerLeftGrip); + + this.leftHand = this.renderer.xr.getHand(1); + this.leftHand.add(handModelFactory.createHandModel(this.leftHand)); + this.scene.add(this.leftHand); + + this.session; + this.renderer.xr.addEventListener("sessionstart", (event) => { + this.session = this.renderer.xr.getSession(); + + }); + this.renderer.xr.addEventListener("sessionend", (event) => { + this.session = null; + }); + + this.addLight(); + + const geometry = new THREE.SphereBufferGeometry(0.4, 30, 30); + const material = new THREE.MeshStandardMaterial({ color: 0xfdffbf }); + + const sphere = new THREE.Mesh(geometry, material); + sphere.position.set(0, 0.6, -1.5); + + this.scene.add(sphere) + + const room = new THREE.LineSegments( + new BoxLineGeometry(6, 6, 6, 10, 10, 10), + new THREE.LineBasicMaterial({ + color: "#151515", + }) + ); + + room.geometry.translate(0, 3, 0); + + this.scene.add(room); + + this.setupVR(); + + this.renderer.setAnimationLoop(this.render.bind(this)); + window.addEventListener('resize', this.resize.bind(this)); + + } + + getHandVisibilityStatus() { + // if (this.session) { + // for (let i = 0; i < this.session.inputSources.length; i++) { + // if (this.session.inputSources[i].hand) { + // let wrist = this.session.inputSources[i].hand.get("wrist"); + // if (!wrist) { + // return; + // } + // // let wristPose = getJointPose(wrist, this.renderer.referenceSpace); + // } + // } + // } + if (this.session) { + for (const inputSource of this.session.inputSources) { + if (inputSource.hand) { + + let name = inputSource.handedness; + let theHand; + if (name === 'right'){ + theHand = this.rightHand; + let indexTip = theHand.joints['index-finger-tip']; + let thumbTip = theHand.joints['thumb-tip']; + this.checkPinch(indexTip, thumbTip, name) + } else { + theHand = this.leftHand; + let indexTip = theHand.joints['index-finger-tip']; + let thumbTip = theHand.joints['thumb-tip']; + this.checkPinch(indexTip, thumbTip, name) + } + + + + // let leftIndexTip = this.leftHand.joints['index-finger-tip']; + // let leftThumbTip = this.leftHand.joints['thumb-tip']; + // this.checkPinch(leftThumbTip, leftIndexTip, 'LEFT') + } + } + } + } + checkPinch(thumbTip, indexTip, hand) { + let diffX = Math.abs(indexTip.position.x - thumbTip.position.x) + let diffY = Math.abs(indexTip.position.y - thumbTip.position.y) + let diffZ = Math.abs(indexTip.position.z - thumbTip.position.z) + if (diffX!=0 || diffY!=0 || diffZ!=0){// When hands are not seen, the diffs initialize at zeroes. Stops once hands are seen. + if (diffX < 0.02 && diffY < 0.02 && diffZ < 0.02) { + console.log(diffX, diffY, diffZ) + console.log(hand + "PINCHING") + let geometryPinch = new THREE.SphereBufferGeometry(0.01, 30, 30); + let materialPinch = new THREE.MeshStandardMaterial({ color: 0xfdffbf }); + + let spherePinch = new THREE.Mesh(geometryPinch, materialPinch); + spherePinch.position.set(indexTip.position.x, indexTip.position.y, indexTip.position.z); + this.scene.add(spherePinch); + } + } + + } + + resize() { + this.camera.aspect = window.innerWidth / window.innerHeight; + this.camera.updateProjectionMatrix(); + this.renderer.setSize(window.innerWidth, window.innerHeight); + } + render() { + this.renderer.render(this.scene, this.camera); + this.getHandVisibilityStatus(); + } + + createCamera() { + const camera = new THREE.PerspectiveCamera( + 50, + window.innerWidth / window.innerHeight, + 0.1, + 100 + ); + + camera.position.set(0, 1.6, 3); + return camera; + } + + createScene() { + const scene = new THREE.Scene(); + scene.background = new THREE.Color(0x808080); + return scene; + } + + createRenderer() { + const renderer = new THREE.WebGLRenderer({ antialias: true }); + renderer.setPixelRatio(window.devicePixelRatio); + renderer.setSize(window.innerWidth, window.innerHeight); + renderer.outputEncoding = THREE.sRGBEncoding; + renderer.xr.enabled = true; + return renderer; + } + + addLight() { + const ambient = new THREE.HemisphereLight(0x606060, 0x404040); + this.scene.add(ambient); + const light = new THREE.DirectionalLight(0xffffff); + light.position.set(1, 1, 1).normalize(); + this.scene.add(light); + } + setupVR() { + this.renderer.xr.enabled = true; + document.body.appendChild(VRButton.createButton(this.renderer)); + } +} + +export default App; diff --git a/apps/hand-grabbing-dist.js b/apps/hand-grabbing-dist.js new file mode 100644 index 0000000..66af058 --- /dev/null +++ b/apps/hand-grabbing-dist.js @@ -0,0 +1,218 @@ + +import * as THREE from 'three'; +import { VRButton } from 'three/examples/jsm/webxr/VRButton'; +import { BoxLineGeometry } from 'three/examples/jsm/geometries/BoxLineGeometry'; +import { OrbitControls } from 'three/examples/jsm/controls/OrbitControls'; +import { XRControllerModelFactory } from 'three/examples/jsm/webxr/XRControllerModelFactory.js'; +import { XRHandModelFactory } from 'three/examples/jsm/webxr/XRHandModelFactory.js'; + +let App = class App { + constructor() { + const container = document.createElement('div'); + document.body.appendChild(container); + + let controllerRight, controllerLeft; + let controllerRightGrip, controllerLeftGrip; + + this.drawFlag = false; + + /*Creating Camera, Scene and Renderer */ + this.camera = this.createCamera(); + this.scene = this.createScene(); + this.renderer = this.createRenderer(); + container.appendChild(this.renderer.domElement); + + this.controls = new OrbitControls(this.camera, container); + this.controls.target.set(0, 1.6, 0); + this.controls.update(); + + /*Initialising controllerModelFactory and handModelFactory from Three.js */ + const controllerModelFactory = new XRControllerModelFactory(); + const handModelFactory = new XRHandModelFactory().setPath( + "/hand-models" + );; + + /* Setting up Right Hand from POV */ + controllerRight = this.renderer.xr.getController(0); + this.scene.add(controllerRight); + controllerRightGrip = this.renderer.xr.getControllerGrip(0); + controllerRightGrip.add(controllerModelFactory.createControllerModel(controllerRightGrip)); + this.scene.add(controllerRightGrip); + + this.rightHand = this.renderer.xr.getHand(0); + this.rightHand.add(handModelFactory.createHandModel(this.rightHand)); + this.scene.add(this.rightHand); + + /* Setting up Left Hand from POV */ + controllerLeft = this.renderer.xr.getController(1); + this.scene.add(controllerLeft); + controllerLeftGrip = this.renderer.xr.getControllerGrip(1); + controllerLeftGrip.add(controllerModelFactory.createControllerModel(controllerLeftGrip)); + this.scene.add(controllerLeftGrip); + + this.leftHand = this.renderer.xr.getHand(1); + this.leftHand.add(handModelFactory.createHandModel(this.leftHand)); + this.scene.add(this.leftHand); + + this.session; + this.renderer.xr.addEventListener("sessionstart", (event) => { + this.session = this.renderer.xr.getSession(); + + }); + this.renderer.xr.addEventListener("sessionend", (event) => { + this.session = null; + }); + + this.addLight(); + + this.spheres = []; + + const room = new THREE.LineSegments( + new BoxLineGeometry(6, 6, 6, 10, 10, 10), + new THREE.LineBasicMaterial({ + color: "#151515", + }) + ); + + room.geometry.translate(0, 3, 0); + + this.scene.add(room); + + this.setupVR(); + + this.renderer.setAnimationLoop(this.render.bind(this)); + window.addEventListener('resize', this.resize.bind(this)); + + } + + getHandVisibilityStatus() { + if (this.session) { + for (const inputSource of this.session.inputSources) { + if (inputSource.hand) { + + let name = inputSource.handedness; + let theHand; + if (name === 'right'){ + theHand = this.rightHand; + let indexTip = theHand.joints['index-finger-tip']; + let thumbTip = theHand.joints['thumb-tip']; + this.checkPinch(indexTip, thumbTip, name) + } else { + theHand = this.leftHand; + let indexTip = theHand.joints['index-finger-tip']; + let thumbTip = theHand.joints['thumb-tip']; + let middleFingerTip = theHand.joints['middle-finger-tip']; + let ringFingerTip = theHand.joints['ring-finger-tip']; + let pinkyFingerTip = theHand.joints['pinky-finger-tip']; + let fingerTips = [indexTip, thumbTip, middleFingerTip, pinkyFingerTip, ringFingerTip]; + this.erase(fingerTips, name); + } + + + + // let leftIndexTip = this.leftHand.joints['index-finger-tip']; + // let leftThumbTip = this.leftHand.joints['thumb-tip']; + // this.checkPinch(leftThumbTip, leftIndexTip, 'LEFT') + } + } + } + } + checkPinch(thumbTip, indexTip, hand) { + let diffX = Math.abs(indexTip.position.x - thumbTip.position.x) + let diffY = Math.abs(indexTip.position.y - thumbTip.position.y) + let diffZ = Math.abs(indexTip.position.z - thumbTip.position.z) + if (diffX!=0 || diffY!=0 || diffZ!=0){// When hands are not seen, the diffs initialize at zeroes. Stops once hands are seen. + if (diffX < 0.02 && diffY < 0.02 && diffZ < 0.02) { + if(hand=="right"){ + console.log(diffX, diffY, diffZ); + console.log(hand + "draw"); + //sphere to be "drawn" + let geometryPinch = new THREE.SphereBufferGeometry(0.01, 30, 30); + let materialPinch = new THREE.MeshStandardMaterial({ color: 0x000000 }); + let spherePinch = new THREE.Mesh(geometryPinch, materialPinch); + spherePinch.position.set(indexTip.position.x, indexTip.position.y, indexTip.position.z); + this.spheres.push(spherePinch); + this.scene.add(spherePinch); + } + } + } + } + + erase(fingerTips, hand){ + if(hand=="left"){ + for (var i = 0; i < fingerTips.length; i++) { + const sphereInProximity = this.checkProximity(fingerTips[i]); + if(sphereInProximity) { + this.currentSphere = sphereInProximity; + this.scene.remove(sphereInProximity); + } + } + } + } + + checkProximity(indexTip){ + for(let i = 0; i< this.spheres.length; i++){ + const sphere = this.spheres[ i ]; + var dx = indexTip.position.x - sphere.position.x; + var dy = indexTip.position.y - sphere.position.y; + var dz = indexTip.position.z - sphere.position.z; + const distance = Math.sqrt(dx*dx+dy*dy+dz*dz); + if(distance <= 0.04 ){ + console.log("move"); + return sphere; + } + } + return null; + } + + resize() { + this.camera.aspect = window.innerWidth / window.innerHeight; + this.camera.updateProjectionMatrix(); + this.renderer.setSize(window.innerWidth, window.innerHeight); + } + render() { + this.renderer.render(this.scene, this.camera); + this.getHandVisibilityStatus(); + } + + createCamera() { + const camera = new THREE.PerspectiveCamera( + 50, + window.innerWidth / window.innerHeight, + 0.1, + 100 + ); + + camera.position.set(0, 1.6, 3); + return camera; + } + + createScene() { + const scene = new THREE.Scene(); + scene.background = new THREE.Color(0x808080); + return scene; + } + + createRenderer() { + const renderer = new THREE.WebGLRenderer({ antialias: true }); + renderer.setPixelRatio(window.devicePixelRatio); + renderer.setSize(window.innerWidth, window.innerHeight); + renderer.outputEncoding = THREE.sRGBEncoding; + renderer.xr.enabled = true; + return renderer; + } + + addLight() { + const ambient = new THREE.HemisphereLight(0x606060, 0x404040); + this.scene.add(ambient); + const light = new THREE.DirectionalLight(0xffffff); + light.position.set(1, 1, 1).normalize(); + this.scene.add(light); + } + setupVR() { + this.renderer.xr.enabled = true; + document.body.appendChild(VRButton.createButton(this.renderer)); + } +} + +export default App; diff --git a/apps/hand-grabbing-three.js b/apps/hand-grabbing-three.js new file mode 100644 index 0000000..544080e --- /dev/null +++ b/apps/hand-grabbing-three.js @@ -0,0 +1,278 @@ +import * as THREE from 'three'; +import { VRButton } from 'three/examples/jsm/webxr/VRButton'; +import { BoxLineGeometry } from 'three/examples/jsm/geometries/BoxLineGeometry'; +import { OrbitControls } from 'three/examples/jsm/controls/OrbitControls'; +import { XRControllerModelFactory } from 'three/examples/jsm/webxr/XRControllerModelFactory.js'; +import { XRHandModelFactory } from 'three/examples/jsm/webxr/XRHandModelFactory.js'; + +const tmpVector1 = new THREE.Vector3(); + +let App = class App{ + constructor(){ + const container = document.createElement( 'div' ); + document.body.appendChild( container ); + + let controllerRight, controllerLeft; + let controllerRightGrip, controllerLeftGrip; + + this.drawFlag = false; + this.spheres = []; + let grabbing = false; + + this.tmpVector1 = new THREE.Vector3(); + this.tmpVector2 = new THREE.Vector3(); + + /*Creating Camera, Scene and Renderer */ + this.camera = this.createCamera(); + this.scene = this.createScene(); + this.renderer = this.createRenderer(); + container.appendChild(this.renderer.domElement); + + this.controls = new OrbitControls( this.camera, container ); + this.controls.target.set( 0, 1.6, 0 ); + this.controls.update(); + + /*Initialising controllerModelFactory and handModelFactory from Three.js */ + const controllerModelFactory = new XRControllerModelFactory(); + const handModelFactory = new XRHandModelFactory().setPath( + "/hand-models" + );; //not sure about setPath ~ trying to fetch .glb files for hand models + + /* Setting up Right Hand from POV */ + controllerRight = this.renderer.xr.getController( 0 ); + this.scene.add( controllerRight ); + controllerRightGrip = this.renderer.xr.getControllerGrip( 0 ); + controllerRightGrip.add( controllerModelFactory.createControllerModel( controllerRightGrip ) ); + this.scene.add( controllerRightGrip ); + + this.rightHand = this.renderer.xr.getHand( 0 ); + this.rightHand.add( handModelFactory.createHandModel( this.rightHand) ); + this.scene.add( this.rightHand ); + + /*Event Listeners for 'Pinch' detection in the Right Hand */ + // this.rightHand.addEventListener( 'pinchstart', this.onPinchStartRight); + this.rightHand.addEventListener('pinchstart', (event) => { + let timeStamp = new Date(); + let currentTimeStamp = timeStamp.getHours() + ":" + timeStamp.getMinutes() + ":" + timeStamp.getSeconds() + ":" + timeStamp.getMilliseconds(); + //console.log("RightHand Pinch Started at: " + currentTimeStamp); + const hand = event.target; + const indexTip = hand.joints[ 'index-finger-tip' ]; + this.drawFlag = true; + if(this.drawFlag == true){ + //console.log("drawing - right hand"); + //console.log("IndexTip RightHand Deets: " + JSON.stringify(indexTip, null, 4)); + + //console.log(indexTip.position); + const geometryR = new THREE.SphereBufferGeometry(0.01, 30, 30); + const materialR = new THREE.MeshStandardMaterial({color: 0x000000}); + + const sphereR = new THREE.Mesh(geometryR, materialR); + sphereR.position.set(indexTip.position.x, indexTip.position.y, indexTip.position.z); + this.spheres.push(sphereR); + //console.log(this.spheres); + + this.scene.add(sphereR); + } + + }) + this.rightHand.addEventListener( 'pinchend', this.onPinchEndRight); + + /* Setting up Left Hand from POV */ + controllerLeft = this.renderer.xr.getController( 1 ); + this.scene.add( controllerLeft ); + controllerLeftGrip = this.renderer.xr.getControllerGrip( 1 ); + controllerLeftGrip.add( controllerModelFactory.createControllerModel( controllerLeftGrip ) ); + this.scene.add( controllerLeftGrip ); + + this.leftHand = this.renderer.xr.getHand( 1 ); + this.leftHand.add( handModelFactory.createHandModel( this.leftHand ) ); + this.scene.add( this.leftHand ); + + this.currentSphere; + + /*Event Listeners for 'Pinch' detection in the Right Hand */ + this.leftHand.addEventListener( 'pinchstart', (event) => { + let timeStamp = new Date(); + let currentTimeStamp = timeStamp.getHours() + ":" + timeStamp.getMinutes() + ":" + timeStamp.getSeconds() + ":" + timeStamp.getMilliseconds(); + //console.log("LeftHand Pinch Started at: " + currentTimeStamp); + const hand = event.target; + const indexTip = hand.joints[ 'index-finger-tip' ]; + this.drawFlag = true; + // if(this.drawFlag == true){ + // console.log("drawing - left hand"); + // console.log("Index Tip Position: " + indexTip.position); + + // const geometryL = new THREE.SphereBufferGeometry(0.01, 30, 30); + // const materialL = new THREE.MeshStandardMaterial({color: 0xff0000}); + + // const sphereL = new THREE.Mesh(geometryL, materialL); + // sphereL.position.set(indexTip.position.x, indexTip.position.y, indexTip.position.z); + + // this.scene.add(sphereL); + // } + let sphereInProximity = this.checkProximity(indexTip); + if(sphereInProximity) { + grabbing = true; + this.currentSphere = (this.currentSphere) + ? this.currentSphere.concat(sphereInProximity) + : sphereInProximity; + // console.log(sphereInProximity) + // console.log(this.currentSphere) + //console.log("hereL : " + JSON.stringify(this.currentSphere, null, 4)); + for (let i = 0; i < this.currentSphere.length; i ++){ + indexTip.attach(this.currentSphere[i]) + } + // indexTip.attach(sphereInProximity); + } + }); + + this.leftHand.addEventListener( 'pinchend', (event) => { + grabbing = false; + const hand = event.target; + const indexTip = hand.joints[ 'index-finger-tip' ]; + //console.log("CS: " + JSON.stringify(this.currentSphere, null, 4)); + let indexCurrentPose = indexTip.getWorldPosition(tmpVector1) + for (let i = 0; i < this.currentSphere.length; i ++){ + let tempSphere = this.currentSphere[i] + this.scene.attach(tempSphere) + this.currentSphere = this.currentSphere.filter(item => item !== this.currentSphere[i]) + } + // this.currentSphere.position.set(indexTip.position); + //this.scene.add(this.currentSphere); + }); + + + this.addLight(); + + // const geometry = new THREE.SphereBufferGeometry(0.4, 30, 30); + // const material = new THREE.MeshStandardMaterial({ color: 0xfdffbf}); + + // const sphere = new THREE.Mesh(geometry, material); + // sphere.position.set(0, 0.6, -1.5); + + // this.scene.add(sphere) + + const room = new THREE.LineSegments( + new BoxLineGeometry(6, 6, 6, 10, 10, 10), + new THREE.LineBasicMaterial({ + color: "#151515", + }) + ); + + room.geometry.translate(0,3,0); + + this.scene.add(room); + + + const controls = new OrbitControls( this.camera, this.renderer.domElement ); + + this.setupVR(); + + this.renderer.setAnimationLoop(this.render.bind(this)); + window.addEventListener('resize', this.resize.bind(this) ); + + } + + checkProximity(indexTip){ + let allspheres = [] + + for(let i = 0; i< this.spheres.length; i++){ + const sphere = this.spheres[ i ]; + var dx = indexTip.position.x - sphere.position.x; + var dy = indexTip.position.y - sphere.position.y; + var dz = indexTip.position.z - sphere.position.z; + const distance = Math.sqrt(dx*dx+dy*dy+dz*dz); + //console.log("Distance: " + distance); + if(distance <= 0.02 ){ + //console.log("move"); + allspheres.push(sphere) + } + } + return allspheres; + } + + onPinchStartRight(event) { + let timeStamp = new Date(); + let currentTimeStamp = timeStamp.getHours() + ":" + timeStamp.getMinutes() + ":" + timeStamp.getSeconds() + ":" + timeStamp.getMilliseconds(); + //console.log("RightHand Pinch Started at: " + currentTimeStamp); + const hand = event.target; + const indexTip = hand.joints[ 'index-finger-tip' ]; + this.drawFlag = true; + if(this.drawFlag == true){ + //console.log("drawing - right hand"); + } + //console.log("IndexTip RightHand Deets: " + JSON.stringify(indexTip, null, 4)); + } + + onPinchEndRight(event) { + this.drawFlag = false; + let timeStamp = new Date(); + let currentTimeStamp = timeStamp.getHours() + ":" + timeStamp.getMinutes() + ":" + timeStamp.getSeconds() + ":" + timeStamp.getMilliseconds(); + //console.log("Righthand Pinch Ended at: " + currentTimeStamp); + if(this.drawFlag == false) { + //console.log("stop drawing"); + } + } + + onPinchStartLeft(event) { + let timeStamp = new Date(); + let currentTimeStamp = timeStamp.getHours() + ":" + timeStamp.getMinutes() + ":" + timeStamp.getSeconds() + ":" + timeStamp.getMilliseconds(); + //console.log("LeftHand Pinch Started at: " + currentTimeStamp); + } + + onPinchEndLeft(event) { + let timeStamp = new Date(); + let currentTimeStamp = timeStamp.getHours() + ":" + timeStamp.getMinutes() + ":" + timeStamp.getSeconds() + ":" + timeStamp.getMilliseconds(); + //console.log("LeftHand Pinch Ended at: " + currentTimeStamp); + } + + resize(){ + this.camera.aspect = window.innerWidth / window.innerHeight; + this.camera.updateProjectionMatrix(); + this.renderer.setSize( window.innerWidth, window.innerHeight ); + } + render() { + this.renderer.render(this.scene, this.camera); + } + + createCamera(){ + const camera = new THREE.PerspectiveCamera( + 50, + window.innerWidth/window.innerHeight, + 0.1, + 100 + ); + + camera.position.set(0, 1.6, 3); + return camera; + } + + createScene() { + const scene = new THREE.Scene(); + scene.background = new THREE.Color( 0x808080 ); + return scene; + } + + createRenderer() { + const renderer = new THREE.WebGLRenderer({ antialias: true }); + renderer.setPixelRatio(window.devicePixelRatio); + renderer.setSize(window.innerWidth, window.innerHeight); + renderer.outputEncoding = THREE.sRGBEncoding; + renderer.xr.enabled = true; + return renderer; + } + + addLight(){ + const ambient = new THREE.HemisphereLight(0x606060, 0x404040); + this.scene.add(ambient); + const light = new THREE.DirectionalLight(0xffffff); + light.position.set(1,1,1).normalize(); + this.scene.add(light); + } + setupVR() { + this.renderer.xr.enabled = true; + document.body.appendChild(VRButton.createButton(this.renderer)); + } +} + +export default App; diff --git a/apps/hand-models/left.glb b/apps/hand-models/left.glb new file mode 100644 index 0000000..2159ffc Binary files /dev/null and b/apps/hand-models/left.glb differ diff --git a/apps/hand-models/profile.json b/apps/hand-models/profile.json new file mode 100644 index 0000000..d03c035 --- /dev/null +++ b/apps/hand-models/profile.json @@ -0,0 +1,4 @@ +{ + "profileId" : "generic-hand", + "overrides" : {} +} diff --git a/apps/hand-models/right.glb b/apps/hand-models/right.glb new file mode 100644 index 0000000..d4adfb2 Binary files /dev/null and b/apps/hand-models/right.glb differ diff --git a/apps/hand-webxr.js b/apps/hand-webxr.js new file mode 100644 index 0000000..ce88092 --- /dev/null +++ b/apps/hand-webxr.js @@ -0,0 +1,274 @@ + +import * as THREE from 'three'; +import { VRButton } from 'three/examples/jsm/webxr/VRButton'; +import { BoxLineGeometry } from 'three/examples/jsm/geometries/BoxLineGeometry'; +import { OrbitControls } from 'three/examples/jsm/controls/OrbitControls'; +import { XRControllerModelFactory } from 'three/examples/jsm/webxr/XRControllerModelFactory.js'; +import { XRHandModelFactory } from 'three/examples/jsm/webxr/XRHandModelFactory.js'; + +let App = class App{ + constructor(){ + const container = document.createElement( 'div' ); + document.body.appendChild( container ); + + let controllerRight, controllerLeft; + let controllerRightGrip, controllerLeftGrip; + + this.drawFlag = false; + this.spheres = []; + let grabbing = false; + + this.tmpVector1 = new THREE.Vector3(); + this.tmpVector2 = new THREE.Vector3(); + + /*Creating Camera, Scene and Renderer */ + this.camera = this.createCamera(); + this.scene = this.createScene(); + this.renderer = this.createRenderer(); + container.appendChild(this.renderer.domElement); + + this.controls = new OrbitControls( this.camera, container ); + this.controls.target.set( 0, 1.6, 0 ); + this.controls.update(); + + /*Initialising controllerModelFactory and handModelFactory from Three.js */ + const controllerModelFactory = new XRControllerModelFactory(); + const handModelFactory = new XRHandModelFactory().setPath( + "/hand-models" + );; //not sure about setPath ~ trying to fetch .glb files for hand models + + /* Setting up Right Hand from POV */ + controllerRight = this.renderer.xr.getController( 0 ); + this.scene.add( controllerRight ); + controllerRightGrip = this.renderer.xr.getControllerGrip( 0 ); + controllerRightGrip.add( controllerModelFactory.createControllerModel( controllerRightGrip ) ); + this.scene.add( controllerRightGrip ); + + this.rightHand = this.renderer.xr.getHand( 0 ); + this.rightHand.add( handModelFactory.createHandModel( this.rightHand) ); + this.scene.add( this.rightHand ); + + /*Event Listeners for 'Pinch' detection in the Right Hand */ + // this.rightHand.addEventListener( 'pinchstart', this.onPinchStartRight); + this.rightHand.addEventListener('pinchstart', (event) => { + let timeStamp = new Date(); + let currentTimeStamp = timeStamp.getHours() + ":" + timeStamp.getMinutes() + ":" + timeStamp.getSeconds() + ":" + timeStamp.getMilliseconds(); + console.log("RightHand Pinch Started at: " + currentTimeStamp); + const hand = event.target; + const indexTip = hand.joints[ 'index-finger-tip' ]; + this.drawFlag = true; + if(this.drawFlag == true){ + console.log("drawing - right hand"); + console.log("IndexTip RightHand Deets: " + JSON.stringify(indexTip, null, 4)); + + console.log(indexTip.position); + const geometryR = new THREE.SphereBufferGeometry(0.01, 30, 30); + const materialR = new THREE.MeshStandardMaterial({color: 0x000000}); + + const sphereR = new THREE.Mesh(geometryR, materialR); + sphereR.position.set(indexTip.position.x, indexTip.position.y, indexTip.position.z); + this.spheres.push(sphereR); + console.log(this.spheres); + + this.scene.add(sphereR); + } + + }) + this.rightHand.addEventListener( 'pinchend', this.onPinchEndRight); + + /* Setting up Left Hand from POV */ + controllerLeft = this.renderer.xr.getController( 1 ); + this.scene.add( controllerLeft ); + controllerLeftGrip = this.renderer.xr.getControllerGrip( 1 ); + controllerLeftGrip.add( controllerModelFactory.createControllerModel( controllerLeftGrip ) ); + this.scene.add( controllerLeftGrip ); + + this.leftHand = this.renderer.xr.getHand( 1 ); + this.leftHand.add( handModelFactory.createHandModel( this.leftHand ) ); + this.scene.add( this.leftHand ); + + this.currentSphere; + + /*Event Listeners for 'Pinch' detection in the Right Hand */ + this.leftHand.addEventListener( 'pinchstart', (event) => { + let timeStamp = new Date(); + let currentTimeStamp = timeStamp.getHours() + ":" + timeStamp.getMinutes() + ":" + timeStamp.getSeconds() + ":" + timeStamp.getMilliseconds(); + console.log("LeftHand Pinch Started at: " + currentTimeStamp); + const hand = event.target; + const indexTip = hand.joints[ 'index-finger-tip' ]; + this.drawFlag = true; + // if(this.drawFlag == true){ + // console.log("drawing - left hand"); + // console.log("Index Tip Position: " + indexTip.position); + + // const geometryL = new THREE.SphereBufferGeometry(0.01, 30, 30); + // const materialL = new THREE.MeshStandardMaterial({color: 0xff0000}); + + // const sphereL = new THREE.Mesh(geometryL, materialL); + // sphereL.position.set(indexTip.position.x, indexTip.position.y, indexTip.position.z); + + // this.scene.add(sphereL); + // } + const sphereInProximity = this.checkProximity(indexTip); + if(sphereInProximity) { + grabbing = true; + this.currentSphere = sphereInProximity; + console.log("hereL : " + JSON.stringify(this.currentSphere, null, 4)); + indexTip.attach(sphereInProximity); + } + }); + + this.leftHand.addEventListener( 'pinchend', (event) => { + grabbing = false; + const hand = event.target; + const indexTip = hand.joints[ 'index-finger-tip' ]; + console.log("CS: " + JSON.stringify(this.currentSphere, null, 4)); + // this.currentSphere.position.set(indexTip.position); + //this.scene.add(this.currentSphere); + }); + + + this.addLight(); + + // const geometry = new THREE.SphereBufferGeometry(0.4, 30, 30); + // const material = new THREE.MeshStandardMaterial({ color: 0xfdffbf}); + + // const sphere = new THREE.Mesh(geometry, material); + // sphere.position.set(0, 0.6, -1.5); + + // this.scene.add(sphere) + + const room = new THREE.LineSegments( + new BoxLineGeometry(6, 6, 6, 10, 10, 10), + new THREE.LineBasicMaterial({ + color: "#151515", + }) + ); + + room.geometry.translate(0,3,0); + + this.scene.add(room); + + + const controls = new OrbitControls( this.camera, this.renderer.domElement ); + + this.setupVR(); + + this.renderer.setAnimationLoop(this.render.bind(this)); + window.addEventListener('resize', this.resize.bind(this) ); + + } + + render(frame) { + console.log("RENDER FRAME: " + frame); + const xr = this.renderer.xr; + const session = xr.getSession(); + + this.renderer.render(this.scene, this.camera); + + } + + checkProximity(indexTip){ + for(let i = 0; i< this.spheres.length; i++){ + const sphere = this.spheres[ i ]; + var dx = indexTip.position.x - sphere.position.x; + var dy = indexTip.position.y - sphere.position.y; + var dz = indexTip.position.z - sphere.position.z; + const distance = Math.sqrt(dx*dx+dy*dy+dz*dz); + console.log("Distance: " + distance); + if(distance <= 0.02 ){ + console.log("move"); + return sphere; + } + } + return null; + } + + onPinchStartRight(event) { + let timeStamp = new Date(); + let currentTimeStamp = timeStamp.getHours() + ":" + timeStamp.getMinutes() + ":" + timeStamp.getSeconds() + ":" + timeStamp.getMilliseconds(); + console.log("RightHand Pinch Started at: " + currentTimeStamp); + const hand = event.target; + const indexTip = hand.joints[ 'index-finger-tip' ]; + this.drawFlag = true; + if(this.drawFlag == true){ + console.log("drawing - right hand"); + } + console.log("IndexTip RightHand Deets: " + JSON.stringify(indexTip, null, 4)); + } + + onPinchEndRight(event) { + this.drawFlag = false; + let timeStamp = new Date(); + let currentTimeStamp = timeStamp.getHours() + ":" + timeStamp.getMinutes() + ":" + timeStamp.getSeconds() + ":" + timeStamp.getMilliseconds(); + console.log("Righthand Pinch Ended at: " + currentTimeStamp); + if(this.drawFlag == false) { + console.log("stop drawing"); + } + } + + onPinchStartLeft(event) { + let timeStamp = new Date(); + let currentTimeStamp = timeStamp.getHours() + ":" + timeStamp.getMinutes() + ":" + timeStamp.getSeconds() + ":" + timeStamp.getMilliseconds(); + console.log("LeftHand Pinch Started at: " + currentTimeStamp); + const hand = event.target; + const indexTip = hand.joints[ 'index-finger-tip' ]; + //console.log("IndexTip RightHand Deets: " + JSON.stringify(indexTip, null, 4)); + } + + onPinchEndLeft(event) { + let timeStamp = new Date(); + let currentTimeStamp = timeStamp.getHours() + ":" + timeStamp.getMinutes() + ":" + timeStamp.getSeconds() + ":" + timeStamp.getMilliseconds(); + console.log("LeftHand Pinch Ended at: " + currentTimeStamp); + } + + resize(){ + this.camera.aspect = window.innerWidth / window.innerHeight; + this.camera.updateProjectionMatrix(); + this.renderer.setSize( window.innerWidth, window.innerHeight ); + } + render() { + this.renderer.render(this.scene, this.camera); + } + + createCamera(){ + const camera = new THREE.PerspectiveCamera( + 50, + window.innerWidth/window.innerHeight, + 0.1, + 100 + ); + + camera.position.set(0, 1.6, 3); + return camera; + } + + createScene() { + const scene = new THREE.Scene(); + scene.background = new THREE.Color( 0x808080 ); + return scene; + } + + createRenderer() { + const renderer = new THREE.WebGLRenderer({ antialias: true }); + renderer.setPixelRatio(window.devicePixelRatio); + renderer.setSize(window.innerWidth, window.innerHeight); + renderer.outputEncoding = THREE.sRGBEncoding; + renderer.xr.enabled = true; + return renderer; + } + + addLight(){ + const ambient = new THREE.HemisphereLight(0x606060, 0x404040); + this.scene.add(ambient); + const light = new THREE.DirectionalLight(0xffffff); + light.position.set(1,1,1).normalize(); + this.scene.add(light); + } + setupVR() { + this.renderer.xr.enabled = true; + document.body.appendChild(VRButton.createButton(this.renderer)); + } +} + +export default App; diff --git a/apps/index.js b/apps/index.js index 1b213dd..bc7f626 100644 --- a/apps/index.js +++ b/apps/index.js @@ -1,5 +1,16 @@ -export { default as ControllerInteraction } from "./controller-interaction"; +export { default as ControllerInteraction } from "./controller-interactions/controller-interaction"; export { default as HandInput } from "./simple-video-layers/hand-input"; -export { default as SimpleScene } from "./simple-scene"; +export { default as SimpleScene } from "./simple-scene/simple-scene"; export { default as SimpleEquirectMediaLayer } from "./simple-video-layers/equirect"; export { default as MultipleLayers } from "./simple-video-layers/multiple-layers"; + +// WebXR Hand Inputs +export { default as SimpleHands } from "./webxr-hands/simple-hands"; +export { default as HandDrawing } from "./webxr-hands/hand-pinch-drawing"; +export { default as TouchObject } from './webxr-hands/touch-object'; +export { default as HandGrabbing } from "./hand-grabbing-three"; +export { default as HandGrabbingDist } from "./hand-grabbing-dist"; +export { default as ResizeObject } from './webxr-hands/resize-object'; +export { default as SnapFingers } from './webxr-hands/snap-fingers'; +export { default as EjectWeb } from './webxr-hands/eject-web'; + diff --git a/apps/simple-scene.js b/apps/simple-scene/simple-scene.js similarity index 100% rename from apps/simple-scene.js rename to apps/simple-scene/simple-scene.js diff --git a/apps/webxr-hands/eject-web.js b/apps/webxr-hands/eject-web.js new file mode 100644 index 0000000..aaa8c17 --- /dev/null +++ b/apps/webxr-hands/eject-web.js @@ -0,0 +1,247 @@ + +import * as THREE from 'three'; +import { VRButton } from 'three/examples/jsm/webxr/VRButton'; +import { BoxLineGeometry } from 'three/examples/jsm/geometries/BoxLineGeometry'; +import { OrbitControls } from 'three/examples/jsm/controls/OrbitControls'; +import { XRControllerModelFactory } from 'three/examples/jsm/webxr/XRControllerModelFactory.js'; +import { XRHandModelFactory } from 'three/examples/jsm/webxr/XRHandModelFactory.js'; + +let App = class App { + constructor() { + const container = document.createElement('div'); + document.body.appendChild(container); + + /*Creating Camera, Scene and Renderer */ + this.camera = this.createCamera(); + this.scene = this.createScene(); + this.renderer = this.createRenderer(); + container.appendChild(this.renderer.domElement); + + // Adding Lights + this.addLight(); + + const room = new THREE.LineSegments( + new BoxLineGeometry(6, 6, 6, 10, 10, 10), + new THREE.LineBasicMaterial({ + color: "#151515", + }) + ); + room.geometry.translate(0, 3, 0); + this.scene.add(room); + + this.controls = new OrbitControls(this.camera, container); + this.controls.target.set(0, 1.6, 0); + this.controls.update(); + + this.session; + this.renderer.xr.addEventListener("sessionstart", (event) => { + this.session = this.renderer.xr.getSession(); + }); + this.renderer.xr.addEventListener("sessionend", (event) => { + this.session = null; + }); + + // Build Hands + this.rightHand; + this.leftHand; + this.buildHands(0); // Right Hand + this.buildHands(1); // Left Hand + this.plasmaBalls = []; //eject-plase-buffer + + //this.midAndThumbReady = false; + //this.didSnap = false; + this.spiderManPoseActive = false; + this.leftPinch = false; + + this.spheres = [] + + this.setupVR(); + this.renderer.setAnimationLoop(this.render.bind(this)); + window.addEventListener('resize', this.resize.bind(this)); + + } + + buildHands(thehand){ + let controller; + let controllerGrip; + + /*Initialising controllerModelFactory and handModelFactory from Three.js */ + const controllerModelFactory = new XRControllerModelFactory(); + const handModelFactory = new XRHandModelFactory().setPath( + "../hand-models" + ); + /* Setting up Hand from POV */ + controller = this.renderer.xr.getController(thehand); + this.scene.add(controller); + controllerGrip = this.renderer.xr.getControllerGrip(thehand); + controllerGrip.add(controllerModelFactory.createControllerModel(controllerGrip)); + this.scene.add(controllerGrip); + if (thehand == 0){ + this.rightHand = this.renderer.xr.getHand(0); + this.rightHand.add(handModelFactory.createHandModel(this.rightHand)); + this.scene.add(this.rightHand); + } + else{ + this.leftHand = this.renderer.xr.getHand(1); + this.leftHand.add(handModelFactory.createHandModel(this.leftHand)); + this.scene.add(this.leftHand); + } + } + + getHandVisibilityStatus() { + if (this.session) { + for (const inputSource of this.session.inputSources) { + if (inputSource.hand) { + let name = inputSource.handedness; + let theHand; + if (name === 'right'){ + theHand = this.rightHand; + let middleFingerTip = theHand.joints['middle-finger-tip']; + let ringFingerTip = theHand.joints['ring-finger-tip']; + let indexFingerMetaCarpal = theHand.joints['index-finger-metacarpal']; + let middleFingerMetaCarpal = theHand.joints['middle-finger-metacarpal']; + let ringFingerMetaCarpal = theHand.joints['ring-finger-metacarpal']; + let indexFinger = theHand.joints['index-finger-tip']; + let pinkyFinger = theHand.joints['pinky-finger-tip']; + //console.log("MFMC: " + JSON.stringify(middleFingerMetaCarpal, null, 4)) + this.checkSpidermanPose(middleFingerTip, ringFingerTip, indexFingerMetaCarpal, middleFingerMetaCarpal, ringFingerMetaCarpal, indexFinger, pinkyFinger); + } + else if (name === 'left'){ + theHand = this.leftHand; + let middleFingerTip = theHand.joints['middle-finger-tip']; + let ringFingerTip = theHand.joints['ring-finger-tip']; + let indexFingerMetaCarpal = theHand.joints['index-finger-metacarpal'] + let middleFingerMetaCarpal = theHand.joints['middle-finger-metacarpal'] + let ringFingerMetaCarpal = theHand.joints['ring-finger-metacarpal'] + let indexFinger = theHand.joints['index-finger-tip']; + let pinkyFinger = theHand.joints['pinky-finger-tip']; + this.checkSpidermanPose(middleFingerTip, ringFingerTip, indexFingerMetaCarpal, middleFingerMetaCarpal, ringFingerMetaCarpal, indexFinger, pinkyFinger); + + } + else{ + console.log('Hands not being tracked...') + } + } + } + } + } + + checkSpidermanPose(middleFingerTip, ringFingerTip, indexFingerMetaCarpal, middleFingerMetaCarpal, ringFingerMetaCarpal, indexFinger, pinkyFinger){ + /*Calculate the distance between the tip of the middle finger and tip of the ring finger */ + let diffMidRingX = Math.abs(middleFingerTip.position.x - ringFingerTip.position.x) + let diffMidRingY = Math.abs(middleFingerTip.position.y - ringFingerTip.position.y) + let diffMidRingZ = Math.abs(middleFingerTip.position.z - ringFingerTip.position.z) + + if(diffMidRingX!=0 || diffMidRingY || diffMidRingZ!=0 ) + { + if(diffMidRingX <= 0.02 && diffMidRingY <= 0.02 && diffMidRingZ<=0.02){ + //fingers are touching, check distance of each finger from respective carpel + //console.log("MF Car: " + middleFingerMetaCarpal.position.x + " " + middleFingerMetaCarpal.position.y + " " + middleFingerMetaCarpal.position.z); + //console.log("RF Car: " + ringFingerMetaCarpal.position.x + " " + ringFingerMetaCarpal.position.y + " " + ringFingerMetaCarpal.position.z); + let diffMidCarpalX = Math.abs(middleFingerTip.position.x - middleFingerMetaCarpal.position.x) + let diffMidCarpalY = Math.abs(middleFingerTip.position.y - middleFingerMetaCarpal.position.y) + let diffMidCarpalZ = Math.abs(middleFingerTip.position.z - middleFingerMetaCarpal.position.z) + + let diffRingCarpalX = Math.abs(ringFingerTip.position.x - ringFingerMetaCarpal.position.x) + let diffRingCarpalY = Math.abs(ringFingerTip.position.x - ringFingerMetaCarpal.position.x) + let diffRingCarpalZ = Math.abs(ringFingerTip.position.x - ringFingerMetaCarpal.position.x) + + // console.log("MiddFinger: " + diffMidCarpalX + diffMidCarpalY + diffMidCarpalZ); + //console.log("Ring Finger: " + diffRingCarpalX + diffRingCarpalY + diffRingCarpalZ); + if(diffRingCarpalX <=0.02 && diffRingCarpalY <=0.05 && diffRingCarpalZ<=0.02){ + if( diffMidCarpalX<=0.02 && diffMidCarpalY <=0.05 && diffMidCarpalZ<=0.02){ + console.log("Spidey Pose Detected"); + this.spiderManPoseActive = true; + this.spidey(indexFinger, pinkyFinger); + } else { + this.spiderManPoseActive = false; + // for (let i = 0; i < this.plasmaBalls; i++) { + // this.scene.remove(this.plasmaBalls[i]); + // this.plasmaBalls = this.spheres.filter(item => item !== this.spheres[i]) + // } + } + } else { + this.spiderManPoseActive = false; + } + } + } + + } + + + spidey(indexFinger, pinkyFinger){ + console.log("inside spidey func"); + const geometryR = new THREE.SphereBufferGeometry(0.01, 30, 30); + const materialR = new THREE.MeshStandardMaterial({color: 0x4e5354}); + const plasmaBall = new THREE.Mesh(geometryR, materialR); + plasmaBall.position.set(((indexFinger.position.x + pinkyFinger.position.x) / 2), indexFinger.position.y, indexFinger.position.z);// start position - the tip of the indexFinger + plasmaBall.quaternion.copy(this.camera.quaternion); // apply camera's quaternion + this.scene.add(plasmaBall); + this.plasmaBalls.push(plasmaBall); + } + + resize() { + this.camera.aspect = window.innerWidth / window.innerHeight; + this.camera.updateProjectionMatrix(); + this.renderer.setSize(window.innerWidth, window.innerHeight); + } + + render() { + this.renderer.render(this.scene, this.camera); + this.getHandVisibilityStatus(); + if(this.plasmaBalls.length>0 && this.spiderManPoseActive == true){ + console.log("ejecting"); + this.plasmaBalls.forEach(b => { + console.log(b); + b.translateZ(-0.05); // move along the local z-axis + }); + + } + else{ + this.plasmaBalls.forEach(b => { + this.scene.remove(b); + }); + this.plasmaBalls = [] + } + } + + createCamera() { + const camera = new THREE.PerspectiveCamera( + 50, + window.innerWidth / window.innerHeight, + 0.1, + 100 + ); + camera.position.set(0, 1.6, 3); + return camera; + } + + createScene() { + const scene = new THREE.Scene(); + scene.background = new THREE.Color(0x808080); + return scene; + } + + createRenderer() { + const renderer = new THREE.WebGLRenderer({ antialias: true }); + renderer.setPixelRatio(window.devicePixelRatio); + renderer.setSize(window.innerWidth, window.innerHeight); + renderer.outputEncoding = THREE.sRGBEncoding; + renderer.xr.enabled = true; + return renderer; + } + + addLight() { + const ambient = new THREE.HemisphereLight(0x606060, 0x404040); + this.scene.add(ambient); + const light = new THREE.DirectionalLight(0xffffff); + light.position.set(1, 1, 1).normalize(); + this.scene.add(light); + } + setupVR() { + this.renderer.xr.enabled = true; + document.body.appendChild(VRButton.createButton(this.renderer)); + } +} + +export default App; diff --git a/apps/webxr-hands/hand-pinch-drawing.js b/apps/webxr-hands/hand-pinch-drawing.js new file mode 100644 index 0000000..75cef6b --- /dev/null +++ b/apps/webxr-hands/hand-pinch-drawing.js @@ -0,0 +1,182 @@ + +import * as THREE from 'three'; +import { VRButton } from 'three/examples/jsm/webxr/VRButton'; +import { BoxLineGeometry } from 'three/examples/jsm/geometries/BoxLineGeometry'; +import { OrbitControls } from 'three/examples/jsm/controls/OrbitControls'; +import { XRControllerModelFactory } from 'three/examples/jsm/webxr/XRControllerModelFactory.js'; +import { XRHandModelFactory } from 'three/examples/jsm/webxr/XRHandModelFactory.js'; + +let App = class App { + constructor() { + const container = document.createElement('div'); + document.body.appendChild(container); + + /*Creating Camera, Scene and Renderer */ + this.camera = this.createCamera(); + this.scene = this.createScene(); + this.renderer = this.createRenderer(); + container.appendChild(this.renderer.domElement); + + // Adding Lights + this.addLight(); + + const room = new THREE.LineSegments( + new BoxLineGeometry(6, 6, 6, 10, 10, 10), + new THREE.LineBasicMaterial({ + color: "#151515", + }) + ); + room.geometry.translate(0, 3, 0); + this.scene.add(room); + + this.controls = new OrbitControls(this.camera, container); + this.controls.target.set(0, 1.6, 0); + this.controls.update(); + + this.session; + this.renderer.xr.addEventListener("sessionstart", (event) => { + this.session = this.renderer.xr.getSession(); + }); + this.renderer.xr.addEventListener("sessionend", (event) => { + this.session = null; + }); + + // Build Hands + this.rightHand; + this.leftHand; + this.buildHands(0); // Right Hand + this.buildHands(1); // Left Hand + + + this.setupVR(); + this.renderer.setAnimationLoop(this.render.bind(this)); + window.addEventListener('resize', this.resize.bind(this)); + + } + + buildHands(thehand){ + let controller; + let controllerGrip; + + /*Initialising controllerModelFactory and handModelFactory from Three.js */ + const controllerModelFactory = new XRControllerModelFactory(); + const handModelFactory = new XRHandModelFactory().setPath( + "../hand-models" + ); + /* Setting up Hand from POV */ + controller = this.renderer.xr.getController(thehand); + this.scene.add(controller); + controllerGrip = this.renderer.xr.getControllerGrip(thehand); + controllerGrip.add(controllerModelFactory.createControllerModel(controllerGrip)); + this.scene.add(controllerGrip); + if (thehand == 0){ + this.rightHand = this.renderer.xr.getHand(0); + this.rightHand.add(handModelFactory.createHandModel(this.rightHand)); + this.scene.add(this.rightHand); + } + else{ + this.leftHand = this.renderer.xr.getHand(1); + this.leftHand.add(handModelFactory.createHandModel(this.leftHand)); + this.scene.add(this.leftHand); + } + } + + getHandVisibilityStatus() { + if (this.session) { + for (const inputSource of this.session.inputSources) { + if (inputSource.hand) { + let name = inputSource.handedness; + let theHand; + if (name === 'right'){ + theHand = this.rightHand; + let indexTip = theHand.joints['index-finger-tip']; + let thumbTip = theHand.joints['thumb-tip']; + this.checkPinch(indexTip, thumbTip, name) + } + else if (name === 'left'){ + theHand = this.leftHand; + let indexTip = theHand.joints['index-finger-tip']; + let thumbTip = theHand.joints['thumb-tip']; + this.checkPinch(indexTip, thumbTip, name) + } + else{ + console.log('Hands not being tracked...') + } + } + } + } + } + + checkPinch(thumbTip, indexTip, hand) { + // Calculate the distance between positions of Index Tip and Thubm Tip + let diffX = Math.abs(indexTip.position.x - thumbTip.position.x) + let diffY = Math.abs(indexTip.position.y - thumbTip.position.y) + let diffZ = Math.abs(indexTip.position.z - thumbTip.position.z) + + if (diffX!=0 || diffY!=0 || diffZ!=0){ + // When hands are not seen, the diffs initialize at zeroes. Stops once hands are seen. + if (diffX < 0.02 && diffY < 0.02 && diffZ < 0.02) { + console.log(diffX, diffY, diffZ) + console.log(hand + "PINCHING") + let geometryPinch = new THREE.SphereBufferGeometry(0.01, 30, 30); + let materialPinch = new THREE.MeshStandardMaterial({ + color: hand == 'right' ? 0xfdffbf : Math.random() * 0xffffff + }); + + let spherePinch = new THREE.Mesh(geometryPinch, materialPinch); + spherePinch.position.set(indexTip.position.x, indexTip.position.y, indexTip.position.z); + this.scene.add(spherePinch); + } + } + } + + resize() { + this.camera.aspect = window.innerWidth / window.innerHeight; + this.camera.updateProjectionMatrix(); + this.renderer.setSize(window.innerWidth, window.innerHeight); + } + render() { + this.renderer.render(this.scene, this.camera); + this.getHandVisibilityStatus(); + } + + createCamera() { + const camera = new THREE.PerspectiveCamera( + 50, + window.innerWidth / window.innerHeight, + 0.1, + 100 + ); + camera.position.set(0, 1.6, 3); + return camera; + } + + createScene() { + const scene = new THREE.Scene(); + scene.background = new THREE.Color(0x808080); + return scene; + } + + createRenderer() { + const renderer = new THREE.WebGLRenderer({ antialias: true }); + renderer.setPixelRatio(window.devicePixelRatio); + renderer.setSize(window.innerWidth, window.innerHeight); + renderer.outputEncoding = THREE.sRGBEncoding; + renderer.xr.enabled = true; + return renderer; + } + + addLight() { + const ambient = new THREE.HemisphereLight(0x606060, 0x404040); + this.scene.add(ambient); + const light = new THREE.DirectionalLight(0xffffff); + light.position.set(1, 1, 1).normalize(); + this.scene.add(light); + } + setupVR() { + this.renderer.xr.enabled = true; + document.body.appendChild(VRButton.createButton(this.renderer)); + } +} + +export default App; diff --git a/apps/webxr-hands/resize-object.js b/apps/webxr-hands/resize-object.js new file mode 100644 index 0000000..5b9665c --- /dev/null +++ b/apps/webxr-hands/resize-object.js @@ -0,0 +1,229 @@ + +import * as THREE from 'three'; +import { VRButton } from 'three/examples/jsm/webxr/VRButton'; +import { BoxLineGeometry } from 'three/examples/jsm/geometries/BoxLineGeometry'; +import { OrbitControls } from 'three/examples/jsm/controls/OrbitControls'; +import { XRControllerModelFactory } from 'three/examples/jsm/webxr/XRControllerModelFactory.js'; +import { XRHandModelFactory } from 'three/examples/jsm/webxr/XRHandModelFactory.js'; + +const tmpVector1 = new THREE.Vector3(); +const tmpVector2 = new THREE.Vector3(); + +let App = class App { + constructor() { + const container = document.createElement('div'); + document.body.appendChild(container); + + /*Creating Camera, Scene and Renderer */ + this.camera = this.createCamera(); + this.scene = this.createScene(); + this.renderer = this.createRenderer(); + container.appendChild(this.renderer.domElement); + + // Adding Lights + this.addLight(); + + const geometry = new THREE.SphereBufferGeometry(0.1, 30, 30); + const material = new THREE.MeshStandardMaterial({ color: 0xfdffbf }); + this.sphere = new THREE.Mesh(geometry, material); + this.sphere.position.set(0, 0.6, -0.2); + this.scene.add(this.sphere) + + const room = new THREE.LineSegments( + new BoxLineGeometry(6, 6, 6, 10, 10, 10), + new THREE.LineBasicMaterial({ + color: "#151515", + }) + ); + room.geometry.translate(0, 3, 0); + this.scene.add(room); + + this.controls = new OrbitControls(this.camera, container); + this.controls.target.set(0, 1.6, 0); + this.controls.update(); + + this.session; + this.renderer.xr.addEventListener("sessionstart", (event) => { + this.session = this.renderer.xr.getSession(); + }); + this.renderer.xr.addEventListener("sessionend", (event) => { + this.session = null; + }); + + // Build Hands + this.rightHand; + this.leftHand; + this.buildHands(0); // Right Hand + this.buildHands(1); // Left Hand + + this.rightHandSelection = false; + this.leftHandSelection = false; + this.initialDistance = 0; + + this.setupVR(); + this.renderer.setAnimationLoop(this.render.bind(this)); + window.addEventListener('resize', this.resize.bind(this)); + + } + + buildHands(thehand){ + let controller; + let controllerGrip; + + /*Initialising controllerModelFactory and handModelFactory from Three.js */ + const controllerModelFactory = new XRControllerModelFactory(); + const handModelFactory = new XRHandModelFactory().setPath( + "../hand-models" + ); + /* Setting up Hand from POV */ + controller = this.renderer.xr.getController(thehand); + this.scene.add(controller); + controllerGrip = this.renderer.xr.getControllerGrip(thehand); + controllerGrip.add(controllerModelFactory.createControllerModel(controllerGrip)); + this.scene.add(controllerGrip); + if (thehand == 0){ + this.rightHand = this.renderer.xr.getHand(0); + this.rightHand.add(handModelFactory.createHandModel(this.rightHand)); + this.scene.add(this.rightHand); + } + else{ + this.leftHand = this.renderer.xr.getHand(1); + this.leftHand.add(handModelFactory.createHandModel(this.leftHand)); + this.scene.add(this.leftHand); + } + } + + getHandVisibilityStatus() { + if (this.session) { + for (const inputSource of this.session.inputSources) { + if (inputSource.hand) { + let name = inputSource.handedness; + let theHand; + if (name === 'right'){ + theHand = this.rightHand; + let indexTip = theHand.joints['index-finger-tip']; + let thumbTip = theHand.joints['thumb-tip']; + this.checkSelection(indexTip, thumbTip, name) + } + else if (name === 'left'){ + theHand = this.leftHand; + let indexTip = theHand.joints['index-finger-tip']; + let thumbTip = theHand.joints['thumb-tip']; + this.checkSelection(indexTip, thumbTip, name) + } + else{ + console.log('Hands not being tracked...') + } + } + } + } + } + + checkSelection(thumbTip, indexTip, hand) { + // Calculate the distance between positions of Index Tip and Thubm Tip + let diffX = Math.abs(indexTip.position.x - thumbTip.position.x) + let diffY = Math.abs(indexTip.position.y - thumbTip.position.y) + let diffZ = Math.abs(indexTip.position.z - thumbTip.position.z) + + // First check for pinching... + if (diffX!=0 || diffY!=0 || diffZ!=0){ + // When hands are not seen, the diffs initialize at zeroes. Stops once hands are seen. + if (diffX < 0.02 && diffY < 0.02 && diffZ < 0.02) { + + // Check for selection + let distance = indexTip.getWorldPosition(tmpVector1).distanceTo(this.sphere.getWorldPosition(tmpVector2)); + if (distance < this.sphere.geometry.boundingSphere.radius * this.sphere.scale.x) { + // console.log(distance) + // console.log(this.sphere.geometry.boundingSphere.radius * this.sphere.scale.x) + if(hand == 'right'){ + this.rightHandSelection = true; + } + if(hand == 'left'){ + this.leftHandSelection = true; + } + } + // else{ + // this.rightHandSelection = false; + // this.leftHandSelection = false; + // } + } + else{ + this.rightHandSelection = false; + this.leftHandSelection = false; + } + } + } + + resizeObject(){ + if (this.rightHandSelection == true && this.leftHandSelection == true){ + console.log('Double Touch Detected...'); + let rightIndexTip = this.rightHand.joints['index-finger-tip']; + let leftIndexTip = this.leftHand.joints['index-finger-tip']; + + let dx = rightIndexTip.position.x - leftIndexTip.position.x; + let dy = rightIndexTip.position.y - leftIndexTip.position.y; + let dz = rightIndexTip.position.z - leftIndexTip.position.z; + let distance = Math.sqrt(dx*dx+dy*dy+dz*dz); + + this.initialDistance = this.initialDistance != 0 ? this.initialDistance : distance; + + let theRatio = distance / this.initialDistance; + + this.sphere.scale.set(theRatio, theRatio, theRatio) + } + else { + this.initialDistance = 0; + } + } + + resize() { + this.camera.aspect = window.innerWidth / window.innerHeight; + this.camera.updateProjectionMatrix(); + this.renderer.setSize(window.innerWidth, window.innerHeight); + } + render() { + this.renderer.render(this.scene, this.camera); + this.getHandVisibilityStatus(); + this.resizeObject(); + } + + createCamera() { + const camera = new THREE.PerspectiveCamera( + 50, + window.innerWidth / window.innerHeight, + 0.1, + 100 + ); + camera.position.set(0, 1.6, 3); + return camera; + } + + createScene() { + const scene = new THREE.Scene(); + scene.background = new THREE.Color(0x808080); + return scene; + } + + createRenderer() { + const renderer = new THREE.WebGLRenderer({ antialias: true }); + renderer.setPixelRatio(window.devicePixelRatio); + renderer.setSize(window.innerWidth, window.innerHeight); + renderer.outputEncoding = THREE.sRGBEncoding; + renderer.xr.enabled = true; + return renderer; + } + + addLight() { + const ambient = new THREE.HemisphereLight(0x606060, 0x404040); + this.scene.add(ambient); + const light = new THREE.DirectionalLight(0xffffff); + light.position.set(1, 1, 1).normalize(); + this.scene.add(light); + } + setupVR() { + this.renderer.xr.enabled = true; + document.body.appendChild(VRButton.createButton(this.renderer)); + } +} + +export default App; diff --git a/apps/webxr-hands/simple-hands.js b/apps/webxr-hands/simple-hands.js new file mode 100644 index 0000000..f3ac54f --- /dev/null +++ b/apps/webxr-hands/simple-hands.js @@ -0,0 +1,152 @@ + +import * as THREE from 'three'; +import { VRButton } from 'three/examples/jsm/webxr/VRButton'; +import { BoxLineGeometry } from 'three/examples/jsm/geometries/BoxLineGeometry'; +import { OrbitControls } from 'three/examples/jsm/controls/OrbitControls'; +import { XRControllerModelFactory } from 'three/examples/jsm/webxr/XRControllerModelFactory.js'; +import { XRHandModelFactory } from 'three/examples/jsm/webxr/XRHandModelFactory.js'; + +let App = class App { + constructor() { + const container = document.createElement('div'); + document.body.appendChild(container); + + /*Creating Camera, Scene and Renderer */ + this.camera = this.createCamera(); + this.scene = this.createScene(); + this.renderer = this.createRenderer(); + container.appendChild(this.renderer.domElement); + + // Adding Lights + this.addLight(); + + const room = new THREE.LineSegments( + new BoxLineGeometry(6, 6, 6, 10, 10, 10), + new THREE.LineBasicMaterial({ + color: "#151515", + }) + ); + room.geometry.translate(0, 3, 0); + this.scene.add(room); + + this.controls = new OrbitControls(this.camera, container); + this.controls.target.set(0, 1.6, 0); + this.controls.update(); + + this.session; + this.renderer.xr.addEventListener("sessionstart", (event) => { + this.session = this.renderer.xr.getSession(); + }); + this.renderer.xr.addEventListener("sessionend", (event) => { + this.session = null; + }); + + // Build Hands + this.rightHand; + this.leftHand; + this.buildHands(0); // Right Hand + this.buildHands(1); // Left Hand + + + this.setupVR(); + this.renderer.setAnimationLoop(this.render.bind(this)); + window.addEventListener('resize', this.resize.bind(this)); + + } + + buildHands(thehand){ + let controller; + let controllerGrip; + + /*Initialising controllerModelFactory and handModelFactory from Three.js */ + const controllerModelFactory = new XRControllerModelFactory(); + const handModelFactory = new XRHandModelFactory().setPath( + "../hand-models" + ); + /* Setting up Hand from POV */ + controller = this.renderer.xr.getController(thehand); + this.scene.add(controller); + controllerGrip = this.renderer.xr.getControllerGrip(thehand); + controllerGrip.add(controllerModelFactory.createControllerModel(controllerGrip)); + this.scene.add(controllerGrip); + if (thehand == 0){ + this.rightHand = this.renderer.xr.getHand(0); + this.rightHand.add(handModelFactory.createHandModel(this.rightHand)); + this.scene.add(this.rightHand); + } + else{ + this.leftHand = this.renderer.xr.getHand(1); + this.leftHand.add(handModelFactory.createHandModel(this.leftHand)); + this.scene.add(this.leftHand); + } + } + + getHandVisibilityStatus() { + if (this.session) { + for (const inputSource of this.session.inputSources) { + if (inputSource.hand) { + let name = inputSource.handedness; + if (name === 'right'){ + console.log('Right Hand Tracking...') + } + else if (name === 'left'){ + console.log('Left Hand Tracking...') + } + else{ + console.log('Hands not being tracked...') + } + } + } + } + } + + resize() { + this.camera.aspect = window.innerWidth / window.innerHeight; + this.camera.updateProjectionMatrix(); + this.renderer.setSize(window.innerWidth, window.innerHeight); + } + render() { + this.renderer.render(this.scene, this.camera); + this.getHandVisibilityStatus(); + } + + createCamera() { + const camera = new THREE.PerspectiveCamera( + 50, + window.innerWidth / window.innerHeight, + 0.1, + 100 + ); + camera.position.set(0, 1.6, 3); + return camera; + } + + createScene() { + const scene = new THREE.Scene(); + scene.background = new THREE.Color(0x808080); + return scene; + } + + createRenderer() { + const renderer = new THREE.WebGLRenderer({ antialias: true }); + renderer.setPixelRatio(window.devicePixelRatio); + renderer.setSize(window.innerWidth, window.innerHeight); + renderer.outputEncoding = THREE.sRGBEncoding; + renderer.xr.enabled = true; + return renderer; + } + + addLight() { + const ambient = new THREE.HemisphereLight(0x606060, 0x404040); + this.scene.add(ambient); + const light = new THREE.DirectionalLight(0xffffff); + light.position.set(1, 1, 1).normalize(); + this.scene.add(light); + } + setupVR() { + this.renderer.xr.enabled = true; + document.body.appendChild(VRButton.createButton(this.renderer)); + } +} + +export default App; diff --git a/apps/webxr-hands/snap-fingers.js b/apps/webxr-hands/snap-fingers.js new file mode 100644 index 0000000..e43d469 --- /dev/null +++ b/apps/webxr-hands/snap-fingers.js @@ -0,0 +1,243 @@ + +import * as THREE from 'three'; +import { VRButton } from 'three/examples/jsm/webxr/VRButton'; +import { BoxLineGeometry } from 'three/examples/jsm/geometries/BoxLineGeometry'; +import { OrbitControls } from 'three/examples/jsm/controls/OrbitControls'; +import { XRControllerModelFactory } from 'three/examples/jsm/webxr/XRControllerModelFactory.js'; +import { XRHandModelFactory } from 'three/examples/jsm/webxr/XRHandModelFactory.js'; + +let App = class App { + constructor() { + const container = document.createElement('div'); + document.body.appendChild(container); + + /*Creating Camera, Scene and Renderer */ + this.camera = this.createCamera(); + this.scene = this.createScene(); + this.renderer = this.createRenderer(); + container.appendChild(this.renderer.domElement); + + // Adding Lights + this.addLight(); + + const room = new THREE.LineSegments( + new BoxLineGeometry(6, 6, 6, 10, 10, 10), + new THREE.LineBasicMaterial({ + color: "#151515", + }) + ); + room.geometry.translate(0, 3, 0); + this.scene.add(room); + + this.controls = new OrbitControls(this.camera, container); + this.controls.target.set(0, 1.6, 0); + this.controls.update(); + + this.session; + this.renderer.xr.addEventListener("sessionstart", (event) => { + this.session = this.renderer.xr.getSession(); + }); + this.renderer.xr.addEventListener("sessionend", (event) => { + this.session = null; + }); + + // Build Hands + this.rightHand; + this.leftHand; + this.buildHands(0); // Right Hand + this.buildHands(1); // Left Hand + + this.midAndThumbReady = false; + this.didSnap = false; + this.leftPinch = false; + + this.spheres = [] + + this.setupVR(); + this.renderer.setAnimationLoop(this.render.bind(this)); + window.addEventListener('resize', this.resize.bind(this)); + + } + + buildHands(thehand){ + let controller; + let controllerGrip; + + /*Initialising controllerModelFactory and handModelFactory from Three.js */ + const controllerModelFactory = new XRControllerModelFactory(); + const handModelFactory = new XRHandModelFactory().setPath( + "../hand-models" + ); + /* Setting up Hand from POV */ + controller = this.renderer.xr.getController(thehand); + this.scene.add(controller); + controllerGrip = this.renderer.xr.getControllerGrip(thehand); + controllerGrip.add(controllerModelFactory.createControllerModel(controllerGrip)); + this.scene.add(controllerGrip); + if (thehand == 0){ + this.rightHand = this.renderer.xr.getHand(0); + this.rightHand.add(handModelFactory.createHandModel(this.rightHand)); + this.scene.add(this.rightHand); + } + else{ + this.leftHand = this.renderer.xr.getHand(1); + this.leftHand.add(handModelFactory.createHandModel(this.leftHand)); + this.scene.add(this.leftHand); + } + } + + getHandVisibilityStatus() { + if (this.session) { + for (const inputSource of this.session.inputSources) { + if (inputSource.hand) { + let name = inputSource.handedness; + let theHand; + if (name === 'right'){ + theHand = this.rightHand; + let middleTip = theHand.joints['middle-finger-tip']; + let thumbTip = theHand.joints['thumb-tip']; + let metaCarpal = theHand.joints['index-finger-metacarpal'] + this.checkSnap(middleTip, thumbTip, metaCarpal, name) + } + else if (name === 'left'){ + theHand = this.leftHand; + let indexTip = theHand.joints['index-finger-tip']; + let thumbTip = theHand.joints['thumb-tip']; + this.checkPinch(indexTip, thumbTip, name) + } + else{ + console.log('Hands not being tracked...') + } + } + } + } + } + + checkPinch(thumbTip, indexTip, hand) { + // Calculate the distance between positions of Index Tip and Thumb Tip + let diffMidThumbX = Math.abs(indexTip.position.x - thumbTip.position.x) + let diffMidThumbY = Math.abs(indexTip.position.y - thumbTip.position.y) + let diffMidThumbZ = Math.abs(indexTip.position.z - thumbTip.position.z) + + if (diffMidThumbX!=0 || diffMidThumbY!=0 || diffMidThumbZ!=0){ + // When hands are not seen, the diffs initialize at zeroes. Stops once hands are seen. + if (diffMidThumbX < 0.02 && diffMidThumbY < 0.02 && diffMidThumbZ < 0.02) { + console.log(hand + "PINCHING") + + console.log(this.leftPinch) + this.leftPinch = this.leftPinch == true ? false : true + + // Add Sphere when left pinched + if (this.leftPinch == true){ + console.log(this.leftPinch) + let geometryPinch = new THREE.SphereBufferGeometry(0.01, 30, 30); + let materialPinch = new THREE.MeshStandardMaterial({ + color: Math.random() * 0xffffff + }); + let spherePinch = new THREE.Mesh(geometryPinch, materialPinch); + spherePinch.position.set(indexTip.position.x, indexTip.position.y, indexTip.position.z); + this.spheres.push(spherePinch); + this.scene.add(spherePinch); + } + // Not to add spheres on loop + this.leftPinch = true; + } + else{ + this.leftPinch = false; + } + } + } + + checkSnap(middleTip, thumbTip, metaCarpal, hand) { + // Calculate the distance between positions of Middle Tip and Thumb Tip + let diffMidThumbX = Math.abs(middleTip.position.x - thumbTip.position.x) + let diffMidThumbY = Math.abs(middleTip.position.y - thumbTip.position.y) + let diffMidThumbZ = Math.abs(middleTip.position.z - thumbTip.position.z) + + // Calculate the distance between positions of Middle Tip and Meta Carpal + let diffMidCarpalX = Math.abs(middleTip.position.x - metaCarpal.position.x) + let diffMidCarpalY = Math.abs(middleTip.position.y - metaCarpal.position.y) + let diffMidCarpalZ = Math.abs(middleTip.position.z - metaCarpal.position.z) + + if (diffMidThumbX!=0 || diffMidThumbY!=0 || diffMidThumbZ!=0){ + // When hands are not seen, the diffs initialize at zeroes. Stops once hands are seen. + if (diffMidThumbX < 0.02 && diffMidThumbY < 0.02 && diffMidThumbZ < 0.02) { + this.midAndThumbReady = true; + console.log('Mid Thumb Ready...') + } + // else{ + // this.midAndThumbReady = false; + // } + } + + if (this.midAndThumbReady == true){ + if (diffMidCarpalX < 0.05 && diffMidCarpalY < 0.05 && diffMidCarpalZ < 0.05) { + this.didSnap = this.didSnap != true ? true : false; + if (this.didSnap == true){ + console.log('Snapped...') + this.thanos() + this.midAndThumbReady = false + } + } + } + } + + thanos(){ + let spheresToDelete = Math.round(this.spheres.length / 2); + for (let i = 0; i < spheresToDelete; i++) { + console.log(i) + this.scene.remove(this.spheres[i]); + this.spheres = this.spheres.filter(item => item !== this.spheres[i]) + } + } + + resize() { + this.camera.aspect = window.innerWidth / window.innerHeight; + this.camera.updateProjectionMatrix(); + this.renderer.setSize(window.innerWidth, window.innerHeight); + } + render() { + this.renderer.render(this.scene, this.camera); + this.getHandVisibilityStatus(); + } + + createCamera() { + const camera = new THREE.PerspectiveCamera( + 50, + window.innerWidth / window.innerHeight, + 0.1, + 100 + ); + camera.position.set(0, 1.6, 3); + return camera; + } + + createScene() { + const scene = new THREE.Scene(); + scene.background = new THREE.Color(0x808080); + return scene; + } + + createRenderer() { + const renderer = new THREE.WebGLRenderer({ antialias: true }); + renderer.setPixelRatio(window.devicePixelRatio); + renderer.setSize(window.innerWidth, window.innerHeight); + renderer.outputEncoding = THREE.sRGBEncoding; + renderer.xr.enabled = true; + return renderer; + } + + addLight() { + const ambient = new THREE.HemisphereLight(0x606060, 0x404040); + this.scene.add(ambient); + const light = new THREE.DirectionalLight(0xffffff); + light.position.set(1, 1, 1).normalize(); + this.scene.add(light); + } + setupVR() { + this.renderer.xr.enabled = true; + document.body.appendChild(VRButton.createButton(this.renderer)); + } +} + +export default App; diff --git a/apps/webxr-hands/touch-object.js b/apps/webxr-hands/touch-object.js new file mode 100644 index 0000000..08880a5 --- /dev/null +++ b/apps/webxr-hands/touch-object.js @@ -0,0 +1,242 @@ + +import * as THREE from 'three'; +import { VRButton } from 'three/examples/jsm/webxr/VRButton'; +import { BoxLineGeometry } from 'three/examples/jsm/geometries/BoxLineGeometry'; +import { OrbitControls } from 'three/examples/jsm/controls/OrbitControls'; +import { XRControllerModelFactory } from 'three/examples/jsm/webxr/XRControllerModelFactory.js'; +import { XRHandModelFactory } from 'three/examples/jsm/webxr/XRHandModelFactory.js'; + +const tmpVector1 = new THREE.Vector3(); +const tmpVector2 = new THREE.Vector3(); + +let App = class App { + constructor() { + const container = document.createElement('div'); + document.body.appendChild(container); + + /*Creating Camera, Scene and Renderer */ + this.camera = this.createCamera(); + this.scene = this.createScene(); + this.renderer = this.createRenderer(); + container.appendChild(this.renderer.domElement); + + // Adding Lights + this.addLight(); + + const room = new THREE.LineSegments( + new BoxLineGeometry(6, 6, 6, 10, 10, 10), + new THREE.LineBasicMaterial({ + color: "#151515", + }) + ); + room.geometry.translate(0, 3, 0); + this.scene.add(room); + + this.controls = new OrbitControls(this.camera, container); + this.controls.target.set(0, 1.6, 0); + this.controls.update(); + + this.session; + this.renderer.xr.addEventListener("sessionstart", (event) => { + this.session = this.renderer.xr.getSession(); + }); + this.renderer.xr.addEventListener("sessionend", (event) => { + this.session = null; + }); + + // Build Hands + this.rightHand; + this.leftHand; + this.buildHands(0); // Right Hand + this.buildHands(1); // Left Hand + + this.midAndThumbReady = false; + this.didSnap = false; + this.rightPinch = false; + + this.spheres = [] + + this.setupVR(); + this.renderer.setAnimationLoop(this.render.bind(this)); + window.addEventListener('resize', this.resize.bind(this)); + + } + + buildHands(thehand){ + let controller; + let controllerGrip; + + /*Initialising controllerModelFactory and handModelFactory from Three.js */ + const controllerModelFactory = new XRControllerModelFactory(); + const handModelFactory = new XRHandModelFactory().setPath( + "../hand-models" + ); + /* Setting up Hand from POV */ + controller = this.renderer.xr.getController(thehand); + this.scene.add(controller); + controllerGrip = this.renderer.xr.getControllerGrip(thehand); + controllerGrip.add(controllerModelFactory.createControllerModel(controllerGrip)); + this.scene.add(controllerGrip); + if (thehand == 0){ + this.rightHand = this.renderer.xr.getHand(0); + this.rightHand.add(handModelFactory.createHandModel(this.rightHand)); + this.scene.add(this.rightHand); + } + else{ + this.leftHand = this.renderer.xr.getHand(1); + this.leftHand.add(handModelFactory.createHandModel(this.leftHand)); + this.scene.add(this.leftHand); + } + } + + getHandVisibilityStatus() { + if (this.session) { + for (const inputSource of this.session.inputSources) { + if (inputSource.hand) { + let name = inputSource.handedness; + let theHand; + if (name == 'right'){ + theHand = this.rightHand; + let rightIndexTip = theHand.joints['index-finger-tip']; + let rightThumbTip = theHand.joints['thumb-tip']; + this.checkPinch(rightIndexTip, rightThumbTip, name) + } + else if (name == 'left'){ + theHand = this.leftHand; + let leftThumbTip = theHand.joints['thumb-tip']; + let leftIndexTip = theHand.joints['index-finger-tip'] + let leftMiddleTip = theHand.joints['middle-finger-tip'] + let leftRingTip = theHand.joints['ring-finger-tip']; + let leftPinkyTip = theHand.joints['pinky-finger-tip'] + this.checkTouch(leftThumbTip, leftIndexTip, leftMiddleTip, leftRingTip, leftPinkyTip, name) + } + else{ + console.log('Hands not being tracked...') + } + } + } + } + } + + checkPinch(thumbTip, indexTip, hand) { + // Calculate the distance between positions of Index Tip and Thumb Tip + let diffMidThumbX = Math.abs(indexTip.position.x - thumbTip.position.x) + let diffMidThumbY = Math.abs(indexTip.position.y - thumbTip.position.y) + let diffMidThumbZ = Math.abs(indexTip.position.z - thumbTip.position.z) + + if (diffMidThumbX!=0 || diffMidThumbY!=0 || diffMidThumbZ!=0){ + // When hands are not seen, the diffs initialize at zeroes. Stops once hands are seen. + if (diffMidThumbX < 0.02 && diffMidThumbY < 0.02 && diffMidThumbZ < 0.02) { + console.log(hand + "PINCHING") + + this.rightPinch = this.rightPinch == true ? false : true + + // Add Sphere when left pinched + if (this.rightPinch == true){ + console.log(this.rightPinch) + let geometryPinch = new THREE.SphereBufferGeometry(0.01, 30, 30); + let materialPinch = new THREE.MeshStandardMaterial({ + color: 0xffffff + }); + let spherePinch = new THREE.Mesh(geometryPinch, materialPinch); + spherePinch.position.set(indexTip.position.x, indexTip.position.y, indexTip.position.z); + this.spheres.push(spherePinch); + this.scene.add(spherePinch); + } + // Not to add spheres on loop + this.rightPinch = true; + } + else{ + this.rightPinch = false; + } + } + } + + checkTouch(thumbTip, indexTip, middleTip, ringTip, pinkyTip, name) { + console.log(this.spheres) + if (this.spheres.length > 0){ + for (let i = 0; i < this.spheres.length; i++) { + + const distanceThumb = thumbTip.getWorldPosition(tmpVector1).distanceTo(this.spheres[i].getWorldPosition(tmpVector2)); + if (distanceThumb < this.spheres[i].geometry.parameters.radius * this.spheres[i].scale.x) { + console.log('Touch') + this.spheres[i].material.color.setHex(0x6feb10); + } + + const distanceIndex = indexTip.getWorldPosition(tmpVector1).distanceTo(this.spheres[i].getWorldPosition(tmpVector2)); + if (distanceIndex < this.spheres[i].geometry.parameters.radius * this.spheres[i].scale.x) { + console.log('Touch') + this.spheres[i].material.color.setHex(0x00ffaa); + } + + const distanceMid = middleTip.getWorldPosition(tmpVector1).distanceTo(this.spheres[i].getWorldPosition(tmpVector2)); + if (distanceMid < this.spheres[i].geometry.parameters.radius * this.spheres[i].scale.x) { + console.log('Touch') + this.spheres[i].material.color.setHex(0x1838d9); + } + + const distanceRing = ringTip.getWorldPosition(tmpVector1).distanceTo(this.spheres[i].getWorldPosition(tmpVector2)); + if (distanceRing < this.spheres[i].geometry.parameters.radius * this.spheres[i].scale.x) { + console.log('Touch') + this.spheres[i].material.color.setHex(0x834ad4); + } + + const distancePinky = pinkyTip.getWorldPosition(tmpVector1).distanceTo(this.spheres[i].getWorldPosition(tmpVector2)); + if (distancePinky < this.spheres[i].geometry.parameters.radius * this.spheres[i].scale.x) { + console.log('Touch') + this.spheres[i].material.color.setHex(0xc72f14); + } + } + } + } + + resize() { + this.camera.aspect = window.innerWidth / window.innerHeight; + this.camera.updateProjectionMatrix(); + this.renderer.setSize(window.innerWidth, window.innerHeight); + } + render() { + this.renderer.render(this.scene, this.camera); + this.getHandVisibilityStatus(); + } + + createCamera() { + const camera = new THREE.PerspectiveCamera( + 50, + window.innerWidth / window.innerHeight, + 0.1, + 100 + ); + camera.position.set(0, 1.6, 3); + return camera; + } + + createScene() { + const scene = new THREE.Scene(); + scene.background = new THREE.Color(0x808080); + return scene; + } + + createRenderer() { + const renderer = new THREE.WebGLRenderer({ antialias: true }); + renderer.setPixelRatio(window.devicePixelRatio); + renderer.setSize(window.innerWidth, window.innerHeight); + renderer.outputEncoding = THREE.sRGBEncoding; + renderer.xr.enabled = true; + return renderer; + } + + addLight() { + const ambient = new THREE.HemisphereLight(0x606060, 0x404040); + this.scene.add(ambient); + const light = new THREE.DirectionalLight(0xffffff); + light.position.set(1, 1, 1).normalize(); + this.scene.add(light); + } + setupVR() { + this.renderer.xr.enabled = true; + document.body.appendChild(VRButton.createButton(this.renderer)); + } +} + +export default App; diff --git a/docs/Oculus-Dev-Setup.md b/docs/Oculus-Dev-Setup.md new file mode 100644 index 0000000..180e369 --- /dev/null +++ b/docs/Oculus-Dev-Setup.md @@ -0,0 +1,114 @@ +# Development Setup: Oculus + +## Table of Contents +- Developer Account Set-up +- Enabling Developer Mode for the Oculus Device +- Development Environment Setup on the OS +- Running a local application in the Oculus Browser + - Network address `https://:port` + - Localhost address `https://localhost:8080` +- Debugging within the Oculus Browser +- Oculus Developer Hub +- Additional Resources + +## Developer Account Set-up +Before getting started, you would require a Facebook account and a corresponding [Oculus Developer Account](https://developer.oculus.com/) + + +## Enabling Developer Mode for the Oculus Device +- Download the Oculus App and login using Facebook +- Turn on the headset +- Navigate to devices and select the device + +- Click on 'Developer Mode' and enable it by toggling the switch + + +## Development Environment Setup on the OS +- Connect the Oculus device to your machine via USC-C, and upon being prompted, [Enable Developer Mode](https://developer.oculus.com/documentation/native/android/mobile-device-setup/) + +- Install [ADB driver](https://developer.oculus.com/downloads/package/oculus-adb-drivers/) for your OS + +__For Windows__ + +- Check if the ADB driver path is configured in your global path variable by using the command `echo %PATH%` + +- If is it configured correctly, it should be visible such as + + +- If you cannot see this path `C:\Users\USERNAME\AppData\Local\Android\sdk\platform-tools` present, add it your environment variables either by using the command `setx PATH "%PATH%;C:\Program Files\android-sdk-windows\platform-tools"` or manually by navigating to environment variables and ammending the environment variable: *PATH* + + + +- To check whether your device is connected correctly, use the command `adb devices`. If all is good and the device is detected, you should see the following output: + + + +- [Enable Wifi Debugging](https://developer.oculus.com/documentation/oculus-browser/browser-remote-debugging/) + +## Running a local application in the Oculus Browser +- To run a WebXR application in the browser, HTTPs is required. Before running your application locally, ensure that OPENSSL is configured + +- Run the `webxr-layers` application using `npm run dev` in your local terminal + + +- The application is now running on `https://localhost:8080` or on your network at `https://:port` + +### __There are 2 ways to view the application running locally within your oculus browser, by accessing the following addresses:__ + + +### 1. Network address `https://:port` +Ensure that your oculus device and local machine are both connected to the same WiFi network, and that the oculus device is connected to your machine + +If prompted with the *your connection is not private* screen, click on *advanced* and then *proceed to _IP:PORT_ url*. This should redirect you to the locally running application and allow you to interact with it. + + + + +Additionally, if the steps so far do not result in the desired result, we might need to by-pass the "https" requirement. For this, +- Navigate to chrome://flags on the device +- Search for `Insecure origins treated as secure` +- Then enable it and add http://192.168.1.110:8080/ in the control attached to that setting. Ensure that you have entered the correct IP and port number + +- Reboot the browser, and access the URL again + +### 2. Localhost address `https://localhost:8080` + +To be able to access the local port of your machine on the oculus browser as is, we need to perform reverse port forwarding. Essentially, anytime we access a particular port on the oculus browser - we want to forward that to our local machine. `adb` has built-in support for port forwarding. + +Use the command `adb reverse tcp:PORT tcp:PORT`. Here, we want the port:8080 on the Oculus Browser to be forwarded to port:8080 on our local machine. Hence, run the command as follows: +`adb reverse tcp:8080 tcp:8080` + + + +Now, access `https://localhost:8080` from within the oculus browser. Ensure that the oculus device is connected to your machine. + + + +The application should be visible as shown above. These steps are required to be followed for each debugging session i.e. every time the device is disconnected and then re-connected + +## Debugging within the Oculus Browser +To debug the application running locally and accessible via the Oculus Browser, visit `chrome://inspect/#devices` on your machine browser + +Here you would be able to view all URLs currently active within the Oculus Browser + + + +Click on the URL that you would like to debug + + + +The developer tool and console here are in sycn with any interactions with the application within the Oculus Browser! + + +## Oculus Developer Hub +*For Windows* +- Download [Oculus Developer Hub](https://developer.oculus.com/downloads/package/oculus-developer-hub-win/) +- [Configure your device on ODH](https://developer.oculus.com/documentation/tools/odh/#connect-headset-to-odh) +- [Oculus Developer Hub Debugging Tools](https://developer.oculus.com/documentation/tools/odh-media/) + +## Additional Resources +- [Oculus Developer Documentation](https://developer.oculus.com/develop/) +- [Oculus Browser Remote Debugging](https://developer.oculus.com/documentation/oculus-browser/browser-remote-debugging/) +- [Getting Started with Oculus Browser and Debugging](https://developer.oculus.com/webxr/) + + diff --git a/docs/Project-Setup.md b/docs/Project-Setup.md new file mode 100644 index 0000000..5dd938a --- /dev/null +++ b/docs/Project-Setup.md @@ -0,0 +1,46 @@ +# Project Setup + +## Clone the repository and Install Dependencies +Simply clone the repository and install dependencies. + +``` +> git clone https://github.com/MLH-Fellowship/webxr-layers.git +> cd webxr-layers +> npm install +``` + +At this point, running `npm run dev` should have run the project. But, it would possibly give an error. +This is because of the snowpack configuration. In snowpack.config.js file, we can see inside 'devOptions', the secure flag is set to true. + +It is because to run a WebXR application in the browser, HTTPs is required. + +## Configure HTTPs for localhost + +We will configure HTTPs for localhost and for signing the SSL certificate, OpenSSL will be used. + +### Install OpenSSL +- Download the OpenSSL. ([Download Link](https://slproweb.com/products/Win32OpenSSL.html)) +- Install OpenSSL. +- Add it to the path in Environment Variables. + +### Create SSL certificates + +- In the project root directory, run `npx devcert-cli generate localhost` + +This should generate two certificate files. + +- Rename those files as snowpack.crt and snowpack.key +- Now run `npm run dev` +- Open https://localhost:8080 on your browser to see the project. + +## Setup Browser Emulator + +WebXR Browser Emulator can be used to run and test WebXR content in desktop browsers without using a real XR device. + +- Update the Google Chrome Browser +- Install the Chrome Browser Extension ([Link](https://chrome.google.com/webstore/detail/webxr-api-emulator/mjddjgeghkdijejnciaefnkjmkafnnje)) +- Open the Project site or https://localhost:8080/ for our project. + +In Developer Tools, there should be a new tab named 'Web XR'. + +- Clicking on that, the Device Emulator should be accessible. diff --git a/docs/VideoSamples.md b/docs/VideoSamples.md new file mode 100644 index 0000000..9f76a4c --- /dev/null +++ b/docs/VideoSamples.md @@ -0,0 +1,546 @@ +# WebXR Layers Sample + +Composition of WebXR layers has [significant benefits](https://www.w3.org/TR/webxrlayers-1/#intro). This application is a small collection of samples that demonstrate creating and interacting with video layers in an XR environment. Because video layers cannot be interacted with directly via a controller, a 3D rendering library is used to achieve the desired interactions, including video playback controls, and moving and resizing layers. [Three.js](https://threejs.org/docs/index.html#manual/en/introduction/Creating-a-scene) is the choice of 3D rendering library for this application. + +This project is a product of an [Major League Hacking (MLH) Spring 2021 Fellowship](https://developers.facebook.com/blog/post/2021/03/31/facebook-open-source-introduces-mlh-fellowship-class-spring-2021/) open source contribution. + +## Design + +This section provides a general overview of the application and a high-level explanation of how it works. The web application consists of several samples, each meant to showcase a particular feature. However, `multiple-layers.js` can be considered to be the most complex sample that is meant to show: + +- Rendering multiple video layers of multiple types (e.g. equirect, quad) + - Independent playback +- Interacting with individual video layers + - Playback controls + - Moving quad layers + - Resizing quad layers + +The following is a labeled screenshot of the `multiple-layers` sample. + +![multiple-layers.js overview static screenshot](./assests/videoLayers/overview-static.jpg) + +## Application Architecture + +![Architecture.png](./assests/videoLayers/Architecture.png) + +This diagram shows a high-level overview of a particular sample. This sample has two media layers: one equirectangular layer and one quadrilateral layer. Breaking the diagram into four main components: + +1. `multiple-layers` is the application sample and the "driver" +2. The two `THREE` libraries are meant to indicate that the application uses Three.js to render the 3D scene and all 3D objects +3. `MediaLayerManager` handles creation of media layers +4. Interactive Media Layer group made up of smaller components + - `MediaLayer` is the main media layer interface + - `GlassLayer` is implementation detail for moving layers + - `Toolbar` handles playback controls, resizing layers, and all other interactions + +Crucially, the driver program contains a map of `MediaLayer` objects. This map is used to dynamically manage multiple media layers—rendering and detecting intersections with multiple `Toolbar` and `GlassLayer` objects under the hood. This, together with a parallel `videos` map, make up the interface to create multiple `XRMediaLayer`s. + +### User Interface + +![UI.png](./assests/videoLayers/UI.png) + +To create a multiple video layers sample, simply supply and/or change these details: + +1. Create a global `this.mediaLayers` map. Use a distinct key for each layer. This is done in the `constructor()` and `render()` method. +2. Create a global `this.videos` map. Use a distinct key for each layer. **Note: this is a parallel map to `this.mediaLayers`; use the same keys.** This is done in the `constructor()`. +3. Supply configurations (dimensions, positions) for each layer and its toolbar, and use `MediaLayerManager` to create the relevant `XRMediaLayer` objects. This is done in the `render()` method. +4. Supply the created `XRMediaLayer` objects to the `session.updateRenderState()` call in the `render()` method. + +An example workflow to create a background equirect video and a quad video is as follows (omitting the other boilerplate): + +```js +// imports + +class App { + constructor() { + // create scene, etc. + + // Create Map of MediaLayers + this.mediaLayers = new Map(); + + // Create Map of Videos for Each Layer + this.videos = this.createVideos({ + equirect: EQUIRECT_VIDEO_SRC, // video.src attribute + quad: QUAD_VIDEO_SRC, // video.src attribute + }); + } + + // ... + + render() { + // ... + if (session && session.renderState.layers && !session.hasMediaLayer && areVideosReady) { + session.hasMediaLayer = true; + + const mediaFactory = new MediaLayerManager(session, this.renderer); + + // toolbar configurations for equirect layer + const uiConfigEquirect = { + panelWidth: 2, + panelHeight: 0.5, + height: 128, + position: { x: 0, y: -1, z: -3 }, + }; + const toolbarGroupConfig = { + rotateXAngle: -Math.PI / 4, + position: { + x: 0, + y: 1.6, + z: -2, + }, + }; + + // create `XRMediaEquirectLayer` object + const equirect = await mediaFactory.createMediaLayer( + this.videos.get("equirect"), + MediaLayerManager.EQUIRECT_LAYER, + { + layout: "stereo-top-bottom", + }, + uiConfigEquirect, + toolbarGroupConfig + ); + + // toolbar configurations for quad layer + const uiConfigQuad = { + panelWidth: 1, + panelHeight: 0.2, + height: 128, + position: { x: 0, y: 0, z: 0 }, + }; + + // create `XRMediaQuadLayer` object + const quad = await mediaFactory.createMediaLayer( + this.videos.get("quad"), + MediaLayerManager.QUAD_LAYER, + { + layout: "stereo-top-bottom", + width: 1.0, + height: 0.5625, + transform: new XRRigidTransform({ + x: 0.0, + y: 1.3, + z: -2.75, + w: 1.0, + }), + }, + uiConfigQuad + ); + + // insert into `mediaLayers` map + this.mediaLayers.set("equirect", equirect); + this.mediaLayers.set("quad", quad); + + // Hide toolbars initially + this.hideToolbars(); + + // Pass `XRMediaLayer` instances to `updateRenderState` + session.updateRenderState({ + layers: [ + equirect.layer, + quad.layer, + session.renderState.layers[0], + ], + }); + + this.videos.forEach((video) => video.play()); + } + } + + // other methods +} +``` + +## Implementation + +### General Challenges + +Some of the key challenges faced in this project were: + +1. Relatively new and changing [documentation](https://www.w3.org/TR/webxrlayers-1/); first public draft released Dec 3 2020 +2. No prior experience in WebXR and 3D development +3. Inability to interact directly with WebXR media layers via raycasting + +Challenge 3 was a significant initial hurdle because we were not yet used to interacting with 3D objects, let alone using proxy 3D objects to handle interactions with media layers. However, we quickly got used to this and learned to use 3D objects as proxy objects to interact indirectly with their coupled media layers for [moving](#moving-video-layers) and [resizing](#resizing-video-layers). + +Challenge 1 was perhaps the most significant because some of the most stumbling bugs we faced were related to outdated Three.js WebXR layer interfaces. Because we primarily used Three.js as our 3D development interface, and its `WebXRManager`, `VRButton`, and `WebGLRenderer` interfaces, important changes to these interfaces for media layers had to be made for them to be useable in our application. A PR was made to the Three.js library by WebXR Layers maintainer Rik, but because it had not been merged, we had to implement those changes ourselves for the purposes of this project. + +### `WebXRManager`, `VRButton`, `WebGLRenderer` + +This section will highlight the important changes we made to the three files `WebXRManager`, `VRButton`, and `WebGLRenderer` from the Three.js [base](https://github.com/mrdoob/three.js/tree/dev/src/renderers) and [examples](https://github.com/mrdoob/three.js/tree/dev/examples/jsm/webxr) libraries. + +These changes reference Rik's PR ([mrdoob/three.js#20696](https://github.com/mrdoob/three.js/pull/20696)) made to Three.js in Nov 2020. + +#### `WebXRManager.js` + +```diff +--- a/node_modules/three/src/renderers/webxr/WebXRManager.js ++++ b/util/webxr/WebXRManager.js ++ let pose = null; ++ let glBinding = null; ++ let glFramebuffer = null; ++ let glProjLayer = null; +... ++ if (session.renderState.layers === undefined) { + const layerInit = { + antialias: attributes.antialias, + alpha: attributes.alpha, + depth: attributes.depth, + stencil: attributes.stencil, + framebufferScaleFactor: framebufferScaleFactor + }; + + // eslint-disable-next-line no-undef + const baseLayer = new XRWebGLLayer( session, gl, layerInit ); + + session.updateRenderState( { baseLayer: baseLayer } ); ++ } else { ++ const projectionLayerInit = { ++ scaleFactor: framebufferScaleFactor, ++ }; ++ ++ glBinding = new XRWebGLBinding(session, gl); ++ ++ glProjLayer = glBinding.createProjectionLayer( ++ projectionLayerInit ++ ); ++ ++ glFramebuffer = gl.createFramebuffer(); ++ ++ session.updateRenderState({ layers: [glProjLayer] }); ++ } +if (pose !== null) { + const views = pose.views; + const baseLayer = session.renderState.baseLayer; + ++ if (session.renderState.layers === undefined) { + renderer.setFramebuffer(baseLayer.framebuffer); ++ } + + let cameraVRNeedsUpdate = false; + + // check if it's necessary to rebuild cameraVR's camera list + if (views.length !== cameraVR.cameras.length) { + cameraVR.cameras.length = 0; + cameraVRNeedsUpdate = true; + } + + for (let i = 0; i < views.length; i++) { + const view = views[i]; +- const viewport = baseLayer.getViewport( view ); ++ let viewport = null; + ++ if (session.renderState.layers === undefined) { + viewport = baseLayer.getViewport(view); ++ } else { ++ const glSubImage = glBinding.getViewSubImage( ++ glProjLayer, ++ view ++ ); ++ ++ gl.bindFramebuffer(gl.FRAMEBUFFER, glFramebuffer); ++ ++ gl.framebufferTexture2D( ++ gl.FRAMEBUFFER, ++ gl.COLOR_ATTACHMENT0, ++ gl.TEXTURE_2D, ++ glSubImage.colorTexture, ++ 0 ++ ); ++ ++ if (glSubImage.depthStencilTexture !== undefined) { ++ gl.framebufferTexture2D( ++ gl.FRAMEBUFFER, ++ gl.DEPTH_ATTACHMENT, ++ gl.TEXTURE_2D, ++ glSubImage.depthStencilTexture, ++ 0 ++ ); ++ } ++ ++ renderer.setFramebuffer(glFramebuffer); ++ ++ viewport = glSubImage.viewport; ++ } + + ... + } +} +``` + +The main changes here are in accounting for `if (session.renderState.layers === undefined)`. Essentially, if the session does not already have a base projection layer, create a `XRWebGLLayer` as the base layer. Otherwise, create and add an `XRProjectionLayer` to the `renderState`'s `layers` array. + +#### `VRButton.js` + +```diff +--- a/node_modules/three/examples/jsm/webxr/VRButton.js ++++ b/util/webxr/VRButton.js +- const sessionInit = { +- optionalFeatures: ['local-floor', 'bounded-floor', 'hand-tracking'] +- }; ++ this.sessionInit = sessionInit || { ++ optionalFeatures: ["local-floor", "bounded-floor", "hand-tracking"] ++ }; +``` + +The `sessionInit` object is intended to be an object that has a `requiredFeatures` and/or `optionalFeatures` field that specifies `layers` for an application that requires `layers`, like so: + +```js +const sessionInit = { requiredFeatures: [ 'layers' ], optionalFeatures: [ 'local-floor' ] } +``` + +#### `WebGLRenderer.js` + +```diff +--- a/node_modules/three/src/renderers/WebGLRenderer.js ++++ b/util/WebGLRenderer.js +- import { WebXRManager } from './webxr/WebXRManager.js'; ++ // Use our Custom WebXRManager instead of three.js ++ import { WebXRManager } from "./webxr/WebXRManager" + +``` + +In this file we simply changed the `WebXRManager` import to use the above altered `WebXRManager.js`. + +### Creating Video Layers + +This section explains the approach taken to create video layers with basic playback controls. + +![MediaLayerManager.png](./assests/videoLayers/MediaLayerManager.png) + +This class diagram outlines the process of creating video layers. The driver `App` creates a `MediaLayerManager` which is in turn used to create a `MediaLayer` object using its `createMediaLayer()` method. + +A `MediaLayer` object comprises the `layer` object that is of type [`XRMediaLayer`](https://www.w3.org/TR/webxrlayers-1/#videolayer). A `MediaLayer` object is also coupled with a corresponding `GlassLayer` and `Toolbar`. The former is used solely in the implementation of moving layers. The latter handles all other interactions with the media layer, including playback controls, fixed resizing, and fluid resizing. + +Suppose we have an empty `medialayers` map but a `videos` map with a video src for an intended quad layer, like so: + +- `mediaLayers: {}` +- `videos: {"quad": "path_to_quad_video.mp4"}` + +To create a simple quad video layer, add this in the driver's `render()` method: + +```js +render() { + ... + // create `MediaLayerManager` + const mediaFactory = new MediaLayerManager(session, this.renderer); + + // position and dimensions of toolbar + const uiConfigQuad = { + panelWidth: 1, + panelHeight: 0.2, + height: 128, + position: { x: 0, y: 0, z: 0 }, + }; + + // calling `MediaLayerManager`'s `createMediaLayer` + const quad = await mediaFactory.createMediaLayer( + this.videos.get("quad"), + MediaLayerManager.QUAD_LAYER, + { + layout: "stereo-top-bottom", + transform: new XRRigidTransform({ + x: 0.0, + y: 1.7, + z: -1.75, + w: 1.0, + }), + }, + uiConfigQuad + ); + + // add to map of `MediaLayer`s + this.mediaLayers.set("quad", quad); + + session.updateRenderState({ + // add newly-created quadLayer, and base layer + layers: [quad.layer, session.renderState.layers[0]], + }); + this.videos.forEach((video) => video.play()); + ... +} +``` + +### Toolbar Interactions + +![Toolbar](./assests/videoLayers/toolbar.png) + +There are four main toolbar interactions, corresponding to video playback controls, excluding fixed and fluid resizing, which will be discussed in the [resizing section](#resizing-video-layers). + +These four playback controls are: + +1. Play/Pause +2. Rewind 15s Backwards/Skip 15s Forward +3. Restart +4. Progress Bar Point Select + +In a nutshell, these interactions are implemented as 3D button objects, abstracted using the [CanvasUI](https://github.com/NikLever/CanvasUI) library. + +### Moving Video Layers + +![moving-controller.gif](./assests/videoLayers/moving-controller.gif) + +When the controller ray intersects the video layer, a Three.js rectangular object, the `glass` object, is attached to the controller's ray. Then moving the controller moves the `glassLayer`, which updates the `XRMediaQuadLayer`'s position using Three.js's `getWorldPosition()` and `getWorldQuaternion()` methods. The `glassLayer` is then recreated at exactly the new position of the `XRMediaQuadLayer`. + +### Resizing Video Layers + +![resizing-fixed.gif](./assests/videoLayers/resizing-fixed.gif) + +`C` (for "compress") negatively scales the `XRMediaQuadLayer`'s `height` and `width` attributes by a fixed factor of `1.25`, i.e. `height /= 1.25; width /= 1.25`. Similarly, `E` (for "expand") positively scales `height` and `width` by a fixed factor of `1.25`, i.e. `height *= 1.25; width *= 1.25`. + +#### Challenges with Fluid Resizing + +![resizing-fluid.gif](./assests/videoLayers/resizing-fluid.gif) + +Our intentions are to somewhat heed Oculus' [hand interactions best practices](https://developer.oculus.com/learn/hands-design-interactions/). The white bar that is below the toolbar is a resizing handle, meant to somewhat mimic the "Resize Something" section in this resource. + +Fluid resizing is currently one of the biggest challenges. The current implementation does the following: + +1. On intersecting resize handle, clone the handle. +2. Attach the cloned handle to the controller (i.e., an invisible line ray). +3. Move the controller left or right. +4. On each render, measure the distance between the cloned handle and the original handle. +5. Moving to the right will upsize the video proportionally to this distance, while moving to the left downsizes the video proportionally to this distance. + +Clearly, the idea behind this implementation is prone to bugs. A much easier and less error-prone approach would be to implement a scrub-able progress bar-like interaction, with the scrub at the center of the bar indicating original size, and scaling according to lateral deviation from the center. + +### Working with Hands + +![moving-controller.gif](./assests/videoLayers/moving-controller.gif) + +![moving-hand.gif](./assests/videoLayers/moving-hand.gif) + +In this application, hands are simply an extension of controllers. Think of a controller as being composed of two parts: + +1. A target ray space (`controller`) which contains all the metadata of the controller + - e.g. 3D position and orientation, pointing ray, etc. +2. A controller texture model (`grip`) that is the rendered appearance of the controller + +Then, hands can be supported by simply replacing the `grip`—instead of rendering a controller texture, render the hand texture model. + +Consider a regular Three.js controller workflow: + +```js +const controllerModelFactory = new XRControllerModelFactory(); +const controllers = []; +for (let i = 0; i <= 1; i++) { + // target ray space + const controller = this.renderer.xr.getController(i); + controller.add(invisibleRay.clone()); + controller.add(ray.clone()); + controller.userData.selectPressed = false; + this.scene.add(controller); + + controller.addEventListener("selectstart", onSelectStart); + controller.addEventListener("selectend", onSelectEnd); + controller.addEventListener("disconnected", onDisconnect); + + controllers.push(controller); + + // texture model + const grip = this.renderer.xr.getControllerGrip(i); + const controllerModel = controllerModelFactory.createControllerModel( + grip + ); + grip.add(controllerModel); + // add to scene to render texture + this.scene.add(grip); +} +``` + +To support hands, we can reuse the same controller target ray space, and simply add the hand texture models which will be rendered when hands are active: + +```diff +const controllerModelFactory = new XRControllerModelFactory(); +const controllers = []; +for (let i = 0; i <= 1; i++) { + // target ray space + const controller = this.renderer.xr.getController(i); + controller.add(invisibleRay.clone()); + controller.add(ray.clone()); + controller.userData.selectPressed = false; + this.scene.add(controller); + + ... + ++ // hand texture model ++ const hand = this.renderer.xr.getHand(i); ++. const handModel = handModelFactory.createHandModel(hand, "oculus"); ++. hand.add(handModel); ++ this.scene.add(hand); +} +``` + +This way, our hand comprises: + +1. The **same** target ray space (`controller`) which contains all the metadata of the controller + - e.g. 3D position and orientation, pointing ray, etc. +2. A hand texture model (`hand`) that is the rendered appearance of the controller + +#### Challenges with Hands Interactions + +Being able to work with hands simply by adding code for rendering the hand textures allowed us to be able to skip having to write code to handle the intricate details of working with [skeleton joints](https://www.w3.org/TR/webxr-hand-input-1/#skeleton-joints-section). For our purposes at least, we hadn't needed to implement near-field hand interactions because we were primarily interacting with video layers which are not meant to be viewed at point-blank range. + +Naturally, the idea is to use the same `controller` target ray space to cast rays and detect intersections with surrounding objects to handle interactions. + +However, at this time the target ray space of `controller` was not available to the `hand` object due to an implementation detail in Three.js's `WebXRController.js`. This caused bugs like not being able to detect intersections with objects because the target ray was simply not applied when `hand` objects were active. + +#### `WebXRController` + +Thankfully, Rik's colleague Felix made a change to Three.js's `WebXRController.js` that made the `controller` target ray available in hands mode: + +```diff +if ( inputSource && frame.session.visibilityState !== 'visible-blurred' ) { + ++ if ( targetRay !== null ) { ++ ++ inputPose = frame.getPose( inputSource.targetRaySpace, referenceSpace ); ++ ++ if ( inputPose !== null ) { ++ ++ targetRay.matrix.fromArray( inputPose.transform.matrix ); ++ targetRay.matrix.decompose( targetRay.position, targetRay.rotation, targetRay.scale ); ++ ++ } ++ ++ } + + if ( hand && inputSource.hand ) { + + handPose = true; + ... + } else { +- if ( targetRay !== null ) { +- +- inputPose = frame.getPose( inputSource.targetRaySpace, referenceSpace ); +- +- if ( inputPose !== null ) { +- +- targetRay.matrix.fromArray( inputPose.transform.matrix ); +- targetRay.matrix.decompose( targetRay.position, targetRay.rotation, targetRay.scale ); +- +- } +... +} +``` + +This change shifts the `targetRay` and `inputPose` transformations and calculations to outside of the `if-else` checking if the input source is a hand, so that `targetRay` is applied and available regardless of whether the input source is a controller or a hand. + +These changes have been applied to Three.js `r127`. Because our project is on Three.js `r125`, we implemented these changes by overriding Three.js's `WebXRController.js` with our own, for the time being. + +## Glossary + +|Terminology|Meaning / Reference| +|--|--| +|Fluid resizing|Resizing proportionally and by [continuous](https://en.wikipedia.org/wiki/Continuous_or_discrete_variable) factors| +|Three.js|3D rendering [library](https://threejs.org/)| + +## Instructions for Manual Testing + +Works with Oculus browser: + +1. Clone [repository](https://github.com/und3finedv01d/webxr-layers) (`master` branch). +2. At project root, `npm install` +3. `npm run dev` + +For debugging with Oculus browser, check out [this resource](https://developer.oculus.com/documentation/oculus-browser/browser-remote-debugging/). diff --git a/docs/assests/oculusDev/ADB_AddedToPath.PNG b/docs/assests/oculusDev/ADB_AddedToPath.PNG new file mode 100644 index 0000000..e65c0fd Binary files /dev/null and b/docs/assests/oculusDev/ADB_AddedToPath.PNG differ diff --git a/docs/assests/oculusDev/ADB_PathVar.PNG b/docs/assests/oculusDev/ADB_PathVar.PNG new file mode 100644 index 0000000..be3bdbf Binary files /dev/null and b/docs/assests/oculusDev/ADB_PathVar.PNG differ diff --git a/docs/assests/oculusDev/EnableDevMode.PNG b/docs/assests/oculusDev/EnableDevMode.PNG new file mode 100644 index 0000000..7e6236a Binary files /dev/null and b/docs/assests/oculusDev/EnableDevMode.PNG differ diff --git a/docs/assests/oculusDev/OculusDeveloperMode.PNG b/docs/assests/oculusDev/OculusDeveloperMode.PNG new file mode 100644 index 0000000..c8e80e1 Binary files /dev/null and b/docs/assests/oculusDev/OculusDeveloperMode.PNG differ diff --git a/docs/assests/oculusDev/adbDevices.PNG b/docs/assests/oculusDev/adbDevices.PNG new file mode 100644 index 0000000..b8db4de Binary files /dev/null and b/docs/assests/oculusDev/adbDevices.PNG differ diff --git a/docs/assests/oculusDev/insecureOrigins.PNG b/docs/assests/oculusDev/insecureOrigins.PNG new file mode 100644 index 0000000..257852e Binary files /dev/null and b/docs/assests/oculusDev/insecureOrigins.PNG differ diff --git a/docs/assests/oculusDev/inspect.PNG b/docs/assests/oculusDev/inspect.PNG new file mode 100644 index 0000000..844ac69 Binary files /dev/null and b/docs/assests/oculusDev/inspect.PNG differ diff --git a/docs/assests/oculusDev/inspectIP8080.PNG b/docs/assests/oculusDev/inspectIP8080.PNG new file mode 100644 index 0000000..ed2b553 Binary files /dev/null and b/docs/assests/oculusDev/inspectIP8080.PNG differ diff --git a/docs/assests/oculusDev/ipaddress.PNG b/docs/assests/oculusDev/ipaddress.PNG new file mode 100644 index 0000000..6b88259 Binary files /dev/null and b/docs/assests/oculusDev/ipaddress.PNG differ diff --git a/docs/assests/oculusDev/localhost8080.PNG b/docs/assests/oculusDev/localhost8080.PNG new file mode 100644 index 0000000..89a2893 Binary files /dev/null and b/docs/assests/oculusDev/localhost8080.PNG differ diff --git a/docs/assests/oculusDev/network_address.PNG b/docs/assests/oculusDev/network_address.PNG new file mode 100644 index 0000000..b725ee5 Binary files /dev/null and b/docs/assests/oculusDev/network_address.PNG differ diff --git a/docs/assests/oculusDev/network_security.PNG b/docs/assests/oculusDev/network_security.PNG new file mode 100644 index 0000000..d15c825 Binary files /dev/null and b/docs/assests/oculusDev/network_security.PNG differ diff --git a/docs/assests/oculusDev/npmrundev.PNG b/docs/assests/oculusDev/npmrundev.PNG new file mode 100644 index 0000000..7186886 Binary files /dev/null and b/docs/assests/oculusDev/npmrundev.PNG differ diff --git a/docs/assests/oculusDev/oculus_flags.PNG b/docs/assests/oculusDev/oculus_flags.PNG new file mode 100644 index 0000000..60927a2 Binary files /dev/null and b/docs/assests/oculusDev/oculus_flags.PNG differ diff --git a/docs/assests/oculusDev/oculus_flags2.PNG b/docs/assests/oculusDev/oculus_flags2.PNG new file mode 100644 index 0000000..966ac69 Binary files /dev/null and b/docs/assests/oculusDev/oculus_flags2.PNG differ diff --git a/docs/assests/oculusDev/portforwarding.PNG b/docs/assests/oculusDev/portforwarding.PNG new file mode 100644 index 0000000..2fdfc57 Binary files /dev/null and b/docs/assests/oculusDev/portforwarding.PNG differ diff --git a/docs/assests/videoLayers/Architecture.png b/docs/assests/videoLayers/Architecture.png new file mode 100644 index 0000000..36fbf66 Binary files /dev/null and b/docs/assests/videoLayers/Architecture.png differ diff --git a/docs/assests/videoLayers/MediaLayerManager.png b/docs/assests/videoLayers/MediaLayerManager.png new file mode 100644 index 0000000..e3dacbd Binary files /dev/null and b/docs/assests/videoLayers/MediaLayerManager.png differ diff --git a/docs/assests/videoLayers/UI.png b/docs/assests/videoLayers/UI.png new file mode 100644 index 0000000..a547048 Binary files /dev/null and b/docs/assests/videoLayers/UI.png differ diff --git a/docs/assests/videoLayers/moving-controller.gif b/docs/assests/videoLayers/moving-controller.gif new file mode 100644 index 0000000..dc94f9a Binary files /dev/null and b/docs/assests/videoLayers/moving-controller.gif differ diff --git a/docs/assests/videoLayers/moving-hand.gif b/docs/assests/videoLayers/moving-hand.gif new file mode 100644 index 0000000..a8e28ac Binary files /dev/null and b/docs/assests/videoLayers/moving-hand.gif differ diff --git a/docs/assests/videoLayers/overview-static.jpg b/docs/assests/videoLayers/overview-static.jpg new file mode 100644 index 0000000..78fb4d6 Binary files /dev/null and b/docs/assests/videoLayers/overview-static.jpg differ diff --git a/docs/assests/videoLayers/resizing-fixed.gif b/docs/assests/videoLayers/resizing-fixed.gif new file mode 100644 index 0000000..64b1246 Binary files /dev/null and b/docs/assests/videoLayers/resizing-fixed.gif differ diff --git a/docs/assests/videoLayers/resizing-fluid.gif b/docs/assests/videoLayers/resizing-fluid.gif new file mode 100644 index 0000000..2c6847e Binary files /dev/null and b/docs/assests/videoLayers/resizing-fluid.gif differ diff --git a/docs/assests/videoLayers/toolbar.png b/docs/assests/videoLayers/toolbar.png new file mode 100644 index 0000000..0b1ddf6 Binary files /dev/null and b/docs/assests/videoLayers/toolbar.png differ diff --git a/index.html b/index.html index ba62363..719dd38 100644 --- a/index.html +++ b/index.html @@ -4,45 +4,195 @@ - WebXR Layer Sample + WebXR Samples + -