diff --git a/examples/ar-wiener-storm.html b/examples/ar-wiener-storm.html new file mode 100644 index 00000000..48e55db8 --- /dev/null +++ b/examples/ar-wiener-storm.html @@ -0,0 +1,495 @@ + + + + + + PlayCanvas Web Components - AR Wiener Storm + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Loading face tracker…
+
+
+
+ DIGNITY +
+ 100% +
+
WIENERS ENDURED: 0
+
+
SIM
+ + + + + diff --git a/examples/assets/models/canonical-head.glb b/examples/assets/models/canonical-head.glb new file mode 100644 index 00000000..e6e23ded Binary files /dev/null and b/examples/assets/models/canonical-head.glb differ diff --git a/examples/assets/models/canonical-head.txt b/examples/assets/models/canonical-head.txt new file mode 100644 index 00000000..8613f520 --- /dev/null +++ b/examples/assets/models/canonical-head.txt @@ -0,0 +1,9 @@ +A full head built for this repo's webcam AR examples by extending the canonical face model +from Google's MediaPipe face geometry module: a cranium ellipsoid whose front is projected +onto the canonical face mesh (exact facial relief in the interior, easing to the skull near +the face's rim), authored in canonical face space centimeters with the head at the origin +facing +Z. Derived from: + +https://github.com/google-ai-edge/mediapipe/blob/master/mediapipe/modules/face_geometry/data/canonical_face_model.obj + +Apache License 2.0 https://www.apache.org/licenses/LICENSE-2.0 diff --git a/examples/assets/models/wiener.glb b/examples/assets/models/wiener.glb new file mode 100644 index 00000000..8b3f737e Binary files /dev/null and b/examples/assets/models/wiener.glb differ diff --git a/examples/assets/models/wiener.txt b/examples/assets/models/wiener.txt new file mode 100644 index 00000000..65161aff --- /dev/null +++ b/examples/assets/models/wiener.txt @@ -0,0 +1,7 @@ +Wiener authored in Blender for this example: a bent capsule with organic radius +variation and dimpled ends, skinned to a six-bone chain (B0-B5) with smooth hat weights so +physics joints can bend it, wearing a slick raw-pork Principled material (roughness 0.17, +coat 0.25). +Authored in meters (0.134 m long, length along +Y). + +CC0 / public domain. diff --git a/examples/assets/scripts/face-tracking.mjs b/examples/assets/scripts/face-tracking.mjs index 9a6bdc17..737b378b 100644 --- a/examples/assets/scripts/face-tracking.mjs +++ b/examples/assets/scripts/face-tracking.mjs @@ -207,6 +207,20 @@ export class FaceTracking extends Script { /** @private */ _rotVel = new Vec3(); + /** + * The smoothed, prediction-led camera pose in canonical face space - the inverse of the + * head pose in camera space. A subclass that clears `_drivesEntityTransform` reads (or + * inverts) this instead of the entity transform. + * @protected + */ + _pos = new Vec3(); + + /** + * The rotation of the smoothed camera pose, alongside {@link _pos}. + * @protected + */ + _rot = new Quat(); + /** @private */ _predPos = new Vec3(); @@ -254,13 +268,13 @@ export class FaceTracking extends Script { /** @private */ _targetRot = new Quat(); - /** @private */ - _pos = new Vec3(); - - /** @private */ - _rot = new Quat(); - - /** @private */ + /** + * The observed nose bridge landmark in normalized video coordinates: the raw and + * velocity fields are maintained on every face inference, while `smooth` is advanced by + * whoever applies the anchor correction - the base class when it drives the camera, or + * a subclass pinning its own transform. + * @protected + */ _bridge = { raw: { x: 0, y: 0 }, smooth: { x: 0, y: 0 }, @@ -786,9 +800,10 @@ export class FaceTracking extends Script { /** * Places the camera on a gentle orbit around the head origin, as if the head were - * turning in front of the webcam. + * turning in front of the webcam. A subclass whose camera is not driven by the base + * class overrides this to move whatever it tracks instead. * @param {number} t - The animation time in seconds. - * @private + * @protected */ _orbitCamera(t) { const yaw = Math.sin(t * 0.5) * 30 * (Math.PI / 180); diff --git a/examples/assets/scripts/head-shadow-catcher.mjs b/examples/assets/scripts/head-shadow-catcher.mjs new file mode 100644 index 00000000..ee2a7ec0 --- /dev/null +++ b/examples/assets/scripts/head-shadow-catcher.mjs @@ -0,0 +1,244 @@ +import { + BLEND_PREMULTIPLIED, + Entity, + LAYERID_WORLD, + Layer, + SHADERLANGUAGE_GLSL, + SHADERLANGUAGE_WGSL, + Script, + StandardMaterial, + Vec3 +} from 'playcanvas'; + +/** + * The version of the engine's shader chunk API that the catcher's output override is written + * against. + */ +const CHUNKS_VERSION = '2.21'; + +/** + * Grounds virtual objects on the user's real head: an invisible head-shaped proxy that renders + * only the shadows the scene casts onto it, in the spirit of the engine's shadow catcher script. + * It also doubles as an occluder - it writes depth before the world layer renders, so world + * geometry passing behind the head depth-fails and the camera feed shows through instead. + * + * Two sources of geometry can make up the catcher: + * + * - Any model attached to this entity. A head extended from MediaPipe's canonical face mesh is + * the natural choice: the facial transformation matrix is defined as the mapping of that very + * mesh onto the tracked face, so in canonical face space its features register with the user's + * face with no transform at all, and shadows bend around the nose, brow and cheeks. + * - An ellipsoid approximating the cranium (`ellipsoid`), as a fallback when no model is + * attached. Disable it when a model supplies the geometry - two overlapping catcher surfaces + * double-darken where both are visible. + * + * The engine's stock shadow catcher darkens whatever the canvas already contains, which works + * over a rendered background but not over webcam AR, where the "background" is a DOM video + * element behind a transparent canvas. This catcher instead writes the accumulated directional + * shadow into the canvas alpha as premultiplied black, so the page compositor darkens the video + * exactly where the shadow falls. + * + * Attach to the entity that carries the head pose: an entity at the scene origin when a face + * tracking script (like `faceTracking`) establishes canonical face space as world space, or a + * child of the tracked head entity when the camera stays fixed and the head moves (like + * `trackedHead`). Local space is canonical face centimeters either way - the head at the local + * origin, facing +Z. Only directional lights with shadow casting enabled contribute, and their + * `shadowIntensity` scales the effect. + */ +export class HeadShadowCatcher extends Script { + static scriptName = 'headShadowCatcher'; + + /** + * Whether to create the fallback cranium ellipsoid. Set to false when a model attached to + * this entity supplies the catcher geometry instead. + * @type {boolean} + * @attribute + */ + ellipsoid = true; + + /** + * The center of the cranium ellipsoid in local space centimeters. For grounded contact + * shadows, place the ellipsoid so its surface coincides with the head's physics proxy + * where objects strike it - a shadow cast onto a recessed surface floats visibly away + * from the thing casting it. + * @type {Vec3} + * @attribute + */ + center = new Vec3(0, 0.5, -1.5); + + /** + * The size of the cranium ellipsoid in local space centimeters. + * @type {Vec3} + * @attribute + */ + size = new Vec3(15, 17.5, 17.5); + + /** + * The opacity of a fully shadowed pixel, from 0 (shadows invisible) to 1 (shadows are pure + * black). + * @type {number} + * @attribute + */ + strength = 0.55; + + /** + * Whether to render the catcher visibly to debug its fit: a plain lit surface instead of the + * invisible shadow-only material, so both the geometry and the shadows landing on it can be + * checked against the tracked head. + * @type {boolean} + * @attribute + */ + debug = false; + + /** + * @type {StandardMaterial|null} + * @private + */ + _material = null; + + /** + * @type {Layer|null} + * @private + */ + _layer = null; + + /** + * @type {Entity|null} + * @private + */ + _ellipsoid = null; + + /** @private */ + _modelConverted = false; + + initialize() { + // The catcher blends, so it lives in a layer whose transparent pass runs before the + // world layer: its depth then occludes world geometry passing behind the head, the same + // trick the headOccluder script uses with opaque depth-only geometry. The layer's opaque + // pass is inserted too, for the opaque surface the debug mode swaps in. + const layers = this.app.scene.layers; + const world = layers.getLayerById(LAYERID_WORLD); + const layer = new Layer({ name: 'headShadowCatcher' }); + layers.insertOpaque(layer, layers.getOpaqueIndex(world)); + layers.insertTransparent(layer, layers.getOpaqueIndex(world)); + + const camera = this.app.root.findComponent('camera'); + if (camera) camera.layers = camera.layers.concat(layer.id); + + // The catcher only sees a light's shadow map if that light is assigned to its + // layer, so join every shadow-casting directional light already in the scene + const lights = this.app.root.findComponents('light').filter( + light => light.type === 'directional' && light.castShadows + ); + for (const light of lights) { + light.layers = light.layers.concat(layer.id); + } + + const material = this.debug ? + this._createDebugMaterial() : this._createCatcherMaterial(lights.length > 0); + this._material = material; + this._layer = layer; + + if (this.ellipsoid) { + const ellipsoid = new Entity('head-shadow-catcher-ellipsoid'); + ellipsoid.addComponent('render', { + type: 'sphere', + material: material, + castShadows: false, + layers: [layer.id] + }); + ellipsoid.setLocalPosition(this.center); + ellipsoid.setLocalScale(this.size); + this.entity.addChild(ellipsoid); + this._ellipsoid = ellipsoid; + } + + this.on('destroy', () => { + if (camera) camera.layers = camera.layers.filter(id => id !== layer.id); + for (const light of lights) { + light.layers = light.layers.filter(id => id !== layer.id); + } + this._ellipsoid?.destroy(); + material.destroy(); + layers.remove(layer); + }); + } + + update(_dt) { + // A model attached to this entity becomes part of the catcher. It may not be + // instantiated yet when the script initializes, so keep looking until its mesh + // instances exist, then convert them once + if (this._modelConverted) return; + + for (const render of this.entity.findComponents('render')) { + if (render.entity === this._ellipsoid || render.meshInstances.length === 0) continue; + + render.layers = [this._layer.id]; + render.castShadows = false; + for (const meshInstance of render.meshInstances) { + meshInstance.material = this._material; + } + this._modelConverted = true; + } + } + + /** + * Creates the shadow catcher material: the engine's `shadowCatcher` flag accumulates the + * directional shadow term into `dShadowCatcher`, and an override of the final shader output + * turns it into premultiplied black with the shadow in alpha. With premultiplied blending + * the canvas gains alpha (and no color) where the shadow falls, darkening the video behind + * it. Everywhere the shadow does not fall the alpha stays 0 and the catcher is invisible. + * @param {boolean} hasLight - Whether a shadow-casting directional light joined the catcher + * layer. Without one the shader has no light uniforms to gate by (and nothing accumulates + * a shadow), so the light-dependent part of the override is dropped to keep it compiling. + * @returns {StandardMaterial} The material. + * @private + */ + _createCatcherMaterial(hasLight) { + const material = new StandardMaterial(); + material.shadowCatcher = true; + material.blendType = BLEND_PREMULTIPLIED; + material.depthWrite = true; + material.opacity = this.strength; + + // The color output is discarded by the override, so keep the shading as cheap as possible + material.diffuse.set(0, 0, 0); + material.specular.set(0, 0, 0); + material.useSkybox = false; + + // The alpha is gated by how much direct light the surface would receive: a face + // turned away from the light has nothing for an occluder to take away, and without + // the gate the shadow map wraps the darkening around the terminator onto the far + // side of the head. The gate assumes the catcher's light is in slot 0, which holds + // here because only shadow-casting directional lights join the catcher layer. + material.shaderChunksVersion = CHUNKS_VERSION; + const facingGlsl = hasLight ? + 'clamp(dot(litArgs_worldNormal, -light0_direction), 0.0, 1.0)' : '1.0'; + const facingWgsl = hasLight ? + 'clamp(dot(litArgs_worldNormal, -uniform.light0_direction), 0.0, 1.0)' : '1.0'; + material.getShaderChunks(SHADERLANGUAGE_GLSL).set('outlineOutputPS', ` + float catcherFacing = ${facingGlsl}; + gl_FragColor = vec4(0.0, 0.0, 0.0, (1.0 - dShadowCatcher) * litArgs_opacity * catcherFacing); + `); + material.getShaderChunks(SHADERLANGUAGE_WGSL).set('outlineOutputPS', ` + let catcherFacing = ${facingWgsl}; + output.color = vec4f(0.0, 0.0, 0.0, (1.0 - dShadowCatcher) * litArgs_opacity * catcherFacing); + `); + + material.update(); + return material; + } + + /** + * Creates the debug material: an ordinary lit surface, so the ellipsoid's fit and the + * shadows landing on it are both visible. + * @returns {StandardMaterial} The material. + * @private + */ + _createDebugMaterial() { + const material = new StandardMaterial(); + material.diffuse.set(0.35, 0.55, 0.9); + material.update(); + return material; + } +} diff --git a/examples/assets/scripts/tracked-head.mjs b/examples/assets/scripts/tracked-head.mjs new file mode 100644 index 00000000..d794122c --- /dev/null +++ b/examples/assets/scripts/tracked-head.mjs @@ -0,0 +1,139 @@ +import { Mat4, Quat, Vec3 } from 'playcanvas'; + +import { FaceTracking } from './face-tracking.mjs'; + +/** + * Face tracking with the world anchored to the room instead of the head: the camera stays + * pinned at the origin of MediaPipe's camera space (the webcam), and the tracked head pose + * drives the `head` entity instead. Everything else in the scene lives in a frame that stays + * put when the user moves - so world-simulated physics can be dodged by moving your head, + * where the base class's head-locked frame would drag it along with the face. + * + * The `faceTracking` base class maintains the smoothed camera-in-face-space pose either way; + * this subclass simply applies its inverse (the head pose in camera space) to the head entity + * each frame. In `?sim` and no-camera fallback modes the synthetic orbit moves the head in + * front of the fixed camera, so the scene stays dodge-able there too. + * + * Everything head-locked (occluders, shadow catchers, hats) should be parented under the head + * entity. Scene units remain centimeters, with the head roughly 30-60 in front of the camera + * at negative Z. + */ +export class TrackedHead extends FaceTracking { + static scriptName = 'trackedHead'; + + /** + * The entity driven with the tracked head pose. + * @type {import('playcanvas').Entity} + * @attribute + */ + head = null; + + /** + * The base class maintains the smoothed pose; this subclass applies it to the head + * entity instead of the camera. + * @protected + */ + _drivesEntityTransform = false; + + /** @private */ + _seen = false; + + /** @private */ + _invRot = new Quat(); + + /** @private */ + _invPos = new Vec3(); + + /** @private */ + _simMat = new Mat4(); + + /** @private */ + _simCamPos = new Vec3(); + + /** @private */ + _simTarget = new Vec3(); + + /** @private */ + _anchorWorld = new Vec3(); + + /** @private */ + _desired = new Vec3(); + + /** @private */ + _corrected = new Vec3(); + + /** + * @param {number} _dt - The delta time in seconds. + * @protected + */ + _onUpdated(_dt) { + if (this._facePresent) this._seen = true; + if (!this._seen || !this.head?.setPosition) return; + + // The head pose in camera space is the inverse of the smoothed camera pose the + // base class maintains in face space + this._invRot.copy(this._rot).invert(); + this._invRot.transformVector(this._pos, this._invPos).mulScalar(-1); + this.head.setPosition(this._invPos); + this.head.setRotation(this._invRot); + + // Pin the pose: shift the head so the canonical anchor point projects exactly onto + // the observed nose bridge landmark - the same correction the base class applies to + // its face-locked camera. The matrix alone drifts sideways on head turns, and the + // drift reads as a horizontal offset once the head (rather than the camera) moves. + if (this.anchorCorrection && this._facePresent && this._bridge.seeded && this.entity.camera) { + const bridge = this._bridge; + bridge.smooth.x += (bridge.raw.x - bridge.smooth.x) * this._k; + bridge.smooth.y += (bridge.raw.y - bridge.smooth.y) * this._k; + this._toScreen(bridge.smooth, this._tmpScreen); + + // screenToWorld's distance is measured along the pixel ray, so use the radial + // camera-to-anchor distance (the camera sits at the origin): the shift then + // preserves that distance and a single pass pins the anchor to the pixel + this._invRot.transformVector(this.anchorPoint, this._anchorWorld).add(this._invPos); + const depth = this._anchorWorld.length(); + if (depth > 5) { + this.entity.camera.screenToWorld(this._tmpScreen.x, this._tmpScreen.y, depth, this._desired); + this._corrected.copy(this._invPos).add(this._desired).sub(this._anchorWorld); + this.head.setPosition(this._corrected); + } + } + } + + /** + * The synthetic orbit of the sim and fallback modes, flipped: the camera stays pinned + * at the origin and the head flies the inverse orbit in front of it. + * @param {number} t - The animation time in seconds. + * @protected + */ + _orbitCamera(t) { + this.entity.setPosition(0, 0, 0); + this.entity.setEulerAngles(0, 0, 0); + + if (!this.head?.setPosition) return; + + const yaw = Math.sin(t * 0.5) * 30 * (Math.PI / 180); + const pitch = Math.sin(t * 0.31) * 9 * (Math.PI / 180); + const dist = 46; + const eyeY = 2.5; + + this._simCamPos.set( + Math.sin(yaw) * Math.cos(pitch) * dist, + eyeY + Math.sin(pitch) * dist, + Math.cos(yaw) * Math.cos(pitch) * dist + ); + this._simMat.setLookAt(this._simCamPos, this._simTarget.set(0, eyeY, 0), Vec3.UP).invert(); + this._simMat.getTranslation(this._invPos); + this._invRot.setFromMat4(this._simMat); + + // The base orbit looks at the head, so its inverse only turns the head in place. + // A real user also leans and bobs - add that, so the synthetic modes show throws + // being dodged as well + this._invPos.x += Math.sin(t * 0.8) * 10; + this._invPos.y += Math.sin(t * 1.1) * 3; + this._invPos.z += Math.sin(t * 0.5) * 5; + + this.head.setPosition(this._invPos); + this.head.setRotation(this._invRot); + } +} diff --git a/examples/assets/scripts/video-ibl.mjs b/examples/assets/scripts/video-ibl.mjs new file mode 100644 index 00000000..19c3fdd5 --- /dev/null +++ b/examples/assets/scripts/video-ibl.mjs @@ -0,0 +1,200 @@ +import { EnvLighting, Script, Texture } from 'playcanvas'; + +/** The fraction of the equirect's width covered by the camera's actual view (~100 degrees). */ +const BAND_WIDTH = 0.28; + +/** The fraction of the equirect's height covered by the camera's actual view (~63 degrees). */ +const BAND_HEIGHT = 0.35; + +/** + * Lights the scene with the live camera feed: a low-frequency image-based lighting estimate + * rebuilt from the video every few moments, so virtual objects pick up the room's real color + * temperature, brightness and rough bright-region direction instead of a canned environment. + * + * A single fixed webcam cannot see the light that actually falls on the subject - the feed + * shows the room behind them, one camera's field of view of it - so this is an estimate in + * the spirit of mobile AR lighting estimation, not a true environment capture. The frame is + * stretched over the whole sphere to set the ambient average everywhere, and the region the + * camera genuinely observes is painted into its band of the panorama, so reflections lean + * roughly the right way. Prefiltering does the rest: at these resolutions the result is soft + * ambient and glossy tint, not mirror reflections. + * + * Pairs with the `cameraFeed` script: this script idles until `camera:ready` fires, leaving + * whatever lighting the scene declares (like a `pc-sky`) untouched as the fallback - so `?sim` + * pages and denied camera permissions keep working unchanged. + */ +export class VideoIbl extends Script { + static scriptName = 'videoIbl'; + + /** + * The intensity of the video lighting, applied to the scene's skybox intensity while the + * estimate is live. The scene's previous intensity is restored if the script is destroyed. + * @type {number} + * @attribute + */ + intensity = 0.5; + + /** + * The seconds between lighting rebuilds. Room lighting changes slowly, so a leisurely + * cadence costs nothing visible. + * @type {number} + * @attribute + */ + interval = 1; + + /** + * Whether the camera feed is displayed mirrored (the `cameraFeed` default). The panorama + * is flipped to match, so on-screen room features tint from the side they appear on. + * @type {boolean} + * @attribute + */ + mirror = true; + + /** + * The rotation of the panorama in degrees, for nudging the observed band away from the + * camera's view direction if directional reflections matter. + * @type {number} + * @attribute + */ + rotation = 0; + + /** + * The width of the panorama in texels (the height is half). Lighting is prefiltered, so + * small stays smooth - raise it only if glossy reflections need more shape. + * @type {number} + * @attribute + */ + resolution = 64; + + /** @private */ + _source = null; + + /** @private */ + _canvas = null; + + /** @private */ + _ctx = null; + + /** @private */ + _texture = null; + + /** @private */ + _lightingSource = null; + + /** @private */ + _atlas = null; + + /** @private */ + _timer = Infinity; + + /** @private */ + _active = false; + + /** @private */ + _prevAtlas = null; + + /** @private */ + _prevIntensity = 1; + + initialize() { + const onReady = (video) => { + this._source = video; + this._timer = Infinity; // rebuild on the next update + }; + this.app.on('camera:ready', onReady); + + this.on('destroy', () => { + this.app.off('camera:ready', onReady); + if (this._active) { + this.app.scene.envAtlas = this._prevAtlas; + this.app.scene.skyboxIntensity = this._prevIntensity; + } + this._atlas?.destroy(); + this._lightingSource?.destroy(); + this._texture?.destroy(); + }); + } + + /** + * @param {number} dt - The delta time since the last frame in seconds. + */ + update(dt) { + if (!this._source) return; + + this._timer += dt; + if (this._timer < this.interval) return; + this._timer = 0; + + this._rebuild(); + } + + /** + * Rebuilds the lighting estimate from the current video frame. + * @private + */ + _rebuild() { + const width = Math.max(16, this.resolution); + const height = width / 2; + + if (!this._canvas) { + this._canvas = document.createElement('canvas'); + this._canvas.width = width; + this._canvas.height = height; + this._ctx = this._canvas.getContext('2d', { willReadFrequently: false }); + // Mipmaps matter: with them the cubemap projection samples each texel once, + // without them it falls back to 1024 samples per texel + this._texture = new Texture(this.app.graphicsDevice, { + name: 'video-ibl-equirect', + width: width, + height: height, + mipmaps: true + }); + } + + const ctx = this._ctx; + ctx.save(); + if (this.mirror) { + ctx.translate(width, 0); + ctx.scale(-1, 1); + } + + // The whole sphere gets the stretched frame - the low-frequency average from every + // direction - and the band the camera actually observes gets the frame at its true + // extent, so bright regions pull reflections from the right way. The camera looks + // down -Z, which the engine's equirect mapping puts at the panorama seam (u = 0), + // so the band is drawn wrapping across both edges. + ctx.drawImage(this._source, 0, 0, width, height); + const bandW = Math.round(width * BAND_WIDTH); + const bandH = Math.round(height * BAND_HEIGHT); + const bandY = Math.round((height - bandH) / 2); + const seam = ((this.rotation / 360) % 1 + 1) % 1; + const bandX = Math.round(seam * width - bandW / 2); + ctx.drawImage(this._source, bandX - width, bandY, bandW, bandH); + ctx.drawImage(this._source, bandX, bandY, bandW, bandH); + ctx.drawImage(this._source, bandX + width, bandY, bandW, bandH); + ctx.restore(); + + this._texture.setSource(this._canvas); + this._lightingSource = EnvLighting.generateLightingSource(this._texture, { + target: this._lightingSource, + size: 64 + }); + // A small atlas with modest sample counts: the source is tiny and low-frequency, + // and the engine's 512px/1024-sample defaults would burn tens of milliseconds of + // GPU time per rebuild for detail that does not exist + this._atlas = EnvLighting.generateAtlas(this._lightingSource, { + target: this._atlas, + size: 128, + numReflectionSamples: 64, + numAmbientSamples: 256 + }); + + if (!this._active) { + this._active = true; + this._prevAtlas = this.app.scene.envAtlas; + this._prevIntensity = this.app.scene.skyboxIntensity; + this.app.scene.skyboxIntensity = this.intensity; + } + this.app.scene.envAtlas = this._atlas; + } +} diff --git a/examples/assets/scripts/wiener-storm.mjs b/examples/assets/scripts/wiener-storm.mjs new file mode 100644 index 00000000..28f81f24 --- /dev/null +++ b/examples/assets/scripts/wiener-storm.mjs @@ -0,0 +1,473 @@ +import { Entity, Quat, Script, Vec2, Vec3 } from 'playcanvas'; + +/** The local Y centers of the six physics segments, one per skin bone, in authored meters. */ +const SEG_CENTERS = [-0.0575, -0.0345, -0.0115, 0.0115, 0.0345, 0.0575]; + +/** The local Y positions of the five flex joints between them, in authored meters. */ +const JOINT_HEIGHTS = [-0.046, -0.023, 0, 0.023, 0.046]; + +/** The capsule length of one segment in authored meters (overlapping the neighbors a little). */ +const SEG_HEIGHT = 0.027; + +/** The capsule radius of one segment in authored meters. */ +const SEG_RADIUS = 0.0115; + +/** How far one flex joint may bend, in degrees. Five joints share the total fold. */ +const BEND_LIMIT = 10; + +/** How far one flex joint may twist about the wiener's length, in degrees - a raw wiener + * flexes but barely twists at all. */ +const TWIST_LIMIT = 1.5; + +/** The angular spring stiffness pulling each joint back straight. */ +const BEND_STIFFNESS = 6; + +/** The angular spring stiffness resisting twist. */ +const TWIST_STIFFNESS = 8; + +/** + * Lobs a steady barrage of wieners at the user's head. Works in MediaPipe's camera space, as + * established by a head tracking script (like `trackedHead`): the camera is pinned at the + * origin, the tracked head moves in front of it at negative Z, and the world - including the + * flying wieners - is anchored to the room, measured in centimeters. Each wiener spawns on the + * hemisphere between the head and the screen and is lobbed on an arc aimed at the head's + * position at launch, so moving your head actually dodges what is already in the air. Gravity + * is softened and a little air drag bleeds speed off on the way in (the launch solve accounts + * for both), so a throw leaves the hand brisk but arrives gently - a toss, not a fastball. An + * invisible physics proxy for the head (see the example markup) bounces them away. + * + * Each wiener bends like the real, raw thing: it is a chain of six capsule rigid bodies linked + * by 6dof joints whose angular springs pull it back straight, and the model is a skinned mesh + * whose six bones ride the segment bodies one to one. With five flex points along the length, + * an impact folds the chain around whatever it hit in a smooth curve and the springs wobble it + * straight again - no scripted deformation, just physics. + * + * Throwing runs while a face is tracked (`face:found`/`face:lost`), which the `?sim` and + * no-camera fallback modes of the face tracking script also report. + * + * The storm escalates while a face is tracked: every `escalation` seconds the category climbs + * (to a maximum of 5) and the throw rate grows with it, so staying in front of the camera gets + * progressively riskier. + * + * Fires the following events on the application: + * + * - `wiener:hit` - Fired each time a wiener strikes the target head entity, with the striking + * segment's speed in centimeters per second. + * - `wiener:missed` - Fired when a wiener retires without ever touching the head - with launch + * aiming leading the target, a miss means it was dodged. + * - `storm:category` - Fired with the storm category (1-5) when it changes, including the + * initial category when throwing starts. + */ +export class WienerStorm extends Script { + static scriptName = 'wienerStorm'; + + /** + * The wiener model, a GLB container asset authored in meters with its length along +Y and + * skinned to a `B0`-`B5` bone chain. + * @type {import('playcanvas').Asset} + * @attribute + */ + wienerAsset = null; + + /** + * The head proxy entity: throws are aimed at its position at launch, and a wiener + * colliding with it fires `wiener:hit`. + * @type {Entity} + * @attribute + */ + target = null; + + /** + * The average number of wieners thrown per second at storm category 1. Each category + * above that adds 30%. + * @type {number} + * @attribute + */ + rate = 2.5; + + /** + * The seconds of active throwing per storm category. 0 keeps the storm steady at + * category 1. + * @type {number} + * @attribute + */ + escalation = 25; + + /** + * The scale applied to the wiener model, converting its authored meters to scene units. + * @type {number} + * @attribute + */ + modelScale = 100; + + /** + * The nearest distance from the head that a wiener may spawn, in centimeters. + * @type {number} + * @attribute + */ + minDistance = 55; + + /** + * The farthest distance from the head that a wiener may spawn, in centimeters. + * @type {number} + * @attribute + */ + maxDistance = 90; + + /** + * The maximum horizontal angle from dead ahead that a wiener may spawn at, in degrees. Up to + * 90 covers the frontal hemisphere; beyond 90 lets throws come in from slightly behind the + * ears. + * @type {number} + * @attribute + */ + spread = 105; + + /** + * The elevation range a wiener may spawn in, in degrees above the head's horizon. + * @type {Vec2} + * @attribute + */ + elevation = new Vec2(-8, 42); + + /** + * The aim offset from the target's position, in centimeters - roughly the middle of + * the face. + * @type {Vec3} + * @attribute + */ + aim = new Vec3(0, 1, 2); + + /** + * The radius of the random aim error, in centimeters. + * @type {number} + * @attribute + */ + aimJitter = 5; + + /** + * Gravity in centimeters per second squared. Real gravity is 981, which slams even a + * gentle lob into the head at around 3 meters per second - the default is softer, so + * arcs stay believable but arrive slower. + * @type {number} + * @attribute + */ + gravity = 450; + + /** + * The air drag on a flying wiener, as an exponential decay rate per second. A throw + * leaves the hand brisk and sheds speed on the way in, arriving gently. The launch + * solve accounts for it, and it matches Ammo's damping model exactly. 0 disables it. + * @type {number} + * @attribute + */ + drag = 1; + + /** + * The bounciness of a wiener, 0 to 1. + * @type {number} + * @attribute + */ + restitution = 0.4; + + /** + * The seconds a wiener lives before it is removed. + * @type {number} + * @attribute + */ + lifetime = 6; + + /** + * The maximum number of wieners alive at once. + * @type {number} + * @attribute + */ + maxLive = 12; + + /** @private */ + _live = []; + + /** @private */ + _active = false; + + /** @private */ + _timer = 0.6; + + /** @private */ + _activeTime = 0; + + /** @private */ + _category = 0; + + /** @private */ + _prevGravity = new Vec3(); + + /** @private */ + _prevTimeStep = 1 / 60; + + /** @private */ + _prevIterations = 10; + + /** @private */ + _tmpVec = new Vec3(); + + /** @private */ + _tmpVec2 = new Vec3(); + + /** @private */ + _tmpQuat = new Quat(); + + initialize() { + // The scene is in centimeters, so gravity is in centimeters per second squared + const gravity = this.app.systems.rigidbody.gravity; + this._prevGravity.copy(gravity); + gravity.set(0, -this.gravity, 0); + + // Six-body chains need more solver iterations than Ammo's default 10, or hard + // head impacts visibly stretch the joints apart for a few frames. The engine + // does not surface this, so it is set on the dynamics world directly. + const solverInfo = this.app.systems.rigidbody.dynamicsWorld?.getSolverInfo?.(); + this._prevIterations = solverInfo?.get_m_numIterations?.() ?? 10; + solverInfo?.set_m_numIterations?.(20); + + // A head impact can spin a light segment through well over 90 degrees inside a + // single 60Hz physics step - past every joint limit before the solver ever sees + // it. Stepping physics at 180Hz keeps the worst violations near the limits. + this._prevTimeStep = this.app.systems.rigidbody.fixedTimeStep; + this.app.systems.rigidbody.fixedTimeStep = 1 / 180; + + const onFound = () => { + this._active = true; + }; + const onLost = () => { + this._active = false; + }; + this.app.on('face:found', onFound); + this.app.on('face:lost', onLost); + + this.on('destroy', () => { + this.app.off('face:found', onFound); + this.app.off('face:lost', onLost); + for (const wiener of this._live) { + wiener.container.destroy(); + } + this._live.length = 0; + this.app.systems.rigidbody.gravity.copy(this._prevGravity); + this.app.systems.rigidbody.fixedTimeStep = this._prevTimeStep; + this.app.systems.rigidbody.dynamicsWorld?.getSolverInfo?.().set_m_numIterations?.(this._prevIterations); + }); + } + + /** + * @param {number} dt - The delta time since the last frame in seconds. + */ + update(dt) { + if (this._active) { + this._activeTime += dt; + const category = this.escalation > 0 ? + Math.min(5, 1 + Math.floor(this._activeTime / this.escalation)) : 1; + if (category !== this._category) { + this._category = category; + this.app.fire('storm:category', category); + } + + this._timer -= dt; + if (this._timer <= 0) { + this._throw(); + // Uneven pacing reads as thrown rather than machine-fired, and the rate + // climbs with the storm category + const rate = Math.max(0.1, this.rate) * (1 + 0.3 * (this._category - 1)); + this._timer = (1 / rate) * (0.7 + Math.random() * 0.6); + } + } + + // Ride the skin bones on the physics segments and retire spent wieners + for (let i = this._live.length - 1; i >= 0; i--) { + const wiener = this._live[i]; + wiener.age += dt; + + for (let b = 0; b < wiener.bones.length; b++) { + const segment = wiener.segments[b]; + const bone = wiener.bones[b]; + const rot = segment.getRotation(); + rot.transformVector(wiener.posOffsets[b], this._tmpVec).add(segment.getPosition()); + this._tmpQuat.mul2(rot, wiener.rotOffsets[b]); + bone.setPosition(this._tmpVec); + bone.setRotation(this._tmpQuat); + } + + if (wiener.age > this.lifetime || wiener.segments[2].getPosition().y < -160) { + if (!wiener.hitHead) this.app.fire('wiener:missed'); + wiener.container.destroy(); + this._live.splice(i, 1); + } + } + } + + /** + * Spawns one wiener on the frontal hemisphere and throws it at the head. + * @private + */ + _throw() { + // An unresolved target reference arrives as a raw string, so guard on the method + if (!this.wienerAsset?.resource || !this.target?.getPosition) return; + + while (this._live.length >= this.maxLive) { + const retired = this._live.shift(); + if (!retired.hitHead) this.app.fire('wiener:missed'); + retired.container.destroy(); + } + + // A spawn point on the hemisphere between the head and the screen: +Z is out of + // the screen toward the viewer, so azimuth 0 comes straight out of the camera + const targetPos = this.target.getPosition(); + const azimuth = (Math.random() * 2 - 1) * this.spread * Math.PI / 180; + const elev = (this.elevation.x + Math.random() * (this.elevation.y - this.elevation.x)) * Math.PI / 180; + const distance = this.minDistance + Math.random() * (this.maxDistance - this.minDistance); + const pos = new Vec3( + Math.sin(azimuth) * Math.cos(elev), + Math.sin(elev), + Math.cos(azimuth) * Math.cos(elev) + ).mulScalar(distance).add(targetPos); + + // Solve the throw that lands on the (jittered) aim point. With drag the flight + // obeys v' = g - drag * v, whose closed form still has an exact launch velocity: + // v0 = g/d + (delta - g*T/d) / ((1 - e^(-d*T)) / d), with gravity g on Y only + const flight = 0.55 + Math.random() * 0.3; + const jitter = this._tmpVec.set( + (Math.random() * 2 - 1) * this.aimJitter, + (Math.random() * 2 - 1) * this.aimJitter, + (Math.random() * 2 - 1) * this.aimJitter * 0.4 + ); + const delta = new Vec3().add2(this.aim, jitter).add(targetPos).sub(pos); + const g = -this.gravity; + const drag = Math.max(0, this.drag); + let velocity; + if (drag > 0.001) { + const fade = (1 - Math.exp(-drag * flight)) / drag; + velocity = new Vec3( + delta.x / fade, + g / drag + (delta.y - g * flight / drag) / fade, + delta.z / fade + ); + } else { + velocity = delta.mulScalar(1 / flight); + velocity.y += 0.5 * this.gravity * flight; + } + + const scale = 0.9 + Math.random() * 0.22; + const k = this.modelScale * scale; + + // The container holds the whole wiener for lifecycle only - the segment bodies fly + // in world space regardless of their parent + const container = new Entity('wiener'); + container.setPosition(pos); + this._tmpQuat.setFromEulerAngles(Math.random() * 360, Math.random() * 360, Math.random() * 360); + container.setRotation(this._tmpQuat); + this.app.root.addChild(container); + + const wiener = { + container: container, + segments: [], + bones: [], + posOffsets: [], + rotOffsets: [], + age: 0, + hitHead: false + }; + + for (const center of SEG_CENTERS) { + const segment = new Entity('wiener-segment'); + segment.setLocalPosition(0, center * k, 0); + container.addChild(segment); + segment.addComponent('collision', { + type: 'capsule', + radius: SEG_RADIUS * k, + height: SEG_HEIGHT * k + }); + segment.addComponent('rigidbody', { + type: 'dynamic', + mass: 0.02, + restitution: this.restitution, + friction: 0.4, + // Ammo damping is exponential per second, matching the launch solve + linearDamping: 1 - Math.exp(-drag), + angularDamping: 0.5 + }); + + segment.collision.on('collisionstart', (result) => { + // A settling wiener restarts the contact every micro-bounce, so a direct + // hit only counts once per wiener + if (!wiener.hitHead && this.target && result.other === this.target) { + wiener.hitHead = true; + // The post-bounce speed is a fine proxy for how hard it landed + this.app.fire('wiener:hit', segment.rigidbody.linearVelocity.length()); + } + }); + + wiener.segments.push(segment); + } + + // The 6dof joints make the chain flex: bending swings about the local X and Z of + // each junction, twisting about Y, and angular springs pull the wiener straight + // again - a raw wiener is floppy, not a rag + for (let j = 0; j < JOINT_HEIGHTS.length; j++) { + const joint = new Entity('wiener-joint'); + joint.setLocalPosition(0, JOINT_HEIGHTS[j] * k, 0); + container.addChild(joint); + joint.addComponent('joint', { + type: '6dof', + entityA: wiener.segments[j], + entityB: wiener.segments[j + 1], + angularMotionX: 'limited', + angularMotionY: 'limited', + angularMotionZ: 'limited', + angularLimitsX: new Vec2(-BEND_LIMIT, BEND_LIMIT), + angularLimitsY: new Vec2(-TWIST_LIMIT, TWIST_LIMIT), + angularLimitsZ: new Vec2(-BEND_LIMIT, BEND_LIMIT), + angularStiffness: new Vec3(BEND_STIFFNESS, TWIST_STIFFNESS, BEND_STIFFNESS) + }); + + // At Ammo's default stop ERP a hard head impact blows straight through the + // limits for a few frames, folding the wiener far past them. Stiffen the + // limit correction on every axis, set on the native constraint directly + // (2 is BT_CONSTRAINT_STOP_ERP) + const constraint = joint.joint.constraint; + for (let axis = 0; axis < 6; axis++) { + constraint?.setParam(2, 0.8, axis); + } + } + + // The skinned model rides along: each bone copies its segment body every frame, + // through the offsets between them captured at this rest pose + const model = this.wienerAsset.resource.instantiateRenderEntity(); + model.setLocalScale(k, k, k); + container.addChild(model); + + for (let b = 0; b < wiener.segments.length; b++) { + const segment = wiener.segments[b]; + const bone = model.findByName(`B${b}`); + const invRot = this._tmpQuat.copy(segment.getRotation()).invert(); + wiener.bones.push(bone); + wiener.posOffsets.push(invRot.transformVector(new Vec3().sub2(bone.getPosition(), segment.getPosition()))); + wiener.rotOffsets.push(new Quat().mul2(invRot, bone.getRotation())); + } + + // Throw the chain as one rigid motion: a shared tumble plus the velocity that + // tumble adds at each segment's offset from the middle + const omega = new Vec3( + Math.random() * 2 - 1, + Math.random() * 2 - 1, + Math.random() * 2 - 1 + ).normalize().mulScalar(2 + Math.random() * 2.5); + const mid = this._tmpVec2.copy(container.getPosition()); + for (const segment of wiener.segments) { + const arm = this._tmpVec.sub2(segment.getPosition(), mid); + const spin = new Vec3().cross(omega, arm); + segment.rigidbody.linearVelocity = spin.add(velocity); + segment.rigidbody.angularVelocity = omega; + } + + this._live.push(wiener); + } +} diff --git a/examples/assets/sounds/wiener-slap.mp3 b/examples/assets/sounds/wiener-slap.mp3 new file mode 100644 index 00000000..173c6848 Binary files /dev/null and b/examples/assets/sounds/wiener-slap.mp3 differ diff --git a/examples/js/example-list.mjs b/examples/js/example-list.mjs index 2a6a3c81..5b41991c 100644 --- a/examples/js/example-list.mjs +++ b/examples/js/example-list.mjs @@ -39,6 +39,7 @@ export const examples = [ { name: 'AR Hand Gestures', path: 'ar-hand-gestures.html', category: 'Webcam AR' }, { name: 'AR Optic Blast', path: 'ar-optic-blast.html', category: 'Webcam AR' }, { name: 'AR Sunglasses', path: 'ar-sunglasses.html', category: 'Webcam AR' }, + { name: 'AR Wiener Storm', path: 'ar-wiener-storm.html', category: 'Webcam AR' }, { name: 'Head Tracked Window', path: 'head-tracked-window.html', category: 'Webcam AR' }, // Controls { name: 'First Person Teleport', path: 'first-person-teleport.html', category: 'Controls' },