From 8f5bd54ceac80cad701146c761eedb474cbdbf52 Mon Sep 17 00:00:00 2001 From: Jamie Cope Date: Fri, 5 Jan 2024 15:56:52 -0500 Subject: [PATCH 01/12] WIP Circle annotation refactor to HTML based solution --- package.json | 1 - source/client/annotations/CircleSprite.ts | 352 +++++++------------ source/client/components/CVARManager.ts | 12 - source/client/components/CVAnnotationView.ts | 19 - source/client/ui/explorer/styles.scss | 85 +++-- 5 files changed, 175 insertions(+), 294 deletions(-) diff --git a/package.json b/package.json index 9f7095ba..4aecf6f0 100755 --- a/package.json +++ b/package.json @@ -71,7 +71,6 @@ "stream-browserify": "^3.0.0", "style-loader": "^3.3.1", "three": "^0.157.0", - "three-bmfont-text": "git+https://github.com/Smithsonian/three-bmfont-text.git#50da80d887", "tinymce": "^6.3.1", "toposort": "^2.0.2", "webdav": "^5.3.0", diff --git a/source/client/annotations/CircleSprite.ts b/source/client/annotations/CircleSprite.ts index 743efde2..32d74d28 100755 --- a/source/client/annotations/CircleSprite.ts +++ b/source/client/annotations/CircleSprite.ts @@ -15,148 +15,46 @@ * limitations under the License. */ -import { Camera, ArrayCamera, PerspectiveCamera, Vector3, Quaternion, Matrix4, Group, - Mesh, RingGeometry, MeshBasicMaterial, BufferGeometry, RawShaderMaterial, - GreaterDepth, CircleGeometry, MathUtils, GLSL3 } from "three"; -import * as createTextGeometry from "three-bmfont-text"; -import * as createTextShader from "three-bmfont-text/shaders/msdf"; +import { Group, Mesh, MeshBasicMaterial, BufferGeometry } from "three"; -import { customElement, html } from "@ff/ui/CustomElement"; +import { customElement, html, render } from "@ff/ui/CustomElement"; +import math from "@ff/core/math"; +import FFColor from "@ff/core/Color"; import "@ff/ui/Button"; -import GPUPicker from "@ff/three/GPUPicker"; - import AnnotationSprite, { Annotation, AnnotationElement } from "./AnnotationSprite"; import UniversalCamera from "@ff/three/UniversalCamera"; import AnnotationFactory from "./AnnotationFactory"; -import CVAssetReader from "client/components/CVAssetReader"; import {unsafeHTML} from 'lit-html/directives/unsafe-html.js'; //////////////////////////////////////////////////////////////////////////////// -const _vec3a = new Vector3(); -const _vec3b = new Vector3(); -const _quat1 = new Quaternion(); -const _mat4 = new Matrix4(); +const _color = new FFColor(); export default class CircleSprite extends AnnotationSprite { static readonly typeName: string = "Circle"; - private _isExpanded = false; - - protected static readonly behindOpacity = 0.2; - protected offset: Group; protected anchorMesh: Mesh; - protected ringMesh: Mesh; - protected ringGeometry: RingGeometry; - protected ringMaterialA: MeshBasicMaterial; - protected ringMaterialB: MeshBasicMaterial; - - protected markerGeometry: BufferGeometry; - protected markerMaterialA: RawShaderMaterial; - protected markerMaterialB: RawShaderMaterial; - protected markerA: Mesh; - protected markerB: Mesh; - - // Temporary until annotation scale implementation is resolved - xrScale: number = 1.0; - - isWebGL2: boolean = false; - - constructor(annotation: Annotation, assetReader: CVAssetReader) + constructor(annotation: Annotation) { super(annotation); - this._isExpanded = annotation.data.expanded; - this.offset = new Group(); this.offset.matrixAutoUpdate = false; this.add(this.offset); - this.ringGeometry = new RingGeometry(0.45, 0.5, 32); - - this.ringMaterialA = new MeshBasicMaterial(); - this.ringMaterialB = new MeshBasicMaterial({ - depthFunc: GreaterDepth, - depthWrite: false, - opacity: CircleSprite.behindOpacity, - transparent: true - }); - - this.ringMesh = new Mesh( - this.ringGeometry, - this.ringMaterialA, - ); - - const ringMeshB = new Mesh( - this.ringGeometry, - this.ringMaterialB, - ); - - this.ringMaterialA.toneMapped = false; - this.ringMaterialB.toneMapped = false; - - const innerCircle = new Mesh( - new CircleGeometry(0.45, 32), - new MeshBasicMaterial({ color: 0, opacity: 0.65, transparent: true }), - ); - - innerCircle.matrixAutoUpdate = false; - innerCircle.position.set(0, 0, 0.005); - innerCircle.updateMatrix(); - this.anchorMesh = new Mesh( new BufferGeometry(), new MeshBasicMaterial() ); this.anchorMesh.frustumCulled = false; - this.offset.add(this.anchorMesh, this.ringMesh, ringMeshB, innerCircle); - - this.markerGeometry = null; - this.markerA = null; - this.markerB = null; - - assetReader.fontReader.load("fonts/Roboto-Bold").then(font => { - this.markerMaterialA = new RawShaderMaterial(createTextShader.default({ - map: font.texture, - transparent: true, - color: 0xffffff, - isWebGL2: this.isWebGL2, - glslVersion: GLSL3, - })); - - this.markerMaterialB = new RawShaderMaterial(createTextShader.default({ - map: font.texture, - transparent: true, - opacity: CircleSprite.behindOpacity, - color: 0xffffff, - depthFunc: GreaterDepth, - depthWrite: false, - isWebGL2: this.isWebGL2, - glslVersion: GLSL3, - })); - - this.markerGeometry = createTextGeometry.default({ font: font.descriptor }); - - this.markerA = new Mesh(this.markerGeometry, this.markerMaterialA); - this.markerA.matrixAutoUpdate = false; - - this.markerB = new Mesh(this.markerGeometry, this.markerMaterialB); - this.markerB.matrixAutoUpdate = false; - - // we're async here, register marker for picking manually - GPUPicker.add(this.markerA, false); - GPUPicker.add(this.markerB, false); - this.offset.add(this.markerA, this.markerB); - - this.update(); - }); + this.offset.add(this.anchorMesh); this.update(); } @@ -165,93 +63,24 @@ export default class CircleSprite extends AnnotationSprite { this.offset = null; this.anchorMesh = null; - this.ringMesh = null; - this.ringGeometry = null; - this.ringMaterialA = null; - this.ringMaterialB = null; - this.markerGeometry = null; - this.markerMaterialA = null; - this.markerMaterialB = null; - this.markerA = null; - this.markerB = null; super.dispose(); } - update() + renderHTMLElement(element: AnnotationElement, container: HTMLElement, camera: UniversalCamera) { - const annotation = this.annotation.data; - - const c = annotation.color; - this.ringMaterialA.color.setRGB(c[0], c[1], c[2]); - this.ringMaterialB.color.setRGB(c[0], c[1], c[2]); - - //this.anchorMesh.position.set(0, 0, annotation.scale * 0.1); + super.renderHTMLElement(element, container, camera, this.anchorMesh); - if (this.markerA) { - const length = annotation.marker.length; - const scale = length > 1 ? 0.013 : 0.016; + const angleOpacity = math.scaleLimit(this.viewAngle * math.RAD2DEG, 90, 100, 1, 0); + const opacity = this.annotation.data.visible ? angleOpacity : 0; - const geometry = this.markerGeometry; - (geometry as any).update(annotation.marker); - geometry.computeBoundingBox(); - geometry.boundingBox.getCenter(_vec3a); + element.setOpacity(opacity); - this.markerA.position.set(-scale * (_vec3a.x + 1), scale * _vec3a.y, 0.01); - this.markerA.scale.set(scale, -scale, -1); - this.markerA.updateMatrix(); - - this.markerB.position.set(-scale * (_vec3a.x + 1), scale * _vec3a.y, 0.01); - this.markerB.scale.set(scale, -scale, -1); - this.markerB.updateMatrix(); - } - - super.update(); - } - - renderHTMLElement(element: AnnotationElement, container: HTMLElement, camera: UniversalCamera) - { const annotation = this.annotation.data; - let matrixCamera : PerspectiveCamera = null; - const isShowing = this.annotation.data.visible; + const isShowing = annotation.visible; this.offset.visible = isShowing; - - if(camera instanceof ArrayCamera) { - matrixCamera = ((camera as Camera) as ArrayCamera).cameras[0]; - } - - // billboard rotation - if(matrixCamera) { - _mat4.copy(matrixCamera.matrixWorldInverse); - } - else { - _mat4.copy(camera.matrixWorldInverse); - } - _mat4.multiply(this.matrixWorld); - _mat4.decompose(_vec3a, _quat1, _vec3b); - this.offset.quaternion.copy(_quat1.invert()); - - // get inverse world scale relative to user scale - this.offset.parent.matrixWorld.decompose(_vec3a, _quat1, _vec3b); - const invWorldScale = 1.0/_vec3b.x * (1.0/annotation.scale) * this.xrScale; - - // scale annotation with respect to camera distance - const vpHeight = container.offsetHeight + 250; - const vpScale = annotation.scale * 55 / vpHeight * invWorldScale; - let scaleFactor = 1; - - if (camera.isPerspectiveCamera) { - const distZ = -_vec3a.set(0, 0, 0).applyMatrix4(_mat4).z; - const theta = camera.fov * MathUtils.DEG2RAD * 0.5; - scaleFactor = Math.tan(theta) * distZ * vpScale; - } - else { - scaleFactor = camera.size * 0.5 * vpScale; - } - - this.offset.scale.setScalar(scaleFactor); - this.offset.position.set(0, (annotation.offset + 1) * scaleFactor * 0.5, 0); + this.offset.position.set(0, (annotation.offset + 1) * 0.5, 0); this.offset.updateMatrix(); @@ -261,34 +90,29 @@ export default class CircleSprite extends AnnotationSprite element.setVisible(this.getVisible()); } + // check if annotation is out of bounds and update if needed if (annotation.expanded) { - // calculate screen position of HTML sprite element - _vec3a.set(0, 0, 0).applyMatrix4(this.anchorMesh.modelViewMatrix).applyMatrix4(camera.projectionMatrix); - _vec3b.set(0.6, 0.5, 0).applyMatrix4(this.anchorMesh.modelViewMatrix).applyMatrix4(camera.projectionMatrix); - const centerX = (_vec3a.x + 1) * 0.5 * container.clientWidth; - const centerY = (1 - _vec3a.y) * 0.5 * container.clientHeight; - const offsetX = (_vec3b.x + 1) * 0.5 * container.clientWidth - centerX; - const offsetY = (1 - _vec3b.y) * 0.5 * container.clientHeight - centerY; - - let x = centerX + offsetX; - let y = centerY + offsetY; - element.classList.remove("sv-align-right", "sv-align-bottom"); - - if (x + element.offsetWidth >= container.offsetWidth) { - x = centerX - offsetX; + element.classList.add("sv-expanded"); + + let x = element.getBoundingClientRect().left - container.getBoundingClientRect().left; + let y = element.getBoundingClientRect().top - container.getBoundingClientRect().top; + + if (x + element.offsetWidth >= container.offsetWidth && !element.classList.contains("sv-align-right")) { element.classList.add("sv-align-right"); + element.requestUpdate(); } - if (y + element.offsetHeight >= container.offsetHeight) { - y = centerY - offsetY; + else if (x + element.offsetWidth < container.offsetWidth && element.classList.contains("sv-align-right")){ + element.classList.remove("sv-align-right"); + element.requestUpdate(); + } + if (y + element.offsetHeight >= container.offsetHeight && !element.classList.contains("sv-align-bottom")) { element.classList.add("sv-align-bottom"); + element.requestUpdate(); + } + else if (y + element.offsetHeight < container.offsetHeight && element.classList.contains("sv-align-bottom")) { + element.classList.remove("sv-align-bottom"); + element.requestUpdate(); } - - element.setPosition(x, y); - } - - if(this._isExpanded !== annotation.expanded) { - element.style.visibility = ""; - this._isExpanded = annotation.expanded } } @@ -296,18 +120,6 @@ export default class CircleSprite extends AnnotationSprite { return new CircleAnnotation(this); } - - protected updateHTMLElement(element: AnnotationElement) - { - element.setVisible(this.getVisible()); - - // Stops annotation box from occasionally showing before it has been positioned - if(this.annotation.data.expanded && this._isExpanded !== this.annotation.data.expanded) { - element.style.visibility = "hidden"; - } - - element.requestUpdate(); - } } AnnotationFactory.registerType(CircleSprite); @@ -317,15 +129,33 @@ AnnotationFactory.registerType(CircleSprite); @customElement("sv-circle-annotation") class CircleAnnotation extends AnnotationElement { + protected markerElement: HTMLDivElement; + protected contentElement: HTMLDivElement; + protected isExpanded = undefined; + constructor(sprite: CircleSprite) { super(sprite); + + this.onClickMarker = this.onClickMarker.bind(this); + this.onClickArticle = this.onClickArticle.bind(this); + this.onClickAudio = this.onClickAudio.bind(this); + this.onKeyDown = this.onKeyDown.bind(this); + + this.markerElement = this.appendElement("div"); + this.markerElement.classList.add("sv-marker"); + this.markerElement.addEventListener("click", this.onClickMarker); + //this.markerElement.addEventListener("keydown", this.onKeyDown); + //this.markerElement.setAttribute("tabindex", "0"); + + this.contentElement = this.appendElement("div"); + this.contentElement.classList.add("sv-annotation-body"); + this.contentElement.style.display = "none"; } setVisible(visible: boolean) { - // element is visible only if the annotation is in expanded state - super.setVisible(visible && this.sprite.annotation.data.expanded); + this.style.display = visible ? "flex" : "none"; } protected firstConnected() @@ -334,23 +164,68 @@ class CircleAnnotation extends AnnotationElement this.classList.add("sv-circle-annotation"); } - protected render() - { + protected updated(changedProperties): void { + super.updated(changedProperties); + const annotation = this.sprite.annotation; const annotationData = annotation.data; - return html`
${annotation.title}
+ // update title + this.markerElement.innerText = annotationData.marker; + + const contentTemplate = html` +
${annotation.title}
${annotationData.imageUri ? html`
${annotation.imageAltText}${annotation.imageCredit ? html`
${annotation.imageCredit}
` : null}
` : null}

${unsafeHTML(annotation.lead)}

${annotationData.audioId ? html`
` : null} - ${annotationData.articleId ? html`` : null}`; - } + ${annotationData.articleId ? html`` : null}`; - protected updated(changedProperties): void { - super.updated(changedProperties); + render(contentTemplate, this.contentElement); - const annotation = this.sprite.annotation; - const annotationData = annotation.data; + // update color + _color.fromArray(annotationData.color); + this.markerElement.style.borderColor = _color.toString(); + + // update expanded height in case annotation changed + if (this.isExpanded) { + this.contentElement.style.height = "auto"; + } + + // update expanded/collapsed + if (this.isExpanded !== annotationData.expanded) { + + this.isExpanded = annotationData.expanded; + + if (this.isExpanded) { + if(annotationData.audioId) { + this.querySelector("#audio_container").append(this.sprite.audioManager.getPlayerById(annotationData.audioId)); + } + + this.classList.add("sv-expanded"); + //this.style.minWidth = annotationData.lead.length < 40 && (!annotationData.audioId || annotationData.audioId.length == 0) ? "0" : ""; + this.contentElement.style.display = "block"; + this.contentElement.style.height = this.contentElement.scrollHeight + "px"; + } + else { + this.classList.remove("sv-expanded"); + this.contentElement.style.display = "none"; + + if(annotationData.audioId) { + this.sprite.audioManager.stop(); + } + } + } + + // Handle shifting annotation body when out-of-bounds + if (this.isExpanded) { + this.contentElement.style.removeProperty("transform"); + if (this.classList.contains("sv-align-right")) { + this.contentElement.style.transform = `translateX(-${this.offsetWidth}px)`; + } + if (this.classList.contains("sv-align-bottom")) { + this.contentElement.style.transform = `translateY(-${this.offsetHeight-this.markerElement.offsetHeight}px)`; + } + } const audioView = this.querySelector(".sv-audio-view"); if(annotationData.audioId) { @@ -364,6 +239,13 @@ class CircleAnnotation extends AnnotationElement } } + protected onClickMarker(event: MouseEvent) + { + this.contentElement.style.display = "block"; // makes sure we have a valid height when doing out-of-bounds check + event.stopPropagation(); + this.sprite.emitClickEvent(); + } + protected onClickArticle(event: MouseEvent) { event.stopPropagation(); @@ -375,4 +257,12 @@ class CircleAnnotation extends AnnotationElement event.stopPropagation(); this.sprite.emitClickEvent(); } + + protected onKeyDown(event: KeyboardEvent) + { + if (event.code === "Space" || event.code === "Enter") { + event.stopPropagation(); + this.sprite.emitClickEvent(); + } + } } \ No newline at end of file diff --git a/source/client/components/CVARManager.ts b/source/client/components/CVARManager.ts index f8e215f9..00ceaaae 100644 --- a/source/client/components/CVARManager.ts +++ b/source/client/components/CVARManager.ts @@ -375,12 +375,6 @@ export default class CVARManager extends Component this.sceneNode.ins.units.setValue(EUnitType.m); } - // Temporary until annotation scale implementation is resolved - const views = scene.getGraphComponents(CVAnnotationView); - views.forEach(component => { - component.setXRScale(ANNOTATION_SCALE); - }); - // Disable any shadow casting lights const lights = scene.getGraphComponents(CVDirectionalLight); lights.forEach(light => { @@ -450,12 +444,6 @@ export default class CVARManager extends Component setup.viewer.ins.shader.setValue(cachedShader); } - // Temporary until annotation scale implementation is resolved - const views = this.sceneNode.getGraphComponents(CVAnnotationView); - views.forEach(component => { - component.setXRScale(1.0); - }); - // Reset shadowing lights this.lightsToReset.forEach(light => { light.ins.shadowEnabled.setValue(true); diff --git a/source/client/components/CVAnnotationView.ts b/source/client/components/CVAnnotationView.ts index 5f04f421..b291ec95 100755 --- a/source/client/components/CVAnnotationView.ts +++ b/source/client/components/CVAnnotationView.ts @@ -418,20 +418,6 @@ export default class CVAnnotationView extends CObject3D this.emit({ type: "tag-update" }); } - // Temporary until annotation scale implementation is resolved - setXRScale(scale: number) - { - for (const key in this._annotations) { - const annotation = this._annotations[key]; - if(annotation.get("style") === "Circle") { - const sprite = this._sprites[annotation.id] as CircleSprite; - if (sprite) { - sprite.xrScale = scale; - } - } - } - } - protected handleARStateChange() { for (const key in this._annotations) { const annotation = this._annotations[key]; @@ -510,11 +496,6 @@ export default class CVAnnotationView extends CObject3D this._sprites[annotation.id] = sprite; this.object3D.add(sprite); this.registerPickableObject3D(sprite, true); - - // set webgl2 for circle annotations - if (isCircle) { - (sprite as CircleSprite).isWebGL2 = this.renderer.views[0].renderer.capabilities.isWebGL2; - } } protected removeSprite(annotation: Annotation) diff --git a/source/client/ui/explorer/styles.scss b/source/client/ui/explorer/styles.scss index b5e8684f..e59210a3 100644 --- a/source/client/ui/explorer/styles.scss +++ b/source/client/ui/explorer/styles.scss @@ -265,46 +265,69 @@ $pad: $canvas-border-width + $main-menu-button-size + 8px; } .sv-circle-annotation { - width: 16%; - min-width: 160px; - max-width: 260px; - padding: 4px 8px; - background-color: rgba(0, 0, 0, 0.7); - border-radius: 4px; + display: flex; - &.sv-align-right { - transform: translateX(-100%); - } - &.sv-align-bottom { - transform: translateY(-100%); + &.sv-expanded { + z-index: 2; + width: 16%; + min-width: 180px; + max-width: 280px; } - .sv-title { - padding: 0 0 2px 0; - font-weight: bold; - } - p { - @include font-ui-normal; - font-size: 0.88em; - line-height: 1.35; - color: $color-text; - margin: 0.5em 0; + .sv-marker { + background-color: rgba(0, 0, 0, 0.65); + border-radius: 50%; + text-align: center; + height: 18px; + aspect-ratio: 1/1; + border-style: solid; + border-width: 1px; + cursor: pointer; + font-size: 0.9em; + color: #fff; + -webkit-tap-highlight-color: transparent; } - .ff-button { - background: rgba(0, 0, 0, 0.01); // hack to get click events - .ff-icon { - fill: $color-icon; + .sv-annotation-body { + padding: 4px 8px; + background-color: rgba(0, 0, 0, 0.7); + border-radius: 4px; + margin: 0px 1px; + width: 100%; + + .sv-title { + padding: 0 0 2px 0; + font-weight: bold; + } + p { + @include font-ui-normal; + font-size: 0.88em; + line-height: 1.35; + color: $color-text; + margin: 0.5em 0; } + .ff-button { + background: rgba(0, 0, 0, 0.01); // hack to get click events - &:hover { - text-decoration: underline; + .ff-icon { + fill: $color-icon; + } + + &:hover { + text-decoration: underline; + } } - } - .sv-content { - overflow: hidden; - overflow-wrap: break-word; + .sv-audio-view { + .ff-icon { + fill: black; + } + } + + .sv-content { + overflow: hidden; + overflow-wrap: break-word; + } } } From de906d63ed0a791de15be0060eb07ffa10870517 Mon Sep 17 00:00:00 2001 From: Jamie Cope Date: Thu, 11 Jan 2024 13:39:27 -0500 Subject: [PATCH 02/12] Cleaning up extraneous Quill and Three references. --- source/client/explorer.hbs | 22 ---------------------- source/client/story.hbs | 21 --------------------- source/client/webpack.config.js | 1 - 3 files changed, 44 deletions(-) diff --git a/source/client/explorer.hbs b/source/client/explorer.hbs index 57c207ba..a155bd3e 100644 --- a/source/client/explorer.hbs +++ b/source/client/explorer.hbs @@ -8,15 +8,6 @@ {{#unless htmlWebpackPlugin.options.isDevelopment}} - - {{/if}} - {{else}} - {{#if htmlWebpackPlugin.options.isOffline}} - - - {{/if}} - {{/if}} - {{{htmlWebpackPlugin.options.element}}} diff --git a/source/client/story.hbs b/source/client/story.hbs index 377c6a72..0ec2349c 100644 --- a/source/client/story.hbs +++ b/source/client/story.hbs @@ -12,27 +12,6 @@ - - {{#if htmlWebpackPlugin.options.isDevelopment}} - {{#if htmlWebpackPlugin.options.isOffline}} - - - - {{else}} - - - {{/if}} - {{else}} - {{#if htmlWebpackPlugin.options.isOffline}} - - - - {{else}} - - - {{/if}} - {{/if}} - {{{htmlWebpackPlugin.options.element}}} diff --git a/source/client/webpack.config.js b/source/client/webpack.config.js index b042e66d..bdf1a57b 100644 --- a/source/client/webpack.config.js +++ b/source/client/webpack.config.js @@ -264,7 +264,6 @@ function createAppConfig(app, isDevMode, isOffline) // assume a corresponding global variable exists and use that instead. externals: { //"three": "THREE", - "quill": "Quill", //"../../../build/three.module.js": "THREE", // patch to handle three jsm modules until there is a better routing option }, From 88693e70a5786c31cab79fb857e1866e4296cdfc Mon Sep 17 00:00:00 2001 From: Jamie Cope Date: Fri, 12 Jan 2024 15:22:41 -0500 Subject: [PATCH 03/12] Circle annotation scaling/offset bug fixed --- source/client/annotations/CircleSprite.ts | 38 ++++++++++++++++++++--- 1 file changed, 33 insertions(+), 5 deletions(-) diff --git a/source/client/annotations/CircleSprite.ts b/source/client/annotations/CircleSprite.ts index 32d74d28..44110aee 100755 --- a/source/client/annotations/CircleSprite.ts +++ b/source/client/annotations/CircleSprite.ts @@ -15,7 +15,7 @@ * limitations under the License. */ -import { Group, Mesh, MeshBasicMaterial, BufferGeometry } from "three"; +import { Group, Mesh, MeshBasicMaterial, BufferGeometry, Vector3 } from "three"; import { customElement, html, render } from "@ff/ui/CustomElement"; import math from "@ff/core/math"; @@ -31,6 +31,11 @@ import {unsafeHTML} from 'lit-html/directives/unsafe-html.js'; //////////////////////////////////////////////////////////////////////////////// const _color = new FFColor(); +const _offset = new Vector3(0, 0, 0); +const _vec3a = new Vector3(); +const _vec3b = new Vector3(); +const _vec3c = new Vector3(); +const _vec3d = new Vector3(); export default class CircleSprite extends AnnotationSprite { @@ -53,6 +58,7 @@ export default class CircleSprite extends AnnotationSprite new MeshBasicMaterial() ); this.anchorMesh.frustumCulled = false; + this.anchorMesh.matrixAutoUpdate = false; this.offset.add(this.anchorMesh); @@ -67,10 +73,35 @@ export default class CircleSprite extends AnnotationSprite super.dispose(); } + update() + { + const annotation = this.annotation.data; + + this.anchorMesh.scale.setScalar(annotation.scale); + this.anchorMesh.position.y = annotation.offset; + this.anchorMesh.updateMatrix(); + + super.update(); + } + renderHTMLElement(element: AnnotationElement, container: HTMLElement, camera: UniversalCamera) { - super.renderHTMLElement(element, container, camera, this.anchorMesh); + super.renderHTMLElement(element, container, camera, this.anchorMesh, _offset); + // Override viewAngle calculation using temporary offset + const anchor = this.anchorMesh; + _vec3a.set(0, 0, 0); + _vec3a.applyMatrix4(anchor.modelViewMatrix); + + _vec3b.set(0, 1, 0); + _vec3b.applyMatrix4(anchor.modelViewMatrix); + + _vec3c.copy(_vec3b).sub(_vec3a).normalize(); + _vec3d.set(0, 0, 1); + + this.viewAngle = _vec3c.angleTo(_vec3d); + + // Set opacity based on viewAngle const angleOpacity = math.scaleLimit(this.viewAngle * math.RAD2DEG, 90, 100, 1, 0); const opacity = this.annotation.data.visible ? angleOpacity : 0; @@ -80,9 +111,6 @@ export default class CircleSprite extends AnnotationSprite const isShowing = annotation.visible; this.offset.visible = isShowing; - this.offset.position.set(0, (annotation.offset + 1) * 0.5, 0); - - this.offset.updateMatrix(); // don't show if behind the camera this.setVisible(!this.isBehindCamera(this.offset, camera) && isShowing); From 054a5250b94631f29d46a00d05e7b257c753b996 Mon Sep 17 00:00:00 2001 From: Jamie Cope Date: Mon, 22 Jan 2024 12:24:04 -0500 Subject: [PATCH 04/12] Auto select new audio element on create --- source/client/components/CVAudioTask.ts | 4 +++- source/client/ui/story/AudioTaskView.ts | 5 +++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/source/client/components/CVAudioTask.ts b/source/client/components/CVAudioTask.ts index 62163bb8..baab2b92 100644 --- a/source/client/components/CVAudioTask.ts +++ b/source/client/components/CVAudioTask.ts @@ -112,13 +112,15 @@ export default class CVAudioTask extends CVTask } if (ins.create.changed) { + const newId = Document.generateId(); audioManager.addAudioClip({ - id: Document.generateId(), + id: newId, name: "New Audio Element", uris: {}, captionUris: {}, durations: {} }); + ins.activeId.setValue(newId); return true; } if (ins.delete.changed) { diff --git a/source/client/ui/story/AudioTaskView.ts b/source/client/ui/story/AudioTaskView.ts index d1bf9cd5..b6a4e212 100644 --- a/source/client/ui/story/AudioTaskView.ts +++ b/source/client/ui/story/AudioTaskView.ts @@ -64,8 +64,9 @@ export default class AudioTaskView extends TaskView const ins = this.task.ins; const narrationFlagClass = "sv-task-option-base-align"; - const audioList = this.task.audioManager.getAudioList(); - const audioElement = audioList[this.selectedIndex]; + const audio = this.task.audioManager; + const audioList = audio.getAudioList(); + const audioElement = audio.getAudioClip(ins.activeId.value); const narrationEnabled = !ins.isNarration.value && audioList.some(clip => clip.id === this.task.audioManager.narrationId); const detailView = audioElement ? html`
From d1d20a0450ae54167683b17df219ee3bdfc138e7 Mon Sep 17 00:00:00 2001 From: Jamie Cope Date: Mon, 22 Jan 2024 12:56:16 -0500 Subject: [PATCH 05/12] ff-scene update to fix directional lighting intensity bug --- libs/ff-scene | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libs/ff-scene b/libs/ff-scene index 17ef1e37..c4759a91 160000 --- a/libs/ff-scene +++ b/libs/ff-scene @@ -1 +1 @@ -Subproject commit 17ef1e37fe9b87af13ea6f60abe5d2fbe8086f4e +Subproject commit c4759a91a1090fa0a92a3a4a2c8b762bf02a65fb From 2a57d75c1df2dee211460c14ac66a1c0df26ffcd Mon Sep 17 00:00:00 2001 From: Jamie Cope Date: Mon, 22 Jan 2024 15:47:18 -0500 Subject: [PATCH 06/12] CaptionView bug fix --- source/client/ui/explorer/CaptionView.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/source/client/ui/explorer/CaptionView.ts b/source/client/ui/explorer/CaptionView.ts index 46fc91e6..6e9404e5 100644 --- a/source/client/ui/explorer/CaptionView.ts +++ b/source/client/ui/explorer/CaptionView.ts @@ -48,7 +48,6 @@ export default class CaptionView extends DocumentView { if (previous) { this.documentProps.off(); - this.audioManager = null; } if (next) { const setup = next.setup; From 3c7b2150a0b2f5ce6183334fac83af6c84610908 Mon Sep 17 00:00:00 2001 From: Jamie Cope Date: Tue, 23 Jan 2024 11:53:27 -0500 Subject: [PATCH 07/12] Temporary fix for application disposal --- source/client/applications/ExplorerApplication.ts | 6 +----- source/client/ui/explorer/MainView.ts | 7 +++++-- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/source/client/applications/ExplorerApplication.ts b/source/client/applications/ExplorerApplication.ts index 814f83b8..4a4e297c 100755 --- a/source/client/applications/ExplorerApplication.ts +++ b/source/client/applications/ExplorerApplication.ts @@ -216,14 +216,10 @@ Version: ${ENV_VERSION} dispose() { // Clean up assuming a component disconnect means it won't be reconnected + // TODO: More complete clean up that doesn't interfere with component disconnect this.assetReader.dispose(); this.documentProvider.activeComponent.clearNodeTree(); this.system.getMainComponent(CRenderer).views.forEach(view => view.dispose()); - //this.documentProvider.activeComponent.setup.node.dispose(); - //this.system.graph.clear(); - this.documentProvider.activeComponent.setup.tape.dispose(); - this.documentProvider.activeComponent.setup.floor.dispose(); - this.documentProvider.activeComponent.setup.grid.dispose(); } setBaseUrl(url: string) diff --git a/source/client/ui/explorer/MainView.ts b/source/client/ui/explorer/MainView.ts index 80d4b800..9c88594a 100755 --- a/source/client/ui/explorer/MainView.ts +++ b/source/client/ui/explorer/MainView.ts @@ -162,10 +162,13 @@ export default class MainView extends CustomElement protected disconnected() { + super.disconnected(); this.fullscreen.fullscreenElement = null; this.viewer.rootElement = null; - this.application.dispose(); - this.application = null; + if(!window["VoyagerStory"]) { + this.application.dispose(); + this.application = null; + } } attributeChangedCallback(name: string, old: string | null, value: string | null) From 07af1459a604ec0de3b245f13af3e1df135d5acf Mon Sep 17 00:00:00 2001 From: Jamie Cope Date: Tue, 23 Jan 2024 12:35:57 -0500 Subject: [PATCH 08/12] Three.js update to v0.158 + shader corrections for this version --- package-lock.json | 6 +++--- package.json | 2 +- source/client/shaders/uberPBRShader.frag | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/package-lock.json b/package-lock.json index 7e7576ce..e7772bb4 100644 --- a/package-lock.json +++ b/package-lock.json @@ -5669,9 +5669,9 @@ } }, "three": { - "version": "0.157.0", - "resolved": "https://registry.npmjs.org/three/-/three-0.157.0.tgz", - "integrity": "sha512-CeAwQrf4x3z0/e+MC4F+nXLW5t0gh3pw+L6CCBqpHvOq3bGYIgRYub7Pv0j/9wR+d++OiEglyZzWyuSYbwWGOA==" + "version": "0.158.0", + "resolved": "https://registry.npmjs.org/three/-/three-0.158.0.tgz", + "integrity": "sha512-TALj4EOpdDPF1henk2Q+s17K61uEAAWQ7TJB68nr7FKxqwyDr3msOt5IWdbGm4TaWKjrtWS8DJJWe9JnvsWOhQ==" }, "three-bmfont-text": { "version": "git+ssh://git@github.com/Smithsonian/three-bmfont-text.git#50da80d887873140dd7f20b28675e68de0b2b1da", diff --git a/package.json b/package.json index 9f7095ba..27b35ec1 100755 --- a/package.json +++ b/package.json @@ -70,7 +70,7 @@ "simple-dropzone": "^0.8.3", "stream-browserify": "^3.0.0", "style-loader": "^3.3.1", - "three": "^0.157.0", + "three": "^0.158.0", "three-bmfont-text": "git+https://github.com/Smithsonian/three-bmfont-text.git#50da80d887", "tinymce": "^6.3.1", "toposort": "^2.0.2", diff --git a/source/client/shaders/uberPBRShader.frag b/source/client/shaders/uberPBRShader.frag index 858c931b..db976702 100644 --- a/source/client/shaders/uberPBRShader.frag +++ b/source/client/shaders/uberPBRShader.frag @@ -216,7 +216,7 @@ void main() { // https://drive.google.com/file/d/1T0D1VSyR4AllqIJTQAraEIzjlb5h4FKH/view?usp=sharing float sheenEnergyComp = 1.0 - 0.157 * max3( material.sheenColor ); - outgoingLight = outgoingLight * sheenEnergyComp + sheenSpecular; + outgoingLight = outgoingLight * sheenEnergyComp + sheenSpecularDirect + sheenSpecularIndirect; #endif From 4bd0ab1410b7dc73f079d37746a7ec38efacc876 Mon Sep 17 00:00:00 2001 From: Jamie Cope Date: Wed, 24 Jan 2024 14:30:42 -0500 Subject: [PATCH 09/12] Cleaning up unnecessary references --- source/client/components/CVARManager.ts | 2 -- source/client/components/CVAnnotationView.ts | 4 +--- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/source/client/components/CVARManager.ts b/source/client/components/CVARManager.ts index 00ceaaae..64f9a1a7 100644 --- a/source/client/components/CVARManager.ts +++ b/source/client/components/CVARManager.ts @@ -40,7 +40,6 @@ import { EUnitType } from "client/schema/common"; import { EDerivativeUsage, EDerivativeQuality, EAssetType } from "client/models/Derivative"; import CVModel2 from "./CVModel2"; import CVAssetManager from "./CVAssetManager"; -import CVAnnotationView from "./CVAnnotationView"; import { Shadow } from "../xr/XRShadow" import CVDirectionalLight from "./CVDirectionalLight"; @@ -57,7 +56,6 @@ const _hitPosition = new Vector3(); const _boundingBox = new Box3(); const _quat = new Quaternion(); const ROTATION_RATE = 1.5; -const ANNOTATION_SCALE = 2.0; export default class CVARManager extends Component { diff --git a/source/client/components/CVAnnotationView.ts b/source/client/components/CVAnnotationView.ts index b291ec95..83cceec1 100755 --- a/source/client/components/CVAnnotationView.ts +++ b/source/client/components/CVAnnotationView.ts @@ -39,7 +39,6 @@ import AnnotationFactory from "../annotations/AnnotationFactory"; import "../annotations/StandardSprite"; import "../annotations/ExtendedSprite"; import "../annotations/CircleSprite"; -import CircleSprite from "../annotations/CircleSprite"; import CVARManager from "./CVARManager"; import CVLanguageManager from "./CVLanguageManager"; import { ELanguageType, EUnitType } from "client/schema/common"; @@ -482,10 +481,9 @@ export default class CVAnnotationView extends CObject3D protected createSprite(annotation: Annotation) { this.removeSprite(annotation); - const isCircle = annotation.data.style === "Circle"; // TODO: Combine when font loading is centralized - const sprite = isCircle ? AnnotationFactory.createInstance(annotation, "Circle", this.assetReader) : AnnotationFactory.createInstance(annotation); + const sprite = AnnotationFactory.createInstance(annotation); sprite.addEventListener("click", this.onSpriteClick); sprite.addEventListener("link", this.onSpriteLink); From 48bc57607a2ece05727bdd2d607208f69bff373b Mon Sep 17 00:00:00 2001 From: Jamie Cope Date: Wed, 24 Jan 2024 15:04:29 -0500 Subject: [PATCH 10/12] Enable audio for Standalone mode --- source/client/applications/taskSets.ts | 1 + source/client/components/CVMediaManager.ts | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/source/client/applications/taskSets.ts b/source/client/applications/taskSets.ts index 62ced1ce..cf824ff3 100644 --- a/source/client/applications/taskSets.ts +++ b/source/client/applications/taskSets.ts @@ -69,6 +69,7 @@ export default { CVAnnotationsTask, CVArticlesTask, CVToursTask, + CVAudioTask, CVSettingsTask, ] } \ No newline at end of file diff --git a/source/client/components/CVMediaManager.ts b/source/client/components/CVMediaManager.ts index 9b9ea538..e782125a 100644 --- a/source/client/components/CVMediaManager.ts +++ b/source/client/components/CVMediaManager.ts @@ -134,7 +134,7 @@ export default class CVMediaManager extends CAssetManager const cleanfileName = decodeURI(file.name); const filenameLower = cleanfileName.toLowerCase(); - if (filenameLower.match(/\.(gltf|glb|bin|svx.json|html|jpg|jpeg|png|usdz)$/)) { + if (filenameLower.match(/\.(gltf|glb|bin|svx.json|html|jpg|jpeg|png|usdz|mp3|vtt)$/)) { if(!documentProvided && filenameLower.match(/\.(jpg|jpeg|png)$/) && !fileArray.some(entry => entry[0].endsWith("gltf"))) { path = CVMediaManager.articleFolder + "/" + cleanfileName; From e1c17c528f55d9a5350b033c541b604c743db5e2 Mon Sep 17 00:00:00 2001 From: Jamie Cope Date: Wed, 24 Jan 2024 15:06:15 -0500 Subject: [PATCH 11/12] Version increment --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 39ee509a..bbf730c1 100755 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "voyager", - "version": "0.35.0", + "version": "0.37.0", "description": "Smithsonian DPO Voyager - 3D Explorer and Tool Suite", "scripts": { "start": "npm run server", From 1d6d0b7c84f6ae12b2057e28c79645b1e0b67dbb Mon Sep 17 00:00:00 2001 From: Jamie Cope Date: Thu, 25 Jan 2024 12:56:22 -0500 Subject: [PATCH 12/12] Schema updates --- source/client/schema/json/meta.schema.json | 8 ++++ source/client/schema/json/setup.schema.json | 43 +++++++++++++++++++++ source/client/schema/setup.ts | 1 + 3 files changed, 52 insertions(+) diff --git a/source/client/schema/json/meta.schema.json b/source/client/schema/json/meta.schema.json index a199ae54..341f4e46 100644 --- a/source/client/schema/json/meta.schema.json +++ b/source/client/schema/json/meta.schema.json @@ -113,6 +113,14 @@ "uris": { "description": "Location of the audio resource, absolute URL or path relative to this document with language key", "type": "object" + }, + "captionUris": { + "description": "Location of the caption resource, absolute URL or path relative to this document with language key", + "type": "object" + }, + "durations": { + "description": "Length of the audio resource with language key", + "type": "object" } }, "required": [ diff --git a/source/client/schema/json/setup.schema.json b/source/client/schema/json/setup.schema.json index b8f76c29..da6a3147 100644 --- a/source/client/schema/json/setup.schema.json +++ b/source/client/schema/json/setup.schema.json @@ -15,12 +15,21 @@ "exposure": { "type": "number" }, + "toneMapping": { + "type": "boolean" + }, "gamma": { "type": "number" }, "annotationsVisible": { "type": "boolean" }, + "isWallMountAR": { + "type": "boolean" + }, + "arScale": { + "type": "number" + }, "activeTags": { "type": "string" }, @@ -85,6 +94,12 @@ "autoZoom": { "type": "boolean" }, + "autoRotation": { + "type": "boolean" + }, + "lightsFollowCamera": { + "type": "boolean" + }, "orbit": { "$comment": "TODO: Implement", @@ -188,6 +203,28 @@ }, "position": { "type": "number" + }, + "color": { + "$ref": "./common.schema.json#/definitions/vector3" + } + } + }, + "audio": { + "type": "object", + "properties": { + "narrationId": { + "type": "string" + } + } + }, + "language": { + "type": "object", + "properties": { + "language": { + "type": "string", + "enum": [ + "EN", "ES", "DE", "NL", "JA", "FR", "HAW" + ] } } }, @@ -303,6 +340,12 @@ }, "tours": { "$ref": "#/definitions/tours" + }, + "language": { + "$ref": "#/definitions/language" + }, + "audio": { + "$ref": "#/definitions/audio" } } } diff --git a/source/client/schema/setup.ts b/source/client/schema/setup.ts index c18139c9..e2006ed6 100644 --- a/source/client/schema/setup.ts +++ b/source/client/schema/setup.ts @@ -49,6 +49,7 @@ export interface ISetup slicer?: ISlicer; tours?: ITours; snapshots?: ISnapshots; + audio?: IAudio; } export interface IInterface