diff --git a/packages/client-core/src/common/services/FileThumbnailJobState.tsx b/packages/client-core/src/common/services/FileThumbnailJobState.tsx index 96f65cb962..d62ae0bc89 100644 --- a/packages/client-core/src/common/services/FileThumbnailJobState.tsx +++ b/packages/client-core/src/common/services/FileThumbnailJobState.tsx @@ -42,7 +42,6 @@ import { import { useTexture } from '@etherealengine/engine/src/assets/functions/resourceLoaderHooks' import { GLTFDocumentState } from '@etherealengine/engine/src/gltf/GLTFDocumentState' import { ModelComponent } from '@etherealengine/engine/src/scene/components/ModelComponent' -import { getModelSceneID } from '@etherealengine/engine/src/scene/functions/loaders/ModelFunctions' import { NO_PROXY, defineState, getMutableState, useHookstate } from '@etherealengine/hyperflux' import { DirectionalLightComponent, TransformComponent } from '@etherealengine/spatial' import { CameraComponent } from '@etherealengine/spatial/src/camera/components/CameraComponent' @@ -66,6 +65,7 @@ import React, { useEffect } from 'react' import { Color, Euler, Material, MathUtils, Matrix4, Mesh, Quaternion, Sphere, SphereGeometry, Vector3 } from 'three' import config from '@etherealengine/common/src/config' +import { GLTFComponent } from '@etherealengine/engine/src/gltf/GLTFComponent' import { ErrorComponent } from '@etherealengine/engine/src/scene/components/ErrorComponent' import { ShadowComponent } from '@etherealengine/engine/src/scene/components/ShadowComponent' import { useFind } from '@etherealengine/spatial/src/common/functions/FeathersHooks' @@ -414,7 +414,7 @@ const ThumbnailJobReactor = () => { const modelEntity = state.modelEntity.value const lightEntity = state.lightEntity.value - const sceneID = getModelSceneID(modelEntity) + const sceneID = GLTFComponent.getInstanceID(modelEntity) if (!sceneState.value[sceneID]) return try { diff --git a/packages/client-core/src/media/webcam/WebcamInput.ts b/packages/client-core/src/media/webcam/WebcamInput.ts index 2bf02b3719..27563de27e 100755 --- a/packages/client-core/src/media/webcam/WebcamInput.ts +++ b/packages/client-core/src/media/webcam/WebcamInput.ts @@ -8,12 +8,12 @@ import { Entity } from '@etherealengine/ecs/src/Entity' import { defineQuery } from '@etherealengine/ecs/src/QueryFunctions' import { AvatarRigComponent } from '@etherealengine/engine/src/avatar/components/AvatarAnimationComponent' import { AvatarComponent } from '@etherealengine/engine/src/avatar/components/AvatarComponent' -import { SkinnedMeshComponent } from '@etherealengine/engine/src/avatar/components/SkinnedMeshComponent' import { AvatarNetworkAction } from '@etherealengine/engine/src/avatar/state/AvatarNetworkActions' import { defineActionQueue, getMutableState } from '@etherealengine/hyperflux' import { GroupComponent } from '@etherealengine/spatial/src/renderer/components/GroupComponent' import { iterateEntityNode } from '@etherealengine/spatial/src/transform/components/EntityTree' +import { SkinnedMeshComponent } from '@etherealengine/spatial/src/renderer/components/SkinnedMeshComponent' import { MediaStreamState } from '../../transports/MediaStreams' import { WebcamInputComponent } from './WebcamInputComponent' diff --git a/packages/client-core/src/world/Location.tsx b/packages/client-core/src/world/Location.tsx index 08104a0e86..f21987fe1b 100755 --- a/packages/client-core/src/world/Location.tsx +++ b/packages/client-core/src/world/Location.tsx @@ -71,7 +71,8 @@ const LocationPage = ({ online }: Props) => { useLoadEngineWithScene() useEffect(() => { - if (ready.value) logger.info({ event_name: 'enter_location' }) + if (!ready.value) return + logger.info({ event_name: 'enter_location' }) return () => logger.info({ event_name: 'exit_location' }) }, [ready.value]) diff --git a/packages/editor/src/components/hierarchy/HierarchyTreeWalker.ts b/packages/editor/src/components/hierarchy/HierarchyTreeWalker.ts index 29b0d85a9d..21c45d6bc8 100644 --- a/packages/editor/src/components/hierarchy/HierarchyTreeWalker.ts +++ b/packages/editor/src/components/hierarchy/HierarchyTreeWalker.ts @@ -31,9 +31,9 @@ import { getState } from '@etherealengine/hyperflux' import { EntityTreeComponent } from '@etherealengine/spatial/src/transform/components/EntityTree' import { UUIDComponent } from '@etherealengine/ecs' +import { GLTFComponent } from '@etherealengine/engine/src/gltf/GLTFComponent' import { GLTFSnapshotState } from '@etherealengine/engine/src/gltf/GLTFState' import { ModelComponent } from '@etherealengine/engine/src/scene/components/ModelComponent' -import { getModelSceneID } from '@etherealengine/engine/src/scene/functions/loaders/ModelFunctions' import { GLTF } from '@gltf-transform/core' import { EditorState } from '../../services/EditorServices' @@ -84,7 +84,7 @@ function buildHierarchyTree( array.push(item) if (hasComponent(entity, ModelComponent) && showModelChildren) { - const modelSceneID = getModelSceneID(entity) + const modelSceneID = GLTFComponent.getInstanceID(entity) const snapshotState = getState(GLTFSnapshotState) const snapshots = snapshotState[modelSceneID] if (snapshots) { diff --git a/packages/editor/src/functions/sceneFunctions.tsx b/packages/editor/src/functions/sceneFunctions.tsx index 6ee490ff1c..e007dc71c2 100644 --- a/packages/editor/src/functions/sceneFunctions.tsx +++ b/packages/editor/src/functions/sceneFunctions.tsx @@ -29,8 +29,8 @@ import config from '@etherealengine/common/src/config' import multiLogger from '@etherealengine/common/src/logger' import { StaticResourceType, fileBrowserPath, staticResourcePath } from '@etherealengine/common/src/schema.type.module' import { cleanString } from '@etherealengine/common/src/utils/cleanString' -import { EntityUUID, UUIDComponent, UndefinedEntity } from '@etherealengine/ecs' -import { getComponent, setComponent } from '@etherealengine/ecs/src/ComponentFunctions' +import { EntityUUID, UndefinedEntity } from '@etherealengine/ecs' +import { setComponent } from '@etherealengine/ecs/src/ComponentFunctions' import { Engine } from '@etherealengine/ecs/src/Engine' import { GLTFComponent } from '@etherealengine/engine/src/gltf/GLTFComponent' import { GLTFDocumentState } from '@etherealengine/engine/src/gltf/GLTFDocumentState' @@ -93,7 +93,7 @@ export const saveSceneGLTF = async ( if (signal.aborted) throw new Error(i18n.t('editor:errors.saveProjectAborted')) const { rootEntity } = getState(EditorState) - const sourceID = `${getComponent(rootEntity, UUIDComponent)}-${getComponent(rootEntity, GLTFComponent).src}` + const sourceID = GLTFComponent.getInstanceID(rootEntity) const sceneName = cleanString(sceneFile!.replace('.scene.json', '').replace('.gltf', '')) const currentSceneDirectory = getState(EditorState).scenePath!.split('/').slice(0, -1).join('/') diff --git a/packages/editor/src/systems/ClickPlacementSystem.tsx b/packages/editor/src/systems/ClickPlacementSystem.tsx index 475a25b9a5..e16c1c9da4 100644 --- a/packages/editor/src/systems/ClickPlacementSystem.tsx +++ b/packages/editor/src/systems/ClickPlacementSystem.tsx @@ -47,7 +47,6 @@ import { ModelComponent } from '@etherealengine/engine/src/scene/components/Mode import { SourceComponent } from '@etherealengine/engine/src/scene/components/SourceComponent' import { entityJSONToGLTFNode } from '@etherealengine/engine/src/scene/functions/GLTFConversion' import { createSceneEntity } from '@etherealengine/engine/src/scene/functions/createSceneEntity' -import { getModelSceneID } from '@etherealengine/engine/src/scene/functions/loaders/ModelFunctions' import { toEntityJson } from '@etherealengine/engine/src/scene/functions/serializeWorld' import { NO_PROXY, @@ -173,7 +172,7 @@ const PlacementModelReactor = (props: { placementEntity: Entity }) => { useEffect(() => { if (!placementModel) return - const sceneID = getModelSceneID(props.placementEntity) + const sceneID = GLTFComponent.getInstanceID(props.placementEntity) if (!sceneState.scenes[sceneID]) return iterateEntityNode(props.placementEntity, (entity) => { const mesh = getOptionalComponent(entity, MeshComponent) diff --git a/packages/engine/src/assets/exporters/gltf/extensions/SourceHandlerExtension.ts b/packages/engine/src/assets/exporters/gltf/extensions/SourceHandlerExtension.ts index f8ab12b5d0..cfc0e70493 100644 --- a/packages/engine/src/assets/exporters/gltf/extensions/SourceHandlerExtension.ts +++ b/packages/engine/src/assets/exporters/gltf/extensions/SourceHandlerExtension.ts @@ -29,8 +29,8 @@ import { getComponent, setComponent } from '@etherealengine/ecs/src/ComponentFun import { Entity, UndefinedEntity } from '@etherealengine/ecs/src/Entity' import { EntityTreeComponent, iterateEntityNode } from '@etherealengine/spatial/src/transform/components/EntityTree' +import { GLTFComponent } from '../../../../gltf/GLTFComponent' import { SourceComponent } from '../../../../scene/components/SourceComponent' -import { getModelSceneID } from '../../../../scene/functions/loaders/ModelFunctions' import { GLTFExporterPlugin, GLTFWriter } from '../GLTFExporter' import { ExporterExtension } from './ExporterExtension' @@ -46,7 +46,7 @@ export default class SourceHandlerExtension extends ExporterExtension implements //we allow saving of any object that has a source equal to or parent of the root's source const validSrcs: Set = new Set() if (!this.writer.options.srcEntity) return - validSrcs.add(getModelSceneID(this.writer.options.srcEntity!)) + validSrcs.add(GLTFComponent.getInstanceID(this.writer.options.srcEntity!)) const root = (Array.isArray(input) ? input[0] : input) as Object3D let walker: Entity | null = root.entity while (walker !== null) { diff --git a/packages/engine/src/assets/loaders/gltf/GLTFExtensions.ts b/packages/engine/src/assets/loaders/gltf/GLTFExtensions.ts index 1db5d27d5d..48bffab99e 100644 --- a/packages/engine/src/assets/loaders/gltf/GLTFExtensions.ts +++ b/packages/engine/src/assets/loaders/gltf/GLTFExtensions.ts @@ -53,28 +53,28 @@ import { assignExtrasToUserData } from './GLTFLoaderFunctions' import { GLTFParser } from './GLTFParser' export const EXTENSIONS = { - KHR_BINARY_GLTF: 'KHR_binary_glTF', - KHR_DRACO_MESH_COMPRESSION: 'KHR_draco_mesh_compression', - KHR_LIGHTS_PUNCTUAL: 'KHR_lights_punctual', - KHR_MATERIALS_CLEARCOAT: 'KHR_materials_clearcoat', - KHR_MATERIALS_IOR: 'KHR_materials_ior', - KHR_MATERIALS_SHEEN: 'KHR_materials_sheen', - KHR_MATERIALS_SPECULAR: 'KHR_materials_specular', - KHR_MATERIALS_TRANSMISSION: 'KHR_materials_transmission', - KHR_MATERIALS_IRIDESCENCE: 'KHR_materials_iridescence', - KHR_MATERIALS_ANISOTROPY: 'KHR_materials_anisotropy', - KHR_MATERIALS_UNLIT: 'KHR_materials_unlit', - KHR_MATERIALS_VOLUME: 'KHR_materials_volume', - KHR_TEXTURE_BASISU: 'KHR_texture_basisu', - KHR_TEXTURE_TRANSFORM: 'KHR_texture_transform', - KHR_MESH_QUANTIZATION: 'KHR_mesh_quantization', - KHR_MATERIALS_EMISSIVE_STRENGTH: 'KHR_materials_emissive_strength', - EXT_MATERIALS_BUMP: 'EXT_materials_bump', - EXT_TEXTURE_WEBP: 'EXT_texture_webp', - EXT_TEXTURE_AVIF: 'EXT_texture_avif', - EXT_MESHOPT_COMPRESSION: 'EXT_meshopt_compression', - EXT_MESH_GPU_INSTANCING: 'EXT_mesh_gpu_instancing', - EE_MATERIAL: 'EE_material' + KHR_BINARY_GLTF: 'KHR_binary_glTF' as const, + KHR_DRACO_MESH_COMPRESSION: 'KHR_draco_mesh_compression' as const, + KHR_LIGHTS_PUNCTUAL: 'KHR_lights_punctual' as const, + KHR_MATERIALS_CLEARCOAT: 'KHR_materials_clearcoat' as const, + KHR_MATERIALS_IOR: 'KHR_materials_ior' as const, + KHR_MATERIALS_SHEEN: 'KHR_materials_sheen' as const, + KHR_MATERIALS_SPECULAR: 'KHR_materials_specular' as const, + KHR_MATERIALS_TRANSMISSION: 'KHR_materials_transmission' as const, + KHR_MATERIALS_IRIDESCENCE: 'KHR_materials_iridescence' as const, + KHR_MATERIALS_ANISOTROPY: 'KHR_materials_anisotropy' as const, + KHR_MATERIALS_UNLIT: 'KHR_materials_unlit' as const, + KHR_MATERIALS_VOLUME: 'KHR_materials_volume' as const, + KHR_TEXTURE_BASISU: 'KHR_texture_basisu' as const, + KHR_TEXTURE_TRANSFORM: 'KHR_texture_transform' as const, + KHR_MESH_QUANTIZATION: 'KHR_mesh_quantization' as const, + KHR_MATERIALS_EMISSIVE_STRENGTH: 'KHR_materials_emissive_strength' as const, + EXT_MATERIALS_BUMP: 'EXT_materials_bump' as const, + EXT_TEXTURE_WEBP: 'EXT_texture_webp' as const, + EXT_TEXTURE_AVIF: 'EXT_texture_avif' as const, + EXT_MESHOPT_COMPRESSION: 'EXT_meshopt_compression' as const, + EXT_MESH_GPU_INSTANCING: 'EXT_mesh_gpu_instancing' as const, + EE_MATERIAL: 'EE_material' as const } /** diff --git a/packages/engine/src/assets/loaders/gltf/GLTFLoader.ts b/packages/engine/src/assets/loaders/gltf/GLTFLoader.ts index 41d9da87c6..97c9da1496 100755 --- a/packages/engine/src/assets/loaders/gltf/GLTFLoader.ts +++ b/packages/engine/src/assets/loaders/gltf/GLTFLoader.ts @@ -69,11 +69,12 @@ import { } from './GLTFExtensions' import { GLTFParser } from './GLTFParser' import { KTX2Loader } from './KTX2Loader' +import { MeshoptDecoder } from './meshopt_decoder' export class GLTFLoader extends Loader { dracoLoader = null as null | DRACOLoader ktx2Loader = null as null | KTX2Loader - meshoptDecoder = null + meshoptDecoder = null as null | MeshoptDecoder pluginCallbacks = [] as any[] diff --git a/packages/engine/src/assets/loaders/gltf/GLTFParser.ts b/packages/engine/src/assets/loaders/gltf/GLTFParser.ts index e0f480ffa0..04452c538a 100644 --- a/packages/engine/src/assets/loaders/gltf/GLTFParser.ts +++ b/packages/engine/src/assets/loaders/gltf/GLTFParser.ts @@ -98,7 +98,7 @@ import { } from './GLTFLoaderFunctions' import { KTX2Loader } from './KTX2Loader' -function getImageURIMimeType(uri) { +export function getImageURIMimeType(uri) { if (uri.search(/\.jpe?g($|\?)/i) > 0 || uri.search(/^data\:image\/jpeg/) === 0) return 'image/jpeg' if (uri.search(/\.webp($|\?)/i) > 0 || uri.search(/^data\:image\/webp/) === 0) return 'image/webp' @@ -123,10 +123,13 @@ declare module '@gltf-transform/core/dist/types/gltf.d.ts' { } } -type GLTFParserOptions = { +export type GLTFParserOptions = { + body: null | ArrayBuffer + documentID: string + document: GLTF.IGLTF crossOrigin: 'anonymous' | string ktx2Loader: KTX2Loader - manager: LoadingManager | any + manager: LoadingManager meshoptDecoder: any path: string requestHeader: Record @@ -688,7 +691,7 @@ export class GLTFParser { if (sourceDef.uri) { const handler = options.manager.getHandler(sourceDef.uri) - if (handler !== null) loader = handler + if (handler !== null) loader = handler as any } return this.loadTextureImage(textureIndex, sourceIndex, loader) @@ -1767,7 +1770,7 @@ export class GLTFParser { /* GLTFREGISTRY */ -class GLTFRegistry { +export class GLTFRegistry { objects = {} get(key) { diff --git a/packages/engine/src/assets/loaders/gltf/meshopt_decoder.d.ts b/packages/engine/src/assets/loaders/gltf/meshopt_decoder.d.ts new file mode 100644 index 0000000000..7c07b9b33c --- /dev/null +++ b/packages/engine/src/assets/loaders/gltf/meshopt_decoder.d.ts @@ -0,0 +1,36 @@ +/* +CPAL-1.0 License + +The contents of this file are subject to the Common Public Attribution License +Version 1.0. (the "License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at +https://github.com/EtherealEngine/etherealengine/blob/dev/LICENSE. +The License is based on the Mozilla Public License Version 1.1, but Sections 14 +and 15 have been added to cover use of software over a computer network and +provide for limited attribution for the Original Developer. In addition, +Exhibit A has been modified to be consistent with Exhibit B. + +Software distributed under the License is distributed on an "AS IS" basis, +WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License for the +specific language governing rights and limitations under the License. + +The Original Code is Ethereal Engine. + +The Original Developer is the Initial Developer. The Initial Developer of the +Original Code is the Ethereal Engine team. + +All portions of the code written by the Ethereal Engine team are Copyright © 2021-2023 +Ethereal Engine. All Rights Reserved. +*/ + + +export type MeshoptDecoder = { + supported: boolean; + ready: Promise; + useWorkers: (numWorkers: number) => void; + decodeVertexBuffer: (buffer: ArrayBuffer, count: number, stride: number, filter: number) => ArrayBuffer; + decodeIndexBuffer: (buffer: ArrayBuffer, count: number, filter: number) => ArrayBuffer; + decodeIndexSequence: (buffer: ArrayBuffer, count: number, filter: number) => ArrayBuffer; + decodeGltfBuffer: (buffer: Uint8Array, count: number, stride: number, source: Uint8Array, mode: number, filter: number) => void; + decodeGltfBufferAsync?: (count: number, stride: number, source: Uint8Array, mode: number, filter: number) => Promise<{ buffer: ArrayBuffer }>; +} \ No newline at end of file diff --git a/packages/engine/src/avatar/components/AvatarAnimationComponent.ts b/packages/engine/src/avatar/components/AvatarAnimationComponent.ts index 39d6d3bde5..1bae2d36d1 100755 --- a/packages/engine/src/avatar/components/AvatarAnimationComponent.ts +++ b/packages/engine/src/avatar/components/AvatarAnimationComponent.ts @@ -23,32 +23,49 @@ All portions of the code written by the Ethereal Engine team are Copyright © 20 Ethereal Engine. All Rights Reserved. */ -import { VRM, VRMHumanBoneName, VRMHumanBones } from '@pixiv/three-vrm' +import { + VRM, + VRM1Meta, + VRMHumanBone, + VRMHumanBoneName, + VRMHumanBones, + VRMHumanoid, + VRMParameters +} from '@pixiv/three-vrm' +import type * as V0VRM from '@pixiv/types-vrm-0.0' + import { useEffect } from 'react' -import { AnimationAction, Group, Matrix4, SkeletonHelper, Vector3 } from 'three' +import { AnimationAction, Euler, Group, Matrix4, Vector3 } from 'three' import { defineComponent, getComponent, + getOptionalComponent, + hasComponent, removeComponent, setComponent, useComponent, useOptionalComponent } from '@etherealengine/ecs/src/ComponentFunctions' -import { Entity } from '@etherealengine/ecs/src/Entity' -import { createEntity, entityExists, removeEntity, useEntityContext } from '@etherealengine/ecs/src/EntityFunctions' -import { getMutableState, matches, none, useHookstate } from '@etherealengine/hyperflux' +import { Entity, EntityUUID } from '@etherealengine/ecs/src/Entity' +import { useEntityContext } from '@etherealengine/ecs/src/EntityFunctions' +import { getMutableState, getState, matches, useHookstate } from '@etherealengine/hyperflux' import { NameComponent } from '@etherealengine/spatial/src/common/NameComponent' import { addObjectToGroup } from '@etherealengine/spatial/src/renderer/components/GroupComponent' -import { setObjectLayers } from '@etherealengine/spatial/src/renderer/components/ObjectLayerComponent' -import { setVisibleComponent, VisibleComponent } from '@etherealengine/spatial/src/renderer/components/VisibleComponent' -import { ObjectLayers } from '@etherealengine/spatial/src/renderer/constants/ObjectLayers' -import { RendererState } from '@etherealengine/spatial/src/renderer/RendererState' import { ComputedTransformComponent } from '@etherealengine/spatial/src/transform/components/ComputedTransformComponent' -import { ModelComponent } from '../../scene/components/ModelComponent' -import { preloadedAnimations } from '../animation/Util' +import { UUIDComponent } from '@etherealengine/ecs' +import { TransformComponent } from '@etherealengine/spatial' +import { BoneComponent } from '@etherealengine/spatial/src/renderer/components/BoneComponent' +import { Object3DComponent } from '@etherealengine/spatial/src/renderer/components/Object3DComponent' +import { EntityTreeComponent, iterateEntityNode } from '@etherealengine/spatial/src/transform/components/EntityTree' +import { GLTF } from '@gltf-transform/core' +import { GLTFComponent } from '../../gltf/GLTFComponent' +import { GLTFDocumentState } from '../../gltf/GLTFDocumentState' +import { proxifyParentChildRelationships } from '../../scene/functions/loadGLTFModel' import { AnimationState } from '../AnimationManager' +import { mixamoVRMRigMap } from '../AvatarBoneMatching' +import { preloadedAnimations } from '../animation/Util' import { retargetAvatarAnimations, setAvatarSpeedFromRootMotion, @@ -57,7 +74,7 @@ import { } from '../functions/avatarFunctions' import { AvatarState } from '../state/AvatarNetworkState' import { AvatarComponent } from './AvatarComponent' -import { AvatarPendingComponent } from './AvatarPendingComponent' +import { VRMComponent } from './VRMComponent' export const AvatarAnimationComponent = defineComponent({ name: 'AvatarAnimationComponent', @@ -91,6 +108,7 @@ export const AvatarAnimationComponent = defineComponent({ }) export type Matrices = { local: Matrix4; world: Matrix4 } + export const AvatarRigComponent = defineComponent({ name: 'AvatarRigComponent', @@ -124,60 +142,36 @@ export const AvatarRigComponent = defineComponent({ reactor: function () { const entity = useEntityContext() - const debugEnabled = useHookstate(getMutableState(RendererState).avatarDebug) const rigComponent = useComponent(entity, AvatarRigComponent) - const pending = useOptionalComponent(entity, AvatarPendingComponent) - const visible = useOptionalComponent(entity, VisibleComponent) - const modelComponent = useOptionalComponent(entity, ModelComponent) + const gltfComponent = useOptionalComponent(entity, GLTFComponent) const locomotionAnimationState = useHookstate( getMutableState(AnimationState).loadedAnimations[preloadedAnimations.locomotion] ) + /** @todo move asset loading to a new VRMComponent */ useEffect(() => { - if (!visible?.value || !debugEnabled.value || pending?.value || !rigComponent.value.normalizedRig?.hips?.node) - return - - const helper = new SkeletonHelper(rigComponent.value.vrm.scene as Group) - helper.frustumCulled = false - helper.name = `target-rig-helper-${entity}` - - const helperEntity = createEntity() - setVisibleComponent(helperEntity, true) - addObjectToGroup(helperEntity, helper) - rigComponent.helperEntity.set(helperEntity) - setComponent(helperEntity, NameComponent, helper.name) - setObjectLayers(helper, ObjectLayers.AvatarHelper) - - setComponent(helperEntity, ComputedTransformComponent, { - referenceEntities: [entity], - computeFunction: () => { - // this updates the bone helper lines - helper.updateMatrixWorld(true) - } - }) - + if (!rigComponent?.avatarURL?.value) return + setComponent(entity, GLTFComponent, { src: rigComponent.avatarURL.value }) return () => { - removeEntity(helperEntity) - rigComponent.helperEntity.set(none) + removeComponent(entity, GLTFComponent) } - }, [visible, debugEnabled, pending, rigComponent.normalizedRig]) + }, [rigComponent?.avatarURL?.value]) + + useEffect(() => { + if (gltfComponent?.progress?.value !== 100) return + const vrm = createVRM(entity) + rigComponent.vrm.set(vrm) + }, [gltfComponent?.progress?.value]) useEffect(() => { - if (!modelComponent?.asset?.value) return - const model = getComponent(entity, ModelComponent) - setupAvatarProportions(entity, model.asset as VRM) - setComponent(entity, AvatarRigComponent, { - vrm: model.asset as VRM, - avatarURL: model.src - }) + if (!rigComponent?.vrm?.value) return + const rig = getComponent(entity, AvatarRigComponent) + setComponent(entity, VRMComponent, rig.vrm) + setupAvatarProportions(entity, rig.vrm) return () => { - if (!entityExists(entity)) return - setComponent(entity, AvatarRigComponent, { - vrm: null!, - avatarURL: null - }) + removeComponent(entity, VRMComponent) } - }, [modelComponent?.asset]) + }, [rigComponent?.vrm?.value]) useEffect(() => { if ( @@ -205,3 +199,176 @@ export const AvatarRigComponent = defineComponent({ return null } }) + +const _rightHandPos = new Vector3(), + _rightUpperArmPos = new Vector3() + +const flip = new Matrix4().makeRotationFromEuler(new Euler(0, Math.PI, 0)) + +export default function createVRM(rootEntity: Entity) { + const documentID = GLTFComponent.getInstanceID(rootEntity) + const gltf = getState(GLTFDocumentState)[documentID] + + if (!hasComponent(rootEntity, Object3DComponent)) { + const obj3d = new Group() + setComponent(rootEntity, Object3DComponent, obj3d) + addObjectToGroup(rootEntity, obj3d) + proxifyParentChildRelationships(obj3d) + } + console.log(gltf) + if (gltf.extensions?.VRM) { + const vrmExtensionDefinition = gltf.extensions!.VRM as V0VRM.VRM + console.log(vrmExtensionDefinition) + const bones = vrmExtensionDefinition.humanoid!.humanBones!.reduce((bones, bone) => { + const nodeID = `${documentID}-${bone.node}` as EntityUUID + const entity = UUIDComponent.getEntityByUUID(nodeID) + bones[bone.bone!] = { node: getComponent(entity, BoneComponent) } + return bones + }, {} as VRMHumanBones) + + /**hacky, @todo test with vrm1 */ + iterateEntityNode(bones.hips.node.parent!.entity, (entity) => { + const bone = getOptionalComponent(entity, BoneComponent) + bone?.matrixWorld.identity() + bone?.matrixWorld.makeRotationY(Math.PI) + }) + bones.hips.node.parent!.rotateY(Math.PI) + + const humanoid = new VRMHumanoid(bones) + + const scene = getComponent(rootEntity, Object3DComponent) as any as Group + + const meta = vrmExtensionDefinition.meta! as any + + const vrm = new VRM({ + scene, + humanoid, + meta + // expressionManager: gltf.userData.vrmExpressionManager, + // firstPerson: gltf.userData.vrmFirstPerson, + // lookAt: gltf.userData.vrmLookAt, + // materials: gltf.userData.vrmMToonMaterials, + // springBoneManager: gltf.userData.vrmSpringBoneManager, + // nodeConstraintManager: gltf.userData.vrmNodeConstraintManager, + }) + + console.log(vrm) + + return vrm + } + + return createVRMFromGLTF(rootEntity, gltf) +} + +const hipsRegex = /hip|pelvis/i +export const recursiveHipsLookup = (entity: Entity): Entity | undefined => { + const name = getComponent(entity, NameComponent).toLowerCase() + if (hipsRegex.test(name)) { + return entity + } + const children = getComponent(entity, EntityTreeComponent).children + for (const child of children) { + const e = recursiveHipsLookup(child) + if (e) return e + } +} + +const createVRMFromGLTF = (rootEntity: Entity, gltf: GLTF.IGLTF) => { + const hipsEntity = recursiveHipsLookup(rootEntity)! + + const hipsName = getComponent(hipsEntity, NameComponent) + + const bones = {} as VRMHumanBones + + /** + * some mixamo rigs do not use the mixamo prefix, if they don't, we add + * a prefix to the rig names for matching to keys in the mixamoVRMRigMap + */ + const mixamoPrefix = hipsName.includes('mixamorig') ? '' : 'mixamorig' + /** + * some mixamo rigs have an identifier or suffix after the mixamo prefix + * that must be removed for matching to keys in the mixamoVRMRigMap + */ + const removeSuffix = mixamoPrefix ? false : !/[hp]/i.test(hipsName.charAt(9)) + + iterateEntityNode(hipsEntity, (entity) => { + // if (!getComponent(entity, BoneComponent)) return + const boneComponent = getOptionalComponent(entity, BoneComponent) || getComponent(entity, TransformComponent) + boneComponent?.matrixWorld.identity() + if (entity != hipsEntity) { + boneComponent.matrixWorld.makeRotationZ(Math.PI) + } + + const name = getComponent(entity, NameComponent) + /**match the keys to create a humanoid bones object */ + let boneName = mixamoPrefix + name + + if (removeSuffix) boneName = boneName.slice(0, 9) + name.slice(10) + + //remove colon from the bone name + if (boneName.includes(':')) boneName = boneName.replace(':', '') + + const bone = mixamoVRMRigMap[boneName] as string + console.log({ name, boneName, removeSuffix, bone, mixamoVRMRigMap }) + if (bone) { + bones[bone] = { node: getComponent(entity, BoneComponent) } as VRMHumanBone + } + }) + const humanoid = enforceTPose(bones) + const scene = getComponent(rootEntity, Object3DComponent) + const children = getComponent(rootEntity, EntityTreeComponent).children + const childName = getComponent(children[0], NameComponent) + + const vrm = new VRM({ + humanoid, + scene: scene, + meta: { name: childName } as VRM1Meta + // expressionManager: gltf.userData.vrmExpressionManager, + // firstPerson: gltf.userData.vrmFirstPerson, + // lookAt: gltf.userData.vrmLookAt, + // materials: gltf.userData.vrmMToonMaterials, + // springBoneManager: gltf.userData.vrmSpringBoneManager, + // nodeConstraintManager: gltf.userData.vrmNodeConstraintManager, + } as VRMParameters) + + if (!vrm.userData) vrm.userData = {} + humanoid.humanBones.rightHand.node.getWorldPosition(_rightHandPos) + humanoid.humanBones.rightUpperArm.node.getWorldPosition(_rightUpperArmPos) + return vrm +} + +const legAngle = new Euler(0, 0, Math.PI) +const rightShoulderAngle = new Euler(Math.PI / 2, 0, Math.PI / 2) +const leftShoulderAngle = new Euler(Math.PI / 2, 0, -Math.PI / 2) +export const enforceTPose = (bones: VRMHumanBones) => { + console.log('enforcing T pose', bones) + + bones.rightShoulder!.node.quaternion.setFromEuler(rightShoulderAngle) + iterateEntityNode(bones.rightShoulder!.node.entity, (entity) => { + getComponent(entity, BoneComponent).matrixWorld.makeRotationFromEuler(rightShoulderAngle) + }) + bones.rightShoulder!.node.matrixWorld.makeRotationFromEuler(rightShoulderAngle) + bones.rightUpperArm.node.quaternion.set(0, 0, 0, 1) + bones.rightLowerArm.node.quaternion.set(0, 0, 0, 1) + + bones.leftShoulder!.node.quaternion.setFromEuler(leftShoulderAngle) + iterateEntityNode(bones.leftShoulder!.node.entity, (entity) => { + getComponent(entity, BoneComponent).matrixWorld.makeRotationFromEuler(leftShoulderAngle) + }) + bones.leftUpperArm.node.quaternion.set(0, 0, 0, 1) + bones.leftLowerArm.node.quaternion.set(0, 0, 0, 1) + + bones.rightUpperLeg.node.quaternion.setFromEuler(legAngle) + iterateEntityNode(bones.rightUpperLeg!.node.entity, (entity) => { + getComponent(entity, BoneComponent).matrixWorld.makeRotationFromEuler(legAngle) + }) + bones.rightLowerLeg.node.quaternion.set(0, 0, 0, 1) + + bones.leftUpperLeg.node.quaternion.setFromEuler(legAngle) + iterateEntityNode(bones.leftUpperLeg!.node.entity, (entity) => { + getComponent(entity, BoneComponent).matrixWorld.makeRotationFromEuler(legAngle) + }) + bones.leftLowerLeg.node.quaternion.set(0, 0, 0, 1) + + return new VRMHumanoid(bones) +} diff --git a/packages/engine/src/avatar/components/AvatarDissolveComponent.ts b/packages/engine/src/avatar/components/AvatarDissolveComponent.ts index 45a0056334..3ab007a559 100644 --- a/packages/engine/src/avatar/components/AvatarDissolveComponent.ts +++ b/packages/engine/src/avatar/components/AvatarDissolveComponent.ts @@ -48,7 +48,7 @@ import { matches } from '@etherealengine/hyperflux' import { MeshComponent } from '@etherealengine/spatial/src/renderer/components/MeshComponent' import { iterateEntityNode } from '@etherealengine/spatial/src/transform/components/EntityTree' -import { SkinnedMeshComponent } from './SkinnedMeshComponent' +import { SkinnedMeshComponent } from '@etherealengine/spatial/src/renderer/components/SkinnedMeshComponent' export type MaterialMap = { entity: Entity diff --git a/packages/engine/src/avatar/components/LoopAnimationComponent.ts b/packages/engine/src/avatar/components/LoopAnimationComponent.ts index 9c69fe25ae..04ca376fd9 100644 --- a/packages/engine/src/avatar/components/LoopAnimationComponent.ts +++ b/packages/engine/src/avatar/components/LoopAnimationComponent.ts @@ -23,7 +23,6 @@ All portions of the code written by the Ethereal Engine team are Copyright © 20 Ethereal Engine. All Rights Reserved. */ -import { VRM } from '@pixiv/three-vrm' import { useEffect } from 'react' import { AnimationAction, @@ -47,9 +46,10 @@ import { NO_PROXY, useHookstate } from '@etherealengine/hyperflux' import { CallbackComponent, StandardCallbacks, setCallback } from '@etherealengine/spatial/src/common/CallbackComponent' import { useGLTF } from '../../assets/functions/resourceLoaderHooks' -import { ModelComponent } from '../../scene/components/ModelComponent' +import { GLTFComponent } from '../../gltf/GLTFComponent' import { bindAnimationClipFromMixamo, retargetAnimationClip } from '../functions/retargetMixamoRig' import { AnimationComponent } from './AnimationComponent' +import { VRMComponent } from './VRMComponent' export const LoopAnimationComponent = defineComponent({ name: 'LoopAnimationComponent', @@ -116,7 +116,8 @@ export const LoopAnimationComponent = defineComponent({ const entity = useEntityContext() const loopAnimationComponent = useComponent(entity, LoopAnimationComponent) - const modelComponent = useOptionalComponent(entity, ModelComponent) + const vrmComponent = useOptionalComponent(entity, VRMComponent) + const gltfComponent = useOptionalComponent(entity, GLTFComponent) const animComponent = useOptionalComponent(entity, AnimationComponent) const animationAction = loopAnimationComponent._action.value as AnimationAction @@ -124,16 +125,14 @@ export const LoopAnimationComponent = defineComponent({ useEffect(() => { if (!animComponent?.animations?.value) return const clip = animComponent.animations.value[loopAnimationComponent.activeClipIndex.value] as AnimationClip - const asset = modelComponent?.asset.get(NO_PROXY) ?? null - if (!modelComponent || !asset?.scene || !clip) { + if (!clip) { loopAnimationComponent._action.set(null) return } animComponent.mixer.time.set(0) - const assetObject = modelComponent.asset.get(NO_PROXY) try { const action = animComponent.mixer.value.clipAction( - assetObject instanceof VRM ? bindAnimationClipFromMixamo(clip, assetObject) : clip + vrmComponent ? bindAnimationClipFromMixamo(clip, getComponent(entity, VRMComponent)) : clip ) loopAnimationComponent._action.set(action) return () => { @@ -142,7 +141,7 @@ export const LoopAnimationComponent = defineComponent({ } catch (e) { console.warn('Failed to bind animation in LoopAnimationComponent', entity, e) } - }, [loopAnimationComponent.activeClipIndex, modelComponent?.asset, animComponent?.animations]) + }, [loopAnimationComponent.activeClipIndex, vrmComponent, animComponent?.animations]) useEffect(() => { if (animationAction?.isRunning()) { @@ -197,23 +196,13 @@ export const LoopAnimationComponent = defineComponent({ setCallback(entity, StandardCallbacks.PAUSE, pause) }, []) - /** - * A model is required for LoopAnimationComponent. - */ - useEffect(() => { - const asset = modelComponent?.asset.get(NO_PROXY) ?? null - if (!asset?.scene) return - const model = getComponent(entity, ModelComponent) - }, [modelComponent?.asset]) - const [gltf] = useGLTF(loopAnimationComponent.animationPack.value, entity) useEffect(() => { - const asset = modelComponent?.asset.get(NO_PROXY) ?? null if ( !gltf || !animComponent || - !asset?.scene || + gltfComponent?.progress.value !== 100 || !loopAnimationComponent.animationPack.value || lastAnimationPack.value === loopAnimationComponent.animationPack.value ) @@ -225,7 +214,7 @@ export const LoopAnimationComponent = defineComponent({ for (let i = 0; i < animations.length; i++) retargetAnimationClip(animations[i], gltf.scene) lastAnimationPack.set(loopAnimationComponent.animationPack.get(NO_PROXY)) animComponent.animations.set(animations) - }, [gltf, animComponent, modelComponent?.asset]) + }, [gltf, animComponent, gltfComponent?.progress]) return null } diff --git a/packages/engine/src/avatar/components/SkinnedMeshComponent.ts b/packages/engine/src/avatar/components/VRMComponent.ts similarity index 72% rename from packages/engine/src/avatar/components/SkinnedMeshComponent.ts rename to packages/engine/src/avatar/components/VRMComponent.ts index 2849274c47..e761f9d84c 100644 --- a/packages/engine/src/avatar/components/SkinnedMeshComponent.ts +++ b/packages/engine/src/avatar/components/VRMComponent.ts @@ -23,17 +23,18 @@ All portions of the code written by the Ethereal Engine team are Copyright © 20 Ethereal Engine. All Rights Reserved. */ -import { SkinnedMesh } from 'three' +import { defineComponent } from '@etherealengine/ecs' +import { VRM } from '@pixiv/three-vrm' -import { defineComponent } from '@etherealengine/ecs/src/ComponentFunctions' +export const VRMComponent = defineComponent({ + name: 'VRMComponent', -export const SkinnedMeshComponent = defineComponent({ - name: 'SkinnedMeshComponent', + onInit(entity) { + return null! as VRM + }, - onInit: (entity) => null! as SkinnedMesh, - - onSet: (entity, component, mesh: SkinnedMesh) => { - if (!mesh || !mesh.isSkinnedMesh) throw new Error('SkinnedMeshComponent: Invalid skinned mesh') - component.set(mesh) + onSet(entity, component, json) { + if (!(json instanceof VRM)) throw new Error('Invalid VRM') + component.set(json as VRM) } }) diff --git a/packages/engine/src/avatar/functions/updateVRMRetargeting.ts b/packages/engine/src/avatar/functions/updateVRMRetargeting.ts index a60f9f3c54..5aa9117bfd 100644 --- a/packages/engine/src/avatar/functions/updateVRMRetargeting.ts +++ b/packages/engine/src/avatar/functions/updateVRMRetargeting.ts @@ -31,8 +31,8 @@ import { Entity } from '@etherealengine/ecs/src/Entity' import { EntityTreeComponent } from '@etherealengine/spatial/src/transform/components/EntityTree' import { TransformComponent } from '@etherealengine/spatial/src/transform/components/TransformComponent' +import { BoneComponent } from '@etherealengine/spatial/src/renderer/components/BoneComponent' import { AvatarComponent } from '../components/AvatarComponent' -import { BoneComponent } from '../components/BoneComponent' export const updateVRMRetargeting = (vrm: VRM, avatarEntity: Entity) => { const humanoidRig = (vrm.humanoid as any)._normalizedHumanBones // as VRMHumanoidRig diff --git a/packages/engine/src/avatar/state/AvatarNetworkState.tsx b/packages/engine/src/avatar/state/AvatarNetworkState.tsx index a02a8be9b5..d02b2f3b3a 100644 --- a/packages/engine/src/avatar/state/AvatarNetworkState.tsx +++ b/packages/engine/src/avatar/state/AvatarNetworkState.tsx @@ -42,8 +42,8 @@ import { import { WorldNetworkAction } from '@etherealengine/network' import { NameComponent } from '@etherealengine/spatial/src/common/NameComponent' +import { AvatarRigComponent } from '../components/AvatarAnimationComponent' import { AvatarColliderComponent } from '../components/AvatarControllerComponent' -import { loadAvatarModelAsset, unloadAvatarForUser } from '../functions/avatarFunctions' import { spawnAvatarReceptor } from '../functions/spawnAvatarReceptor' import { AvatarNetworkAction } from './AvatarNetworkActions' @@ -142,11 +142,11 @@ const AvatarReactor = ({ entityUUID }: { entityUUID: EntityUUID }) => { if (!isClient) return if (!entity || !userAvatarDetails.value) return - loadAvatarModelAsset(entity, userAvatarDetails.value) + setComponent(entity, AvatarRigComponent, { avatarURL: userAvatarDetails.value }) return () => { if (!entityExists(entity)) return - unloadAvatarForUser(entity) + setComponent(entity, AvatarRigComponent, { avatarURL: '' }) } }, [userAvatarDetails, entity]) diff --git a/packages/engine/src/avatar/systems/AnimationSystem.ts b/packages/engine/src/avatar/systems/AnimationSystem.ts index 69f88a66ac..98f9cddcf0 100644 --- a/packages/engine/src/avatar/systems/AnimationSystem.ts +++ b/packages/engine/src/avatar/systems/AnimationSystem.ts @@ -23,8 +23,6 @@ All portions of the code written by the Ethereal Engine team are Copyright © 20 Ethereal Engine. All Rights Reserved. */ -import { VRM } from '@pixiv/three-vrm' - import { getComponent, getOptionalMutableComponent } from '@etherealengine/ecs/src/ComponentFunctions' import { ECSState } from '@etherealengine/ecs/src/ECSState' import { defineQuery } from '@etherealengine/ecs/src/QueryFunctions' @@ -35,14 +33,14 @@ import { TransformComponent } from '@etherealengine/spatial/src/transform/compon import { TweenComponent } from '@etherealengine/spatial/src/transform/components/TweenComponent' import { TransformDirtyUpdateSystem } from '@etherealengine/spatial/src/transform/systems/TransformSystem' -import { ModelComponent } from '../../scene/components/ModelComponent' import { AnimationComponent } from '.././components/AnimationComponent' import { LoopAnimationComponent } from '../components/LoopAnimationComponent' +import { VRMComponent } from '../components/VRMComponent' import { updateVRMRetargeting } from '../functions/updateVRMRetargeting' const tweenQuery = defineQuery([TweenComponent]) const animationQuery = defineQuery([AnimationComponent, VisibleComponent]) -const loopAnimationQuery = defineQuery([AnimationComponent, LoopAnimationComponent, ModelComponent, TransformComponent]) +const loopAnimationQuery = defineQuery([AnimationComponent, LoopAnimationComponent, VRMComponent, TransformComponent]) const execute = () => { const { deltaSeconds } = getState(ECSState) @@ -62,10 +60,7 @@ const execute = () => { } for (const entity of loopAnimationQuery()) { - const model = getComponent(entity, ModelComponent) - if (model.asset instanceof VRM) { - updateVRMRetargeting(model.asset, entity) - } + updateVRMRetargeting(getComponent(entity, VRMComponent), entity) } } diff --git a/packages/engine/src/avatar/systems/AvatarAnimationSystem.tsx b/packages/engine/src/avatar/systems/AvatarAnimationSystem.tsx index 252bcb3411..c72e5f03dd 100644 --- a/packages/engine/src/avatar/systems/AvatarAnimationSystem.tsx +++ b/packages/engine/src/avatar/systems/AvatarAnimationSystem.tsx @@ -59,6 +59,7 @@ import { XRLeftHandComponent, XRRightHandComponent } from '@etherealengine/spati import { XRState } from '@etherealengine/spatial/src/xr/XRState' import { EngineState } from '@etherealengine/spatial/src/EngineState' +import { SkinnedMeshComponent } from '@etherealengine/spatial/src/renderer/components/SkinnedMeshComponent' import React from 'react' import { useBatchGLTF } from '../../assets/functions/resourceLoaderHooks' import { GLTF } from '../../assets/loaders/gltf/GLTFLoader' @@ -72,7 +73,6 @@ import { AnimationComponent } from '../components/AnimationComponent' import { AvatarAnimationComponent, AvatarRigComponent } from '../components/AvatarAnimationComponent' import { AvatarComponent } from '../components/AvatarComponent' import { AvatarIKTargetComponent } from '../components/AvatarIKComponents' -import { SkinnedMeshComponent } from '../components/SkinnedMeshComponent' import { retargetAnimationClip } from '../functions/retargetMixamoRig' import { updateVRMRetargeting } from '../functions/updateVRMRetargeting' import { IKSerialization } from '../IKSerialization' diff --git a/packages/engine/src/avatar/systems/AvatarTransparencySystem.tsx b/packages/engine/src/avatar/systems/AvatarTransparencySystem.tsx index 0222531bd1..a5d00464a2 100644 --- a/packages/engine/src/avatar/systems/AvatarTransparencySystem.tsx +++ b/packages/engine/src/avatar/systems/AvatarTransparencySystem.tsx @@ -50,8 +50,8 @@ import { ditherCalculationType } from '@etherealengine/spatial/src/renderer/materials/constants/plugins/TransparencyDitheringComponent' import React, { useEffect } from 'react' +import { GLTFComponent } from '../../gltf/GLTFComponent' import { SourceComponent } from '../../scene/components/SourceComponent' -import { useModelSceneID } from '../../scene/functions/loaders/ModelFunctions' import { AvatarComponent } from '../components/AvatarComponent' const headDithering = 0 @@ -115,7 +115,7 @@ export const AvatarTransparencySystem = defineSystem({ const AvatarReactor = (props: { entity: Entity }) => { const entity = props.entity - const sceneInstanceID = useModelSceneID(entity) + const sceneInstanceID = GLTFComponent.useInstanceID(entity) const childEntities = useHookstate(SourceComponent.entitiesBySourceState[sceneInstanceID]) return ( <> diff --git a/packages/engine/src/gltf/GLTFComponent.tsx b/packages/engine/src/gltf/GLTFComponent.tsx index c2803003e4..74141d08ad 100644 --- a/packages/engine/src/gltf/GLTFComponent.tsx +++ b/packages/engine/src/gltf/GLTFComponent.tsx @@ -30,75 +30,120 @@ import { parseStorageProviderURLs } from '@etherealengine/common/src/utils/parse import { defineComponent, Entity, - EntityUUID, getComponent, getMutableComponent, getOptionalComponent, hasComponent, useComponent, useEntityContext, + useOptionalComponent, useQuery, UUIDComponent } from '@etherealengine/ecs' -import { dispatchAction, getState, useHookstate } from '@etherealengine/hyperflux' - +import { + dispatchAction, + getMutableState, + getState, + none, + useHookstate, + useMutableState +} from '@etherealengine/hyperflux' + +import { MeshComponent } from '@etherealengine/spatial/src/renderer/components/MeshComponent' +import { ObjectLayerMaskComponent } from '@etherealengine/spatial/src/renderer/components/ObjectLayerComponent' +import { SceneComponent } from '@etherealengine/spatial/src/renderer/components/SceneComponents' +import { ObjectLayers } from '@etherealengine/spatial/src/renderer/constants/ObjectLayers' +import { useAncestorWithComponents } from '@etherealengine/spatial/src/transform/components/EntityTree' import { FileLoader } from '../assets/loaders/base/FileLoader' -import { BINARY_EXTENSION_HEADER_MAGIC, EXTENSIONS, GLTFBinaryExtension } from '../assets/loaders/gltf/GLTFExtensions' -import { ModelComponent } from '../scene/components/ModelComponent' +import { + BINARY_EXTENSION_CHUNK_TYPES, + BINARY_EXTENSION_HEADER_LENGTH, + BINARY_EXTENSION_HEADER_MAGIC +} from '../assets/loaders/gltf/GLTFExtensions' import { SourceComponent } from '../scene/components/SourceComponent' import { SceneJsonType } from '../scene/types/SceneTypes' import { migrateSceneJSONToGLTF } from './convertJsonToGLTF' import { GLTFDocumentState, GLTFSnapshotAction } from './GLTFDocumentState' +import { GLTFSourceState } from './GLTFState' import { ResourcePendingComponent } from './ResourcePendingComponent' export const GLTFComponent = defineComponent({ name: 'GLTFComponent', + jsonID: 'EE_model', onInit(entity) { return { src: '', + /** @todo move this to it's own component */ + cameraOcclusion: false, // internals - extensions: {}, + body: null as null | ArrayBuffer, progress: 0 } }, onSet(entity, component, json) { if (typeof json?.src === 'string') component.src.set(json.src) + if (typeof json?.cameraOcclusion === 'boolean') component.cameraOcclusion.set(json.cameraOcclusion) }, reactor: () => { const entity = useEntityContext() const gltfComponent = useComponent(entity, GLTFComponent) + useEffect(() => { + const occlusion = gltfComponent.cameraOcclusion.value + if (!occlusion) ObjectLayerMaskComponent.disableLayer(entity, ObjectLayers.Camera) + else ObjectLayerMaskComponent.enableLayer(entity, ObjectLayers.Camera) + }, [gltfComponent.cameraOcclusion]) + useGLTFDocument(gltfComponent.src.value, entity) - const documentID = useComponent(entity, SourceComponent).value + const sourceID = GLTFComponent.getInstanceID(entity) - return + useEffect(() => { + getMutableState(GLTFSourceState)[sourceID].set(entity) + return () => { + getMutableState(GLTFSourceState)[sourceID].set(none) + } + }, []) + + return + }, + + getInstanceID: (entity) => { + return `${getComponent(entity, UUIDComponent)}-${getComponent(entity, GLTFComponent).src}` + }, + + useInstanceID: (entity) => { + const uuid = useComponent(entity, UUIDComponent)?.value + const src = useComponent(entity, GLTFComponent)?.src.value + if (!uuid || !src) return '' + return `${uuid}-${src}` } }) const ResourceReactor = (props: { documentID: string; entity: Entity }) => { const resourceQuery = useQuery([SourceComponent, ResourcePendingComponent]) + const gltfDocumentState = useMutableState(GLTFDocumentState) const sourceEntities = useHookstate(SourceComponent.entitiesBySourceState[props.documentID]) useEffect(() => { if (getComponent(props.entity, GLTFComponent).progress === 100) return if (!getState(GLTFDocumentState)[props.documentID]) return - const document = getState(GLTFDocumentState)[props.documentID] - const modelNodes = document.nodes?.filter((node) => !!node.extensions?.[ModelComponent.jsonID]) - if (modelNodes) { - for (const node of modelNodes) { - //check if an entity exists for this node, and has a model component - const uuid = node.extensions![UUIDComponent.jsonID] as EntityUUID - if (!UUIDComponent.entitiesByUUIDState[uuid]) return - const entity = UUIDComponent.entitiesByUUIDState[uuid].value - const model = getOptionalComponent(entity, ModelComponent) - //ensure that model contents have been loaded into the scene - if (!model?.scene) return - } - } + // const document = getState(GLTFDocumentState)[props.documentID] + // const modelNodes = document.nodes?.filter((node) => !!node.extensions?.[ModelComponent.jsonID]) + // if (modelNodes) { + // for (const node of modelNodes) { + // //check if an entity exists for this node, and has a model component + // const uuid = node.extensions![UUIDComponent.jsonID] as EntityUUID + // if (!UUIDComponent.entitiesByUUIDState[uuid]) return + // const entity = UUIDComponent.entitiesByUUIDState[uuid].value + // const model = getOptionalComponent(entity, ModelComponent) + // //ensure that model contents have been loaded into the scene + // if (!model?.scene) return + // } + // } const entities = resourceQuery.filter((e) => getComponent(e, SourceComponent) === props.documentID) if (!entities.length) { getMutableComponent(props.entity, GLTFComponent).progress.set(100) @@ -124,7 +169,7 @@ const ResourceReactor = (props: { documentID: string; entity: Entity }) => { const percentage = total === 0 ? 100 : (progress / total) * 100 getMutableComponent(props.entity, GLTFComponent).progress.set(percentage) - }, [resourceQuery, sourceEntities]) + }, [resourceQuery, sourceEntities, gltfDocumentState[props.documentID]]) return null } @@ -139,10 +184,9 @@ const onProgress: (event: ProgressEvent) => void = (event) => { const useGLTFDocument = (url: string, entity: Entity) => { const state = useComponent(entity, GLTFComponent) - const sourceComponent = useComponent(entity, SourceComponent) + const source = GLTFComponent.getInstanceID(entity) useEffect(() => { - const source = sourceComponent.value return () => { dispatchAction(GLTFSnapshotAction.unload({ source })) } @@ -167,14 +211,13 @@ const useGLTFDocument = (url: string, entity: Entity) => { if (magic === BINARY_EXTENSION_HEADER_MAGIC) { try { - /** TODO we will need to refactor and persist this */ - state.extensions.merge({ [EXTENSIONS.KHR_BINARY_GLTF]: new GLTFBinaryExtension(data) }) + const { json: jsonContent, body } = parseBinaryData(data) + state.body.set(body) + json = jsonContent } catch (error) { if (onError) onError(error) return } - - json = JSON.parse(state.extensions.value[EXTENSIONS.KHR_BINARY_GLTF].content) } else { json = JSON.parse(textDecoder.decode(data)) } @@ -189,7 +232,7 @@ const useGLTFDocument = (url: string, entity: Entity) => { dispatchAction( GLTFSnapshotAction.createSnapshot({ - source: getComponent(entity, SourceComponent), + source, data: parseStorageProviderURLs(JSON.parse(JSON.stringify(json))) }) ) @@ -206,9 +249,69 @@ const useGLTFDocument = (url: string, entity: Entity) => { return () => { abortController.abort() if (!hasComponent(entity, GLTFComponent)) return - state.merge({ - extensions: {} - }) + state.body.set(null) } }, [url]) } + +export const parseBinaryData = (data) => { + const headerView = new DataView(data, 0, BINARY_EXTENSION_HEADER_LENGTH) + const textDecoder = new TextDecoder() + + const header = { + magic: textDecoder.decode(new Uint8Array(data.slice(0, 4))), + version: headerView.getUint32(4, true), + length: headerView.getUint32(8, true) + } + + if (header.magic !== BINARY_EXTENSION_HEADER_MAGIC) { + throw new Error('THREE.GLTFLoader: Unsupported glTF-Binary header.') + } else if (header.version < 2.0) { + throw new Error('THREE.GLTFLoader: Legacy binary file detected.') + } + + const chunkContentsLength = header.length - BINARY_EXTENSION_HEADER_LENGTH + const chunkView = new DataView(data, BINARY_EXTENSION_HEADER_LENGTH) + let chunkIndex = 0 + + let content = null as string | null + let body = null as ArrayBuffer | null + + while (chunkIndex < chunkContentsLength) { + const chunkLength = chunkView.getUint32(chunkIndex, true) + chunkIndex += 4 + + const chunkType = chunkView.getUint32(chunkIndex, true) + chunkIndex += 4 + + if (chunkType === BINARY_EXTENSION_CHUNK_TYPES.JSON) { + const contentArray = new Uint8Array(data, BINARY_EXTENSION_HEADER_LENGTH + chunkIndex, chunkLength) + content = textDecoder.decode(contentArray) + } else if (chunkType === BINARY_EXTENSION_CHUNK_TYPES.BIN) { + const byteOffset = BINARY_EXTENSION_HEADER_LENGTH + chunkIndex + body = data.slice(byteOffset, byteOffset + chunkLength) + } + + // Clients must ignore chunks with unknown types. + + chunkIndex += chunkLength + } + + if (content === null) { + throw new Error('THREE.GLTFLoader: JSON content not found.') + } + + return { json: JSON.parse(content), body } +} + +/** + * Returns true if the entity is part of a model or a mesh component that is not a child of model + * @param entity + * @returns {boolean} + */ +export const useHasModelOrIndependentMesh = (entity: Entity) => { + const hasModel = !!useOptionalComponent(entity, GLTFComponent) + const isChildOfModel = !!useAncestorWithComponents(entity, [GLTFComponent, SceneComponent]) + const hasMesh = !!useOptionalComponent(entity, MeshComponent) + return hasModel || (hasMesh && !isChildOfModel) +} diff --git a/packages/engine/src/gltf/GLTFDocumentState.ts b/packages/engine/src/gltf/GLTFDocumentState.ts index afef86140f..81afecd8a1 100644 --- a/packages/engine/src/gltf/GLTFDocumentState.ts +++ b/packages/engine/src/gltf/GLTFDocumentState.ts @@ -100,18 +100,18 @@ export const GLTFNodeState = defineState({ } }, - convertGltfToNodeDictionary: (gltf: GLTF.IGLTF) => { + convertGltfToNodeDictionary: (gltf: GLTF.IGLTF, source: string) => { const nodes: Record = {} const addNode = (nodeIndex: number, childIndex: number, parentUUID: EntityUUID | null) => { const node = gltf.nodes![nodeIndex] - const uuid = node.extensions?.[UUIDComponent.jsonID] as any as EntityUUID + const uuid = (node.extensions?.[UUIDComponent.jsonID] as any as EntityUUID) ?? source + '-' + nodeIndex if (uuid) { nodes[uuid] = { nodeIndex, childIndex, parentUUID } } else { /** @todo generate a globally deterministic UUID here */ - console.warn('Node does not have a UUID:', node) - return + // console.warn('Node does not have a UUID:', node) + // return } if (node.children) { for (let i = 0; i < node.children.length; i++) { @@ -129,7 +129,7 @@ export const GLTFNodeState = defineState({ for (let i = 0; i < gltf.scenes![0].nodes!.length; i++) { const nodeIndex = gltf.scenes![0].nodes![i] const node = gltf.nodes![nodeIndex] - const uuid = node.extensions?.[UUIDComponent.jsonID] as any as EntityUUID + const uuid = (node.extensions?.[UUIDComponent.jsonID] as any as EntityUUID) ?? source + '-' + nodeIndex if (uuid) { nodes[uuid] = { nodeIndex, diff --git a/packages/engine/src/gltf/GLTFExtensions.ts b/packages/engine/src/gltf/GLTFExtensions.ts new file mode 100644 index 0000000000..f6142b6da7 --- /dev/null +++ b/packages/engine/src/gltf/GLTFExtensions.ts @@ -0,0 +1,203 @@ +/* +CPAL-1.0 License + +The contents of this file are subject to the Common Public Attribution License +Version 1.0. (the "License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at +https://github.com/EtherealEngine/etherealengine/blob/dev/LICENSE. +The License is based on the Mozilla Public License Version 1.1, but Sections 14 +and 15 have been added to cover use of software over a computer network and +provide for limited attribution for the Original Developer. In addition, +Exhibit A has been modified to be consistent with Exhibit B. + +Software distributed under the License is distributed on an "AS IS" basis, +WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License for the +specific language governing rights and limitations under the License. + +The Original Code is Ethereal Engine. + +The Original Developer is the Initial Developer. The Initial Developer of the +Original Code is the Ethereal Engine team. + +All portions of the code written by the Ethereal Engine team are Copyright © 2021-2023 +Ethereal Engine. All Rights Reserved. +*/ + +import { getState, startReactor } from '@etherealengine/hyperflux' +import { GLTF } from '@gltf-transform/core' +import { useEffect } from 'react' +import { BufferGeometry, NormalBufferAttributes } from 'three' +import { ATTRIBUTES, WEBGL_COMPONENT_TYPES } from '../assets/loaders/gltf/GLTFConstants' +import { EXTENSIONS } from '../assets/loaders/gltf/GLTFExtensions' +import { GLTFParserOptions } from '../assets/loaders/gltf/GLTFParser' +import { AssetLoaderState } from '../assets/state/AssetLoaderState' +import { GLTFLoaderFunctions } from './GLTFLoaderFunctions' + +export const KHR_DRACO_MESH_COMPRESSION = { + decodePrimitive(options: GLTFParserOptions, primitive: GLTF.IMeshPrimitive) { + const json = options.document + const dracoMeshCompressionExtension = primitive.extensions![EXTENSIONS.KHR_DRACO_MESH_COMPRESSION] as any + const bufferViewIndex = dracoMeshCompressionExtension.bufferView + const gltfAttributeMap = dracoMeshCompressionExtension.attributes + const threeAttributeMap = {} as { [key: string]: string } + const attributeNormalizedMap = {} as { [key: string]: boolean } + const attributeTypeMap = {} as { [key: string]: string } + + for (const attributeName in gltfAttributeMap) { + const threeAttributeName = ATTRIBUTES[attributeName] || attributeName.toLowerCase() + + threeAttributeMap[threeAttributeName] = gltfAttributeMap[attributeName] + } + + for (const attributeName in primitive.attributes) { + const threeAttributeName = ATTRIBUTES[attributeName] || attributeName.toLowerCase() + + if (gltfAttributeMap[attributeName] !== undefined) { + const accessorDef = json.accessors![primitive.attributes[attributeName]] + const componentType = WEBGL_COMPONENT_TYPES[accessorDef.componentType] + + attributeTypeMap[threeAttributeName] = componentType.name + attributeNormalizedMap[threeAttributeName] = accessorDef.normalized === true + } + } + + return new Promise>(function (resolve) { + /** + * Using an inline reactor here allows us to use reference counting & resource caching, + * and release the uncompressed buffer as soon as it is no longer required + */ + const reactor = startReactor(() => { + const bufferView = GLTFLoaderFunctions.useLoadBufferView(options, bufferViewIndex) + useEffect(() => { + if (!bufferView) return + const dracoLoader = getState(AssetLoaderState).gltfLoader.dracoLoader! + dracoLoader.preload().decodeDracoFile( + bufferView, + function (geometry) { + for (const attributeName in geometry.attributes) { + const attribute = geometry.attributes[attributeName] + const normalized = attributeNormalizedMap[attributeName] + if (normalized !== undefined) attribute.normalized = normalized + } + resolve(geometry) + reactor.stop() + }, + threeAttributeMap, + attributeTypeMap + ) + }, [bufferView]) + return null + }) + }) + } +} + +export type KHRMeshOptExtensionType = { + buffer: number + byteOffset?: number + byteLength?: number + byteStride?: number + count: number + mode?: number + filter?: number +} + +/** + * meshopt BufferView Compression Extension + * + * Specification: https://github.com/KhronosGroup/glTF/tree/master/extensions/2.0/Vendor/EXT_meshopt_compression + */ +export const EXT_MESHOPT_COMPRESSION = { + loadBuffer: (options: GLTFParserOptions, bufferViewIndex: number) => { + const json = options.document + const bufferViewDef = json.bufferViews![bufferViewIndex] + const extensionDef = bufferViewDef.extensions![EXTENSIONS.EXT_MESHOPT_COMPRESSION] as KHRMeshOptExtensionType + return [ + extensionDef.buffer, + (bufferView: ArrayBuffer) => + new Promise((resolve, reject) => { + const json = options.document + + const byteOffset = extensionDef.byteOffset || 0 + const byteLength = extensionDef.byteLength || 0 + + const count = extensionDef.count + const stride = extensionDef.byteStride! + + const source = new Uint8Array(bufferView, byteOffset, byteLength) + + const decoder = getState(AssetLoaderState).gltfLoader.meshoptDecoder + if (!decoder || !decoder.supported) { + if (json.extensionsRequired && json.extensionsRequired.indexOf(EXTENSIONS.EXT_MESHOPT_COMPRESSION) >= 0) { + return reject('THREE.GLTFLoader: setMeshoptDecoder must be called before loading compressed files') + } else { + // Assumes that the extension is optional and that fallback buffer data is present + return resolve(null) + } + } + + if (decoder.decodeGltfBufferAsync) { + decoder + .decodeGltfBufferAsync(count, stride, source, extensionDef.mode!, extensionDef.filter!) + .then(function (res) { + resolve(res.buffer) + }) + } else { + // Support for MeshoptDecoder 0.18 or earlier, without decodeGltfBufferAsync + decoder.ready!.then(function () { + const result = new ArrayBuffer(count * stride) + decoder.decodeGltfBuffer( + new Uint8Array(result), + count, + stride, + source, + extensionDef.mode!, + extensionDef.filter! + ) + resolve(result) + }) + } + }) + ] as [number | null, (bufferView: ArrayBuffer) => Promise] + } +} + +type GLTFExtensionType = { + decodePrimitive?: ( + options: GLTFParserOptions, + primitive: GLTF.IMeshPrimitive + ) => Promise> + loadBuffer?: ( + options: GLTFParserOptions, + index: number + ) => [number | null, (bufferView: ArrayBuffer) => Promise] +} + +export const getBufferIndex = (options: GLTFParserOptions, bufferViewIndex?: number) => { + const json = options.document + if (typeof bufferViewIndex !== 'number') + return [null, async (buffer: ArrayBuffer) => buffer] as [ + number | null, + (bufferView: ArrayBuffer) => Promise + ] + const bufferViewDef = json.bufferViews![bufferViewIndex] + for (const extensionName in bufferViewDef.extensions) { + const extension = GLTFExtensions[extensionName] + if (extension.loadBuffer) { + return extension.loadBuffer(options, bufferViewIndex!) + } + } + return [ + bufferViewDef.buffer, + async (buffer) => { + const byteLength = bufferViewDef!.byteLength || 0 + const byteOffset = bufferViewDef!.byteOffset || 0 + return buffer.slice(byteOffset, byteOffset + byteLength) + } + ] as [number | null, (bufferView: ArrayBuffer) => Promise] +} + +export const GLTFExtensions = { + [EXTENSIONS.KHR_DRACO_MESH_COMPRESSION]: KHR_DRACO_MESH_COMPRESSION, + [EXTENSIONS.EXT_MESHOPT_COMPRESSION]: EXT_MESHOPT_COMPRESSION +} as Record diff --git a/packages/engine/src/gltf/GLTFLoaderFunctions.ts b/packages/engine/src/gltf/GLTFLoaderFunctions.ts new file mode 100644 index 0000000000..747ca9fdba --- /dev/null +++ b/packages/engine/src/gltf/GLTFLoaderFunctions.ts @@ -0,0 +1,1194 @@ +/* +CPAL-1.0 License + +The contents of this file are subject to the Common Public Attribution License +Version 1.0. (the "License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at +https://github.com/EtherealEngine/etherealengine/blob/dev/LICENSE. +The License is based on the Mozilla Public License Version 1.1, but Sections 14 +and 15 have been added to cover use of software over a computer network and +provide for limited attribution for the Original Developer. In addition, +Exhibit A has been modified to be consistent with Exhibit B. + +Software distributed under the License is distributed on an "AS IS" basis, +WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License for the +specific language governing rights and limitations under the License. + +The Original Code is Ethereal Engine. + +The Original Developer is the Initial Developer. The Initial Developer of the +Original Code is the Ethereal Engine team. + +All portions of the code written by the Ethereal Engine team are Copyright © 2021-2023 +Ethereal Engine. All Rights Reserved. +*/ + +import { + ComponentType, + EntityUUID, + UUIDComponent, + getComponent, + getOptionalComponent, + useOptionalComponent +} from '@etherealengine/ecs' +import { NO_PROXY, getState, startReactor, useHookstate } from '@etherealengine/hyperflux' +import { NameComponent } from '@etherealengine/spatial/src/common/NameComponent' +import { BoneComponent } from '@etherealengine/spatial/src/renderer/components/BoneComponent' +import { MeshComponent } from '@etherealengine/spatial/src/renderer/components/MeshComponent' +import { MaterialPrototypeComponent } from '@etherealengine/spatial/src/renderer/materials/MaterialComponent' +import { GLTF } from '@gltf-transform/core' +import { useEffect } from 'react' +import { + AnimationClip, + Bone, + Box3, + BufferAttribute, + BufferGeometry, + Color, + ColorManagement, + DoubleSide, + FrontSide, + ImageBitmapLoader, + ImageLoader, + InterleavedBuffer, + InterleavedBufferAttribute, + InterpolateLinear, + KeyframeTrack, + LinearFilter, + LinearMipmapLinearFilter, + LinearSRGBColorSpace, + Loader, + LoaderUtils, + Mesh, + MeshBasicMaterial, + MeshStandardMaterial, + NumberKeyframeTrack, + Object3D, + QuaternionKeyframeTrack, + RepeatWrapping, + SRGBColorSpace, + SkinnedMesh, + Sphere, + Texture, + TextureLoader, + Vector2, + Vector3, + VectorKeyframeTrack +} from 'three' +import { FileLoader } from '../assets/loaders/base/FileLoader' +import { + ALPHA_MODES, + ATTRIBUTES, + INTERPOLATION, + PATH_PROPERTIES, + WEBGL_COMPONENT_TYPES, + WEBGL_FILTERS, + WEBGL_TYPE_SIZES, + WEBGL_WRAPPINGS +} from '../assets/loaders/gltf/GLTFConstants' +import { EXTENSIONS } from '../assets/loaders/gltf/GLTFExtensions' +import { + GLTFCubicSplineInterpolant, + GLTFCubicSplineQuaternionInterpolant, + assignExtrasToUserData, + getNormalizedComponentScale +} from '../assets/loaders/gltf/GLTFLoaderFunctions' +import { GLTFParserOptions, GLTFRegistry, getImageURIMimeType } from '../assets/loaders/gltf/GLTFParser' +import { KTX2Loader } from '../assets/loaders/gltf/KTX2Loader' +import { AssetLoaderState } from '../assets/state/AssetLoaderState' +import { KHR_DRACO_MESH_COMPRESSION, getBufferIndex } from './GLTFExtensions' +import { KHRTextureTransformExtensionComponent, MaterialDefinitionComponent } from './MaterialDefinitionComponent' + +// todo make this a state +const cache = new GLTFRegistry() + +const useLoadPrimitive = (options: GLTFParserOptions, nodeIndex: number, primitiveIndex: number) => { + const result = useHookstate(null as null | BufferGeometry) + + const json = options.document + const node = json.nodes![nodeIndex]! + const mesh = json.meshes![node.mesh!] + + const primitive = mesh.primitives[primitiveIndex] + + const hasDracoCompression = primitive.extensions && primitive.extensions[EXTENSIONS.KHR_DRACO_MESH_COMPRESSION] + + useEffect(() => { + if (ColorManagement.workingColorSpace !== LinearSRGBColorSpace && 'COLOR_0' in primitive.attributes) { + console.warn( + `THREE.GLTFLoader: Converting vertex colors from "srgb-linear" to "${ColorManagement.workingColorSpace}" not supported.` + ) + } + + if (hasDracoCompression) { + KHR_DRACO_MESH_COMPRESSION.decodePrimitive(options, primitive).then((geom) => { + GLTFLoaderFunctions.computeBounds(json, geom, primitive) + assignExtrasToUserData(geom, primitive as GLTF.IMeshPrimitive) + result.set(geom) + }) + } else { + const geometry = new BufferGeometry() + + /** @todo we need to figure out a better way of handling reactivity for both draco and regular buffers */ + const reactor = startReactor(() => { + const attributes = primitive.attributes + const resourcesState = useHookstate( + () => + ({ + ...Object.fromEntries(Object.keys(attributes).map((key) => [key, false])), + index: false + }) as Record + ) + + for (const attributeName of Object.keys(attributes)) { + const threeAttributeName = ATTRIBUTES[attributeName] || attributeName.toLowerCase() + const attribute = primitive.attributes[attributeName] + const accessor = GLTFLoaderFunctions.useLoadAccessor(options, attribute) + useEffect(() => { + if (!accessor) return + geometry.setAttribute(threeAttributeName, accessor) + resourcesState[attributeName].set(true) + }, [accessor]) + } + + const accessor = GLTFLoaderFunctions.useLoadAccessor(options, primitive.indices!) + + useEffect(() => { + if (!accessor) return + geometry.setIndex(accessor) + resourcesState.index.set(true) + }, [accessor]) + + useEffect(() => { + const attributeCount = Object.keys(attributes).length + const resourcesLoaded = Object.values(resourcesState.get(NO_PROXY)).filter(Boolean).length + if (resourcesLoaded !== attributeCount + (typeof primitive.indices === 'number' ? 1 : 0)) return + + GLTFLoaderFunctions.computeBounds(json, geometry, primitive) + assignExtrasToUserData(geometry, primitive as GLTF.IMeshPrimitive) + result.set(geometry) + reactor.stop() + }, [resourcesState]) + + return null + }) + return () => { + reactor.stop() + } + } + }, [primitive.extensions]) + + return result.get(NO_PROXY) as BufferGeometry | null +} + +const useLoadAccessor = (options: GLTFParserOptions, accessorIndex?: number) => { + const json = options.document + + const result = useHookstate(null) + + const accessorDef = typeof accessorIndex === 'number' ? json.accessors![accessorIndex] : null + + const bufferView = GLTFLoaderFunctions.useLoadBufferView(options, accessorDef?.bufferView) + + const sparseBufferViewIndices = GLTFLoaderFunctions.useLoadBufferView( + options, + accessorDef?.sparse?.indices?.bufferView + ) + const sparseBufferViewValues = GLTFLoaderFunctions.useLoadBufferView(options, accessorDef?.sparse?.values?.bufferView) + + useEffect(() => { + if (!accessorDef || !bufferView) return + + if (accessorDef.bufferView === undefined && accessorDef.sparse === undefined) { + const itemSize = WEBGL_TYPE_SIZES[accessorDef.type] + const TypedArray = WEBGL_COMPONENT_TYPES[accessorDef.componentType] + const normalized = accessorDef.normalized === true + + const array = new TypedArray(accessorDef.count * itemSize) + result.set(new BufferAttribute(array, itemSize, normalized)) + return + } + + if (typeof accessorDef.bufferView === 'number' && !bufferView) return + if (accessorDef.sparse && !sparseBufferViewIndices && !sparseBufferViewValues) return + + const itemSize = WEBGL_TYPE_SIZES[accessorDef.type] + const TypedArray = WEBGL_COMPONENT_TYPES[accessorDef.componentType] + + // For VEC3: itemSize is 3, elementBytes is 4, itemBytes is 12. + const elementBytes = TypedArray.BYTES_PER_ELEMENT + const itemBytes = elementBytes * itemSize + const byteOffset = accessorDef.byteOffset || 0 + const byteStride = + accessorDef.bufferView !== undefined ? json.bufferViews![accessorDef.bufferView].byteStride : undefined + const normalized = accessorDef.normalized === true + let array, bufferAttribute + + // The buffer is not interleaved if the stride is the item size in bytes. + if (byteStride && byteStride !== itemBytes) { + // Each "slice" of the buffer, as defined by 'count' elements of 'byteStride' bytes, gets its own InterleavedBuffer + // This makes sure that IBA.count reflects accessor.count properly + const ibSlice = Math.floor(byteOffset / byteStride) + const ibCacheKey = + 'InterleavedBuffer:' + + accessorDef.bufferView + + ':' + + accessorDef.componentType + + ':' + + ibSlice + + ':' + + accessorDef.count + let ib = cache.get(ibCacheKey) + + if (!ib) { + array = new TypedArray(bufferView!, ibSlice * byteStride, (accessorDef.count * byteStride) / elementBytes) + + // Integer parameters to IB/IBA are in array elements, not bytes. + ib = new InterleavedBuffer(array, byteStride / elementBytes) + + cache.add(ibCacheKey, ib) + } + + bufferAttribute = new InterleavedBufferAttribute( + ib, + itemSize, + (byteOffset % byteStride) / elementBytes, + normalized + ) + } else { + if (bufferView === null) { + array = new TypedArray(accessorDef.count * itemSize) + } else { + array = new TypedArray(bufferView, byteOffset, accessorDef.count * itemSize) + } + + bufferAttribute = new BufferAttribute(array, itemSize, normalized) + } + + // https://github.com/KhronosGroup/glTF/blob/master/specification/2.0/README.md#sparse-accessors + if (accessorDef.sparse !== undefined) { + const itemSizeIndices = WEBGL_TYPE_SIZES.SCALAR + const TypedArrayIndices = WEBGL_COMPONENT_TYPES[accessorDef.sparse.indices.componentType] + + const byteOffsetIndices = accessorDef.sparse.indices.byteOffset || 0 + const byteOffsetValues = accessorDef.sparse.values.byteOffset || 0 + + const sparseIndices = new TypedArrayIndices( + sparseBufferViewIndices!, + byteOffsetIndices, + accessorDef.sparse.count * itemSizeIndices + ) + const sparseValues = new TypedArray( + sparseBufferViewValues!, + byteOffsetValues, + accessorDef.sparse.count * itemSize + ) + + if (bufferView !== null) { + // Avoid modifying the original ArrayBuffer, if the bufferView wasn't initialized with zeroes. + bufferAttribute = new BufferAttribute( + bufferAttribute.array.slice(), + bufferAttribute.itemSize, + bufferAttribute.normalized + ) + } + + for (let i = 0, il = sparseIndices.length; i < il; i++) { + const index = sparseIndices[i] + + bufferAttribute.setX(index, sparseValues[i * itemSize]) + if (itemSize >= 2) bufferAttribute.setY(index, sparseValues[i * itemSize + 1]) + if (itemSize >= 3) bufferAttribute.setZ(index, sparseValues[i * itemSize + 2]) + if (itemSize >= 4) bufferAttribute.setW(index, sparseValues[i * itemSize + 3]) + if (itemSize >= 5) throw new Error('THREE.GLTFLoader: Unsupported itemSize in sparse BufferAttribute.') + } + } + + result.set(bufferAttribute) + }, [bufferView, sparseBufferViewIndices, sparseBufferViewValues]) + + return result.get(NO_PROXY) +} + +const useLoadBufferView = (options: GLTFParserOptions, bufferViewIndex?: number) => { + const result = useHookstate(null) + + const [bufferIndex, callback] = getBufferIndex(options, bufferViewIndex) + + const buffer = GLTFLoaderFunctions.useLoadBuffer(options, bufferIndex) + + useEffect(() => { + if (!buffer) return result.set(null) + callback(buffer).then((buffer) => result.set(buffer)) + }, [buffer]) + + return result.get(NO_PROXY) as ArrayBuffer | null +} + +const useLoadBuffer = (options: GLTFParserOptions, bufferIndex) => { + const json = options.document + const result = useHookstate(null) + + const bufferDef = typeof bufferIndex === 'number' ? json.buffers![bufferIndex] : null + + useEffect(() => { + if (!bufferDef) return + + if (bufferDef.type && bufferDef.type !== 'arraybuffer') { + console.warn('THREE.GLTFLoader: ' + bufferDef.type + ' buffer type is not supported.') + } + }, [bufferDef?.type]) + + useEffect(() => { + if (!bufferDef) return + + // If present, GLB container is required to be the first buffer. + if (bufferDef.uri === undefined && bufferIndex === 0) { + result.set(options.body!) + return + } + + /** @todo use resource hooks */ + const fileLoader = new FileLoader(options.manager) + fileLoader.setResponseType('arraybuffer') + if (options.crossOrigin === 'use-credentials') { + fileLoader.setWithCredentials(true) + } + + fileLoader.load( + LoaderUtils.resolveURL(bufferDef.uri!, options.path), + (val: ArrayBuffer) => { + result.set(val) + }, + undefined, + function () { + result.set(null) + console.error(new Error('THREE.GLTFLoader: Failed to load buffer "' + bufferDef.uri + '".')) + } + ) + }, [bufferDef?.uri]) + + return result.get(NO_PROXY) as ArrayBuffer | null +} + +export function computeBounds(json: GLTF.IGLTF, geometry: BufferGeometry, primitiveDef: GLTF.IMeshPrimitive) { + const attributes = primitiveDef.attributes + + const box = new Box3() + + if (attributes.POSITION !== undefined) { + const accessor = json.accessors![attributes.POSITION] + + const min = accessor.min + const max = accessor.max + + // glTF requires 'min' and 'max', but VRM (which extends glTF) currently ignores that requirement. + + if (min !== undefined && max !== undefined) { + box.set(new Vector3(min[0], min[1], min[2]), new Vector3(max[0], max[1], max[2])) + + if (accessor.normalized) { + const boxScale = getNormalizedComponentScale(WEBGL_COMPONENT_TYPES[accessor.componentType]) + box.min.multiplyScalar(boxScale) + box.max.multiplyScalar(boxScale) + } + } else { + console.warn('THREE.GLTFLoader: Missing min/max properties for accessor POSITION.') + + return + } + } else { + return + } + + const targets = primitiveDef.targets + + if (targets !== undefined) { + const maxDisplacement = new Vector3() + const vector = new Vector3() + + for (let i = 0, il = targets.length; i < il; i++) { + const target = targets[i] + + if (target.POSITION !== undefined) { + const accessor = json.accessors![target.POSITION] + const min = accessor.min + const max = accessor.max + + // glTF requires 'min' and 'max', but VRM (which extends glTF) currently ignores that requirement. + + if (min !== undefined && max !== undefined) { + // we need to get max of absolute components because target weight is [-1,1] + vector.setX(Math.max(Math.abs(min[0]), Math.abs(max[0]))) + vector.setY(Math.max(Math.abs(min[1]), Math.abs(max[1]))) + vector.setZ(Math.max(Math.abs(min[2]), Math.abs(max[2]))) + + if (accessor.normalized) { + const boxScale = getNormalizedComponentScale(WEBGL_COMPONENT_TYPES[accessor.componentType]) + vector.multiplyScalar(boxScale) + } + + // Note: this assumes that the sum of all weights is at most 1. This isn't quite correct - it's more conservative + // to assume that each target can have a max weight of 1. However, for some use cases - notably, when morph targets + // are used to implement key-frame animations and as such only two are active at a time - this results in very large + // boxes. So for now we make a box that's sometimes a touch too small but is hopefully mostly of reasonable size. + maxDisplacement.max(vector) + } else { + console.warn('THREE.GLTFLoader: Missing min/max properties for accessor POSITION.') + } + } + } + + // As per comment above this box isn't conservative, but has a reasonable size for a very large number of morph targets. + box.expandByVector(maxDisplacement) + } + + geometry.boundingBox = box + + const sphere = new Sphere() + + box.getCenter(sphere.center) + sphere.radius = box.min.distanceTo(box.max) / 2 + + geometry.boundingSphere = sphere +} + +/** + * Specification: https://github.com/KhronosGroup/glTF/blob/master/specification/2.0/README.md#materials + * @param {number} materialIndex + * @return {Promise} + */ +const useLoadMaterial = ( + options: GLTFParserOptions, + materialDef: ComponentType +) => { + const result = useHookstate(null as null | MeshStandardMaterial | MeshBasicMaterial) + + useEffect(() => { + /** @todo refactor this into a proper registry, rather than prototype definition entities */ + const materialPrototypeEntity = NameComponent.entitiesByName[materialDef.type]?.[0] + const materialPrototype = materialPrototypeEntity + ? (getComponent(materialPrototypeEntity, MaterialPrototypeComponent).prototypeConstructor as any)[ + materialDef.type + ] + : null + const materialConstructor = materialPrototype ?? MeshStandardMaterial + const material = new materialConstructor() + + result.set(material) + + assignExtrasToUserData(material, materialDef) + + /** @todo */ + // parser.associations.set(material, { materials: materialIndex }) + + // if (materialDef.extensions) addUnknownExtensionsToUserData(GLTFExtensions, material, materialDef) + + result.set(material) + }, [materialDef.type]) + + const material = result.get(NO_PROXY) as MeshStandardMaterial | MeshBasicMaterial + const map = GLTFLoaderFunctions.useAssignTexture(options, materialDef.pbrMetallicRoughness?.baseColorTexture) + + useEffect(() => { + if (!map) return + map.colorSpace = SRGBColorSpace + result.value?.setValues({ map }) + if (material) material.needsUpdate = true + }, [material, map]) + + useEffect(() => { + if (Array.isArray(materialDef.pbrMetallicRoughness?.baseColorFactor)) { + const array = materialDef.pbrMetallicRoughness.baseColorFactor + result.value?.setValues({ + color: new Color().setRGB(array[0], array[1], array[2], LinearSRGBColorSpace), + opacity: array[3] + }) + if (material) material.needsUpdate = true + } + }, [material, materialDef.pbrMetallicRoughness?.baseColorFactor]) + + useEffect(() => { + result.value?.setValues({ + metalness: + materialDef.pbrMetallicRoughness?.metallicFactor !== undefined + ? materialDef.pbrMetallicRoughness.metallicFactor + : 1.0 + }) + if (material) material.needsUpdate = true + }, [material, materialDef.pbrMetallicRoughness?.metallicFactor]) + + useEffect(() => { + result.value?.setValues({ + roughness: + materialDef.pbrMetallicRoughness?.roughnessFactor !== undefined + ? materialDef.pbrMetallicRoughness.roughnessFactor + : 1.0 + }) + if (material) material.needsUpdate = true + }, [material, materialDef.pbrMetallicRoughness?.roughnessFactor]) + + const metalnessMap = GLTFLoaderFunctions.useAssignTexture( + options, + materialDef.type === 'MeshBasicMaterial' ? undefined : materialDef.pbrMetallicRoughness?.metallicRoughnessTexture + ) + + useEffect(() => { + if (!metalnessMap) return + result.value?.setValues({ metalnessMap }) + if (material) material.needsUpdate = true + }, [material, metalnessMap]) + + const roughnessMap = GLTFLoaderFunctions.useAssignTexture( + options, + materialDef.type === 'MeshBasicMaterial' ? undefined : materialDef.pbrMetallicRoughness?.metallicRoughnessTexture + ) + + useEffect(() => { + if (!roughnessMap) return + result.value?.setValues({ roughnessMap }) + if (material) material.needsUpdate = true + }, [material, roughnessMap]) + + useEffect(() => { + result.value?.setValues({ side: materialDef.doubleSided === true ? DoubleSide : FrontSide }) + if (material) material.needsUpdate = true + }, [material, materialDef.doubleSided]) + + useEffect(() => { + const alphaMode = materialDef.alphaMode || ALPHA_MODES.OPAQUE + result.value?.setValues({ transparent: alphaMode === ALPHA_MODES.BLEND }) + + // See: https://github.com/mrdoob/three.js/issues/17706 + if (alphaMode === ALPHA_MODES.BLEND) { + result.value?.setValues({ depthWrite: false }) + } + if (material) material.needsUpdate = true + }, [material, materialDef.alphaMode]) + + useEffect(() => { + if (materialDef.alphaMode === ALPHA_MODES.MASK) { + result.value?.setValues({ alphaTest: materialDef.alphaCutoff !== undefined ? materialDef.alphaCutoff : 0.5 }) + } else { + result.value?.setValues({ alphaTest: 0 }) + } + if (material) material.needsUpdate = true + }, [material, materialDef.alphaCutoff]) + + const normalMap = GLTFLoaderFunctions.useAssignTexture( + options, + materialDef.type === 'MeshBasicMaterial' ? undefined : materialDef.normalTexture + ) + + useEffect(() => { + if (!normalMap) return + result.value?.setValues({ normalMap }) + if (material) material.needsUpdate = true + }, [material, normalMap]) + + useEffect(() => { + if (materialDef.normalTexture?.scale) { + const scale = materialDef.normalTexture.scale + result.value?.setValues({ normalScale: new Vector2(scale, scale) }) + } else { + result.value?.setValues({ normalScale: new Vector2(1, 1) }) + } + if (material) material.needsUpdate = true + }, [material, materialDef.normalTexture?.scale]) + + const aoMap = GLTFLoaderFunctions.useAssignTexture( + options, + materialDef.type === 'MeshBasicMaterial' ? undefined : materialDef.occlusionTexture + ) + + useEffect(() => { + if (!aoMap) return + result.value?.setValues({ aoMap }) + if (material) material.needsUpdate = true + }, [material, aoMap]) + + useEffect(() => { + result.value?.setValues({ aoMapIntensity: materialDef.occlusionTexture?.strength ?? 1.0 }) + if (material) material.needsUpdate = true + }, [material, materialDef.occlusionTexture?.strength]) + + useEffect(() => { + const emissiveFactor = materialDef.emissiveFactor + if (!emissiveFactor) return + + result.value?.setValues({ + emissive: new Color().setRGB(emissiveFactor[0], emissiveFactor[1], emissiveFactor[2], LinearSRGBColorSpace) + }) + if (material) material.needsUpdate = true + }, [material, materialDef.emissiveFactor]) + + const emissiveMap = GLTFLoaderFunctions.useAssignTexture( + options, + materialDef.type === 'MeshBasicMaterial' ? undefined : materialDef.emissiveTexture + ) + + useEffect(() => { + if (!emissiveMap) return + emissiveMap.colorSpace = SRGBColorSpace + result.value?.setValues({ emissiveMap }) + if (material) material.needsUpdate = true + }, [material, emissiveMap]) + + return result.get(NO_PROXY) as MeshStandardMaterial | null +} + +const useLoadMorphTargets = (options: GLTFParserOptions, targetsList: Record[]) => { + const result = useHookstate(null as null | Record) + + useEffect(() => { + /** @todo make individual targets individually reactive */ + const reactor = startReactor(() => { + const targetState = useHookstate( + () => + targetsList.map((target) => Object.fromEntries(Object.entries(target).map(([key]) => [key, null]))) as Record< + string, + BufferAttribute | null + >[] + ) + + for (let i = 0, il = targetsList.length; i < il; i++) { + const target = targetsList[i] + for (const [key, accessorIndex] of Object.entries(target)) { + const accessor = GLTFLoaderFunctions.useLoadAccessor(options, accessorIndex) + useEffect(() => { + if (!accessor) return + targetState[i][key].set(accessor) + }, [accessor]) + } + } + + useEffect(() => { + for (const target of targetState.value) { + if (Object.values(target).includes(null)) return + } + result.set( + targetState.get(NO_PROXY).reduce( + (acc, target: Record) => { + for (const [key, value] of Object.entries(target)) { + if (!acc[key]) acc[key] = [] + acc[key].push(value) + } + return acc + }, + {} as Record + ) + ) + reactor.stop() + }, [targetState]) + + return null + }) + return () => { + reactor.stop() + } + }, [targetsList]) + + return result.get(NO_PROXY) as Record | null +} + +/** + * Asynchronously assigns a texture to the given material parameters. + * @param {Object} materialParams + * @param {string} mapName + * @param {Object} mapDef + * @return {Promise} + */ +const useAssignTexture = (options: GLTFParserOptions, mapDef?: GLTF.ITextureInfo) => { + const result = useHookstate(null) + + const texture = GLTFLoaderFunctions.useLoadTexture(options, mapDef?.index) + + useEffect(() => { + if (!texture) { + result.set(null) + return + } + + if (!mapDef) return + + if (mapDef.texCoord !== undefined && mapDef.texCoord > 0) { + texture.channel = mapDef.texCoord + } + + /** @todo properly support extensions */ + const transform = + mapDef.extensions !== undefined ? mapDef.extensions[KHRTextureTransformExtensionComponent.jsonID] : undefined + + if (transform) { + // const gltfReference = parser.associations.get(texture) + const extendedTexture = KHRTextureTransformExtensionComponent.extendTexture(texture, transform) + // parser.associations.set(texture, gltfReference) + result.set(extendedTexture) + } else { + result.set(texture) + } + }, [texture, mapDef]) + + return result.get(NO_PROXY) as Texture | null +} + +let isSafari = false +let isFirefox = false +let firefoxVersion = -1 as any // ??? + +if (typeof navigator !== 'undefined') { + isSafari = /^((?!chrome|android).)*safari/i.test(navigator.userAgent) === true + isFirefox = navigator.userAgent.indexOf('Firefox') > -1 + firefoxVersion = isFirefox ? navigator.userAgent.match(/Firefox\/([0-9]+)\./)![1] : -1 +} + +let textureLoader: TextureLoader | ImageBitmapLoader + +/** @todo use resource loader hooks */ +if (typeof createImageBitmap === 'undefined' || isSafari || (isFirefox && firefoxVersion < 98)) { + textureLoader = new TextureLoader() +} else { + textureLoader = new ImageBitmapLoader() +} + +type KHRTextureBasisu = { + source: number +} + +/** + * Specification: https://github.com/KhronosGroup/glTF/tree/master/specification/2.0#textures + * @param {number} textureIndex + * @return {Promise} + */ +const useLoadTexture = (options: GLTFParserOptions, textureIndex?: number) => { + const json = options.document + + const textureDef = typeof textureIndex === 'number' ? json.textures![textureIndex] : null + + const extensions = textureDef?.extensions as Record> | null + const basisu = extensions && (extensions[EXTENSIONS.KHR_TEXTURE_BASISU] as KHRTextureBasisu) + + /** @todo properly support texture extensions, this is a hack */ + const sourceIndex = + (extensions && Object.values(extensions).find((ext) => typeof ext.source === 'number')?.source) ?? + textureDef?.source! + const sourceDef = typeof sourceIndex === 'number' ? json.images![sourceIndex] : null + + const handler = typeof sourceDef?.uri === 'string' && options.manager.getHandler(sourceDef.uri) + let loader: ImageLoader | ImageBitmapLoader | TextureLoader | KTX2Loader | Loader + + if (handler) loader = handler + if (basisu) loader = getState(AssetLoaderState).gltfLoader.ktx2Loader! + else { + loader = textureLoader + loader.setCrossOrigin(options.crossOrigin) + loader.setRequestHeader(options.requestHeader) + } + + const texture = GLTFLoaderFunctions.useLoadTextureImage(options, textureIndex, sourceIndex, loader) + + return texture +} + +const useLoadTextureImage = ( + options: GLTFParserOptions, + textureIndex?: number, + sourceIndex?: number, + loader?: ImageLoader | ImageBitmapLoader | TextureLoader | KTX2Loader | Loader +) => { + const json = options.document + const result = useHookstate(null) + + const textureDef = typeof textureIndex === 'number' ? json.textures![textureIndex] : null + const sourceDef = typeof sourceIndex === 'number' ? json.images![sourceIndex] : null + + /** @todo cache */ + // const cacheKey = (sourceDef.uri || sourceDef.bufferView) + ':' + textureDef.sampler + + // if (textureCache[cacheKey]) { + // // See https://github.com/mrdoob/three.js/issues/21559. + // return textureCache[cacheKey] + // } + + const texture = GLTFLoaderFunctions.useLoadImageSource(options, sourceIndex, loader) + + useEffect(() => { + if (!texture || !sourceDef || !textureDef) return + + texture.flipY = false + + texture.name = textureDef.name || sourceDef.name || '' + + if (texture.name === '' && typeof sourceDef.uri === 'string' && sourceDef.uri.startsWith('data:image/') === false) { + texture.name = sourceDef.uri + } + + const samplers = json.samplers || {} + const sampler = samplers[textureDef.sampler!] || {} + + texture.magFilter = WEBGL_FILTERS[sampler.magFilter] || LinearFilter + texture.minFilter = WEBGL_FILTERS[sampler.minFilter] || LinearMipmapLinearFilter + texture.wrapS = WEBGL_WRAPPINGS[sampler.wrapS] || RepeatWrapping + texture.wrapT = WEBGL_WRAPPINGS[sampler.wrapT] || RepeatWrapping + + /** @todo */ + // parser.associations.set(texture, { textures: textureIndex }) + + result.set(texture) + }, [textureDef, sourceDef, texture]) + + // textureCache[cacheKey] = promise + + return result.get(NO_PROXY) as Texture | null +} + +// const sourceCache = {} as any // todo + +const URL = self.URL || self.webkitURL + +const useLoadImageSource = ( + options: GLTFParserOptions, + sourceIndex?: number, + loader?: ImageLoader | ImageBitmapLoader | TextureLoader | KTX2Loader | Loader +) => { + const json = options.document + const result = useHookstate(null) + + /** @todo caching */ + // if (sourceCache[sourceIndex] !== undefined) { + // return sourceCache[sourceIndex].then((texture) => texture.clone()) + // } + + const sourceDef = typeof sourceIndex === 'number' ? json.images![sourceIndex] : null + + const sourceURI = useHookstate(null as null | string) + let isObjectURL = false + + const bufferViewSourceURI = GLTFLoaderFunctions.useLoadBufferView(options, sourceDef?.bufferView) + + useEffect(() => { + if (!sourceDef) return + + if (sourceDef.uri === undefined && sourceDef.bufferView === undefined) { + console.error('THREE.GLTFLoader: Image ' + sourceIndex + ' is missing URI and bufferView') + return + } + + if (sourceDef.uri) { + const url = LoaderUtils.resolveURL(sourceDef.uri, options.path) + sourceURI.set(url) + return () => { + sourceURI.set(null) + } + } + + if (bufferViewSourceURI) { + isObjectURL = true + const blob = new Blob([bufferViewSourceURI], { type: sourceDef.mimeType }) + const url = URL.createObjectURL(blob) + sourceURI.set(url) + return () => { + URL.revokeObjectURL(url) + sourceURI.set(null) + } + } + }, [sourceDef?.uri, bufferViewSourceURI]) + + useEffect(() => { + if (!sourceURI.value) return + loader!.load( + LoaderUtils.resolveURL(sourceURI.value as string, options.path), + (imageBitmap: Texture | ImageBitmap) => { + if ((loader as ImageBitmapLoader).isImageBitmapLoader === true) { + const texture = new Texture(imageBitmap as ImageBitmap) + texture.needsUpdate = true + result.set(texture) + } else { + result.set(imageBitmap as Texture) + } + }, + undefined, + (e) => { + console.error(e) + console.error("THREE.GLTFLoader: Couldn't load image", sourceURI.value) + } + ) + }, [sourceURI.value]) + + useEffect(() => { + if (!result.value || !sourceURI.value || !sourceDef) return + + const texture = result.value as Texture + + // Clean up resources and configure Texture. + + if (isObjectURL === true) { + URL.revokeObjectURL(sourceURI.value!) + } else { + texture.userData.src = sourceURI + } + + texture.userData.mimeType = sourceDef.mimeType || getImageURIMimeType(sourceDef.uri) + + // sourceCache[sourceIndex] = promise + }, [result.value]) + + return result.get(NO_PROXY) as Texture | null +} + +const getNodeUUID = (node: GLTF.INode, documentID: string, nodeIndex: number) => + (node.extensions?.[UUIDComponent.jsonID] as EntityUUID) ?? (`${documentID}-${nodeIndex}` as EntityUUID) + +const useLoadAnimation = (options: GLTFParserOptions, animationIndex?: number) => { + const result = useHookstate(null as null | AnimationClip) + + const json = options.document + + const animationDef = typeof animationIndex === 'number' ? json.animations![animationIndex] : null + const animationName = animationDef ? (animationDef.name ? animationDef.name : 'animation_' + animationIndex) : null + + useEffect(() => { + if (!animationDef || !animationName) return + + const channels = animationDef.channels.filter((channel) => channel.target.node !== undefined) + + const reactor = startReactor(() => { + const channelData = useHookstate(() => + Object.fromEntries( + channels.map((channel, i) => [ + i, + { + nodes: null as null | Mesh | Bone | Object3D, + inputAccessors: null as null | BufferAttribute, + outputAccessors: null as null | BufferAttribute, + samplers: animationDef.samplers[channel.sampler], + targets: channel.target + } + ]) + ) + ) + + for (let i = 0, il = channels.length; i < il; i++) { + const channel = channels[i] + const sampler = animationDef.samplers[channel.sampler] + const target = channel.target + const nodeIndex = target.node! + const input = animationDef.parameters !== undefined ? animationDef.parameters[sampler.input] : sampler.input + const output = animationDef.parameters !== undefined ? animationDef.parameters[sampler.output] : sampler.output + const node = json.nodes![nodeIndex] + const mesh = typeof node.mesh === 'number' ? json.meshes?.[node.mesh!] : null + const meshHasWeights = mesh?.weights !== undefined && mesh.weights.length > 0 + + const targetNodeUUID = getNodeUUID(json.nodes![nodeIndex], options.documentID, nodeIndex) + const targetNodeEntity = UUIDComponent.useEntityByUUID(targetNodeUUID) + + /** @todo we should probably jsut use GroupComponent or something here once we stop creating Object3Ds for all nodes */ + const meshComponent = useOptionalComponent(targetNodeEntity, MeshComponent) + const boneComponent = useOptionalComponent(targetNodeEntity, BoneComponent) + + useEffect(() => { + const meshWeightsLoaded = meshHasWeights + ? meshComponent?.get(NO_PROXY)?.morphTargetInfluences !== undefined + : true + if (!meshWeightsLoaded && !boneComponent) return + channelData[i].nodes.set( + getOptionalComponent(targetNodeEntity, MeshComponent) ?? + getOptionalComponent(targetNodeEntity, BoneComponent)! + ) + }, [meshComponent, boneComponent]) + + const inputAccessor = GLTFLoaderFunctions.useLoadAccessor(options, input) + + useEffect(() => { + if (!inputAccessor) return + channelData[i].inputAccessors.set(inputAccessor) + }, [inputAccessor]) + + const outputAccessor = GLTFLoaderFunctions.useLoadAccessor(options, output) + + useEffect(() => { + if (!outputAccessor) return + channelData[i].outputAccessors.set(outputAccessor) + }, [outputAccessor]) + } + + useEffect(() => { + if ( + Object.values(channelData.get(NO_PROXY)).some( + (data) => !data.nodes || !data.inputAccessors || !data.outputAccessors + ) + ) + return + + const values = Object.values(channelData.get(NO_PROXY)) + const nodes = values.map((data) => data.nodes) + const inputAccessors = values.map((data) => data.inputAccessors) as BufferAttribute[] + const outputAccessors = values.map((data) => data.outputAccessors) as BufferAttribute[] + const samplers = values.map((data) => data.samplers) as GLTF.IAnimationSampler[] + const targets = values.map((data) => data.targets) as GLTF.IAnimationChannelTarget[] + + const tracks = [] as any[] // todo + + for (let i = 0, il = nodes.length; i < il; i++) { + const node = nodes[i] as Mesh | SkinnedMesh + const inputAccessor = inputAccessors[i] + const outputAccessor = outputAccessors[i] + const sampler = samplers[i] + const target = targets[i] + + if (node === undefined) continue + + if (node.updateMatrix) { + node.updateMatrix() + } + + const createdTracks = _createAnimationTracks(node, inputAccessor, outputAccessor, sampler, target) + + if (createdTracks) { + for (let k = 0; k < createdTracks.length; k++) { + tracks.push(createdTracks[k]) + } + } + } + + result.set(new AnimationClip(animationName, undefined, tracks)) + + reactor.stop() + }, [channelData]) + + return null + }) + return () => { + reactor.stop() + } + }, [animationDef]) + + return result.get(NO_PROXY) as AnimationClip | null +} + +const _createAnimationTracks = ( + node: Mesh | SkinnedMesh, + inputAccessor: BufferAttribute, + outputAccessor: BufferAttribute, + sampler: GLTF.IAnimationSampler, + target: GLTF.IAnimationChannelTarget +) => { + const tracks = [] as any[] // todo + + const targetName = node.name + if (!targetName) throw new Error('THREE.GLTFLoader: Node has no name.') + const targetNames = [] as string[] + + if (PATH_PROPERTIES[target.path] === PATH_PROPERTIES.weights) { + node.traverse(function (object: Mesh | SkinnedMesh) { + if (object.morphTargetInfluences) { + if (!object.name) throw new Error('THREE.GLTFLoader: Node has no name.') + targetNames.push(object.name) + } + }) + } else { + targetNames.push(targetName) + } + + let TypedKeyframeTrack + + switch (PATH_PROPERTIES[target.path]) { + case PATH_PROPERTIES.weights: + TypedKeyframeTrack = NumberKeyframeTrack + break + + case PATH_PROPERTIES.rotation: + TypedKeyframeTrack = QuaternionKeyframeTrack + break + + case PATH_PROPERTIES.position: + case PATH_PROPERTIES.scale: + TypedKeyframeTrack = VectorKeyframeTrack + break + + default: + switch (outputAccessor.itemSize) { + case 1: + TypedKeyframeTrack = NumberKeyframeTrack + break + case 2: + case 3: + default: + TypedKeyframeTrack = VectorKeyframeTrack + break + } + + break + } + + const interpolation = sampler.interpolation !== undefined ? INTERPOLATION[sampler.interpolation] : InterpolateLinear + + const outputArray = _getArrayFromAccessor(outputAccessor) + + for (let j = 0, jl = targetNames.length; j < jl; j++) { + const track = new TypedKeyframeTrack( + targetNames[j] + '.' + PATH_PROPERTIES[target.path], + inputAccessor.array, + outputArray, + interpolation + ) + + // Override interpolation with custom factory method. + if (sampler.interpolation === 'CUBICSPLINE') { + _createCubicSplineTrackInterpolant(track) + } + + tracks.push(track) + } + + return tracks +} + +const _getArrayFromAccessor = (accessor: BufferAttribute) => { + let outputArray = accessor.array + + if (accessor.normalized) { + const scale = getNormalizedComponentScale(outputArray.constructor) + const scaled = new Float32Array(outputArray.length) + + for (let j = 0, jl = outputArray.length; j < jl; j++) { + scaled[j] = outputArray[j] * scale + } + + outputArray = scaled + } + + return outputArray +} + +const _createCubicSplineTrackInterpolant = (track: KeyframeTrack) => { + // @ts-ignore + track.createInterpolant = function InterpolantFactoryMethodGLTFCubicSpline(result) { + // A CUBICSPLINE keyframe in glTF has three output values for each input value, + // representing inTangent, splineVertex, and outTangent. As a result, track.getValueSize() + // must be divided by three to get the interpolant's sampleSize argument. + + const interpolantType = + this instanceof QuaternionKeyframeTrack ? GLTFCubicSplineQuaternionInterpolant : GLTFCubicSplineInterpolant + + return new interpolantType(this.times, this.values, this.getValueSize() / 3, result) + } + + // @ts-ignore Mark as CUBICSPLINE. `track.getInterpolation()` doesn't support custom interpolants. + track.createInterpolant.isInterpolantFactoryMethodGLTFCubicSpline = true +} + +export const GLTFLoaderFunctions = { + computeBounds, + useLoadPrimitive, + useLoadAccessor, + useLoadBufferView, + useLoadBuffer, + useLoadMaterial, + useLoadMorphTargets, + useAssignTexture, + useLoadTexture, + useLoadImageSource, + useLoadTextureImage, + useLoadAnimation +} diff --git a/packages/engine/src/gltf/GLTFState.tsx b/packages/engine/src/gltf/GLTFState.tsx index f02c52d949..e897e37272 100644 --- a/packages/engine/src/gltf/GLTFState.tsx +++ b/packages/engine/src/gltf/GLTFState.tsx @@ -25,7 +25,21 @@ Ethereal Engine. All Rights Reserved. import { GLTF } from '@gltf-transform/core' import React, { useEffect, useLayoutEffect } from 'react' -import { Group, MathUtils, Matrix4, Quaternion, Vector3 } from 'three' +import { + AnimationClip, + AnimationMixer, + Bone, + Group, + LoaderUtils, + MathUtils, + Matrix4, + Mesh, + MeshBasicMaterial, + Quaternion, + Skeleton, + SkinnedMesh, + Vector3 +} from 'three' import { staticResourcePath } from '@etherealengine/common/src/schema.type.module' import { @@ -34,13 +48,16 @@ import { Entity, EntityUUID, getComponent, + getMutableComponent, getOptionalComponent, + getOptionalMutableComponent, hasComponent, removeComponent, removeEntity, setComponent, UndefinedEntity, useComponent, + useOptionalComponent, UUIDComponent } from '@etherealengine/ecs' import { @@ -59,19 +76,32 @@ import { import { TransformComponent } from '@etherealengine/spatial' import { useGet } from '@etherealengine/spatial/src/common/functions/FeathersHooks' import { NameComponent } from '@etherealengine/spatial/src/common/NameComponent' -import { addObjectToGroup } from '@etherealengine/spatial/src/renderer/components/GroupComponent' +import { addObjectToGroup, removeObjectFromGroup } from '@etherealengine/spatial/src/renderer/components/GroupComponent' import { MeshComponent } from '@etherealengine/spatial/src/renderer/components/MeshComponent' import { Object3DComponent } from '@etherealengine/spatial/src/renderer/components/Object3DComponent' import { VisibleComponent } from '@etherealengine/spatial/src/renderer/components/VisibleComponent' -import { EntityTreeComponent } from '@etherealengine/spatial/src/transform/components/EntityTree' +import { + EntityTreeComponent, + getAncestorWithComponent +} from '@etherealengine/spatial/src/transform/components/EntityTree' +import { CameraComponent } from '@etherealengine/spatial/src/camera/components/CameraComponent' import { EngineState } from '@etherealengine/spatial/src/EngineState' import { Physics } from '@etherealengine/spatial/src/physics/classes/Physics' +import { BoneComponent } from '@etherealengine/spatial/src/renderer/components/BoneComponent' import { SceneComponent } from '@etherealengine/spatial/src/renderer/components/SceneComponents' +import { SkinnedMeshComponent } from '@etherealengine/spatial/src/renderer/components/SkinnedMeshComponent' +import { MaterialInstanceComponent } from '@etherealengine/spatial/src/renderer/materials/MaterialComponent' +import { GLTFParserOptions } from '../assets/loaders/gltf/GLTFParser' +import { AssetLoaderState } from '../assets/state/AssetLoaderState' +import { AnimationComponent } from '../avatar/components/AnimationComponent' import { SourceComponent } from '../scene/components/SourceComponent' import { proxifyParentChildRelationships } from '../scene/functions/loadGLTFModel' import { GLTFComponent } from './GLTFComponent' import { GLTFDocumentState, GLTFModifiedState, GLTFNodeState, GLTFSnapshotAction } from './GLTFDocumentState' +import { GLTFLoaderFunctions } from './GLTFLoaderFunctions' +import { MaterialDefinitionComponent } from './MaterialDefinitionComponent' +import './MeshExtensionComponents' export const GLTFAssetState = defineState({ name: 'ee.engine.gltf.GLTFAssetState', @@ -120,13 +150,10 @@ export const GLTFSourceState = defineState({ setComponent(entity, Object3DComponent, obj3d) addObjectToGroup(entity, obj3d) proxifyParentChildRelationships(obj3d) - getMutableState(GLTFSourceState)[sourceID].set(entity) return entity }, unload: (entity: Entity) => { - const sourceID = `${getComponent(entity, UUIDComponent)}-${getComponent(entity, GLTFComponent).src}` - getMutableState(GLTFSourceState)[sourceID].set(none) removeEntity(entity) } }) @@ -357,6 +384,8 @@ const ChildGLTFReactor = (props: { source: string }) => { const source = props.source const index = useHookstate(getMutableState(GLTFSnapshotState)[source].index).value + const entity = useHookstate(getMutableState(GLTFSourceState)[source]).value + const parentUUID = useComponent(entity, UUIDComponent).value useLayoutEffect(() => { return () => { @@ -366,6 +395,7 @@ const ChildGLTFReactor = (props: { source: string }) => { }, []) useLayoutEffect(() => { + const index = getState(GLTFSnapshotState)[source].index // update the modified state if (index > 0) getMutableState(GLTFModifiedState)[source].set(true) @@ -374,19 +404,53 @@ const ChildGLTFReactor = (props: { source: string }) => { getMutableState(GLTFDocumentState)[source].set(data) // update the nodes dictionary - const nodesDictionary = GLTFNodeState.convertGltfToNodeDictionary(data) + const nodesDictionary = GLTFNodeState.convertGltfToNodeDictionary(data, source) getMutableState(GLTFNodeState)[source].set(nodesDictionary) }, [index]) - const entity = useHookstate(getMutableState(GLTFSourceState)[source]).value - const parentUUID = useComponent(entity, UUIDComponent).value + const nodeState = useHookstate(getMutableState(GLTFNodeState))[source] + const documentState = useMutableState(GLTFDocumentState)[source] + const physicsWorld = Physics.useWorld(entity) + + if (!physicsWorld || !documentState.value || !nodeState.value) return null return } export const DocumentReactor = (props: { documentID: string; parentUUID: EntityUUID }) => { - const nodeState = useHookstate(getMutableState(GLTFNodeState)[props.documentID]) - if (!nodeState.value) return null + const nodeState = useHookstate(getMutableState(GLTFNodeState))[props.documentID] + const documentState = useMutableState(GLTFDocumentState)[props.documentID] + const animationState = useHookstate([] as AnimationClip[]) + const rootEntity = UUIDComponent.useEntityByUUID(props.parentUUID) + + useEffect(() => { + if (animationState.length !== documentState.value?.animations?.length) return + + const hasObject3d = hasComponent(rootEntity, Object3DComponent) + if (!hasObject3d) { + /** @todo this is a temporary hack */ + const obj3d = new Group() + setComponent(rootEntity, Object3DComponent, obj3d) + addObjectToGroup(rootEntity, obj3d) + proxifyParentChildRelationships(obj3d) + } + + const scene = getComponent(rootEntity, Object3DComponent) + scene.animations = animationState.get(NO_PROXY) as AnimationClip[] + const mixer = new AnimationMixer(scene) + setComponent(rootEntity, AnimationComponent, { + mixer, + animations: scene.animations + }) + return () => { + removeComponent(rootEntity, AnimationComponent) + if (!hasObject3d) { + removeObjectFromGroup(rootEntity, getComponent(rootEntity, Object3DComponent)) + removeComponent(rootEntity, Object3DComponent) + } + } + }, [animationState]) + return ( <> {Object.entries(nodeState.get(NO_PROXY)).map(([uuid, { nodeIndex, childIndex, parentUUID }]) => ( @@ -398,10 +462,145 @@ export const DocumentReactor = (props: { documentID: string; parentUUID: EntityU documentID={props.documentID} /> ))} + {documentState + .get(NO_PROXY) + .materials?.map((material, index) => ( + + ))} + {documentState.get(NO_PROXY).animations?.map((animation, index) => { + return ( + + ) + })} ) } +const MaterialEntityReactor = (props: { index: number; parentUUID: EntityUUID; documentID: string }) => { + const documentState = useMutableState(GLTFDocumentState)[props.documentID] + const materials = documentState.materials! + + const materialDef = materials[props.index]!.get(NO_PROXY) as GLTF.IMaterial + + const parentEntity = UUIDComponent.useEntityByUUID(props.parentUUID) + + const entityState = useHookstate(UndefinedEntity) + const entity = entityState.value + + useEffect(() => { + const uuid = (props.documentID + '-material-' + props.index) as EntityUUID + const entity = UUIDComponent.getOrCreateEntityByUUID(uuid) + + setComponent(entity, UUIDComponent, uuid) + setComponent(entity, SourceComponent, props.documentID) + + /** Ensure all base components are added for synchronous mount */ + setComponent(entity, EntityTreeComponent, { parentEntity, childIndex: props.index }) + setComponent(entity, NameComponent, materialDef.name ?? 'Material-' + props.index) + + entityState.set(entity) + + return () => { + //check if entity is in some other document + if (hasComponent(entity, UUIDComponent)) { + const uuid = getComponent(entity, UUIDComponent) + const documents = getState(GLTFDocumentState) + for (const documentID in documents) { + const document = documents[documentID] + if (!document?.materials) continue + for (const material of document.materials) { + if (material.extensions?.[UUIDComponent.jsonID] === uuid) return + } + } + } + removeEntity(entity) + } + }, []) + + useLayoutEffect(() => { + if (!entity) return + + setComponent(entity, EntityTreeComponent, { parentEntity, childIndex: props.index }) + }, [entity, parentEntity, props.index]) + + useLayoutEffect(() => { + if (!entity) return + + setComponent(entity, NameComponent, materialDef.name ?? 'Material-' + props.index) + }, [entity, materialDef.name]) + + if (!entity) return null + return ( + <> + + + ) +} + +const MaterialStateReactor = (props: { index: number; parentUUID: EntityUUID; documentID: string; entity: Entity }) => { + const documentState = useMutableState(GLTFDocumentState)[props.documentID] + const entity = props.entity + + const materialDefinition = documentState.materials![props.index]!.get(NO_PROXY) as GLTF.IMaterial + + useLayoutEffect(() => { + if (!materialDefinition) return + return () => { + removeComponent(entity, MaterialDefinitionComponent) + } + }, []) + + useLayoutEffect(() => { + if (!materialDefinition) return + setComponent(entity, MaterialDefinitionComponent, materialDefinition) + }, [materialDefinition]) + + return ( + <> + {materialDefinition.extensions && + Object.entries(materialDefinition.extensions).map(([id, ext]) => { + return + })} + + ) +} + +const MaterialStateExtensionReactor = (props: { keyName: string; value: any; entity: Entity }) => { + const entity = props.entity + + useEffect(() => { + const Component = ComponentJSONIDMap.get(props.keyName) + if (!Component) return console.warn('No component found for extension', props.keyName) + return () => { + removeComponent(entity, Component) + } + }, []) + + useEffect(() => { + const Component = ComponentJSONIDMap.get(props.keyName) + if (!Component) return console.warn('No component found for extension', props.keyName) + setComponent(entity, Component, props.value) + }, [props.keyName, props.value]) + + return null +} + const ParentNodeReactor = (props: { nodeIndex: number childIndex: number @@ -409,22 +608,38 @@ const ParentNodeReactor = (props: { documentID: string }) => { const parentEntity = UUIDComponent.useEntityByUUID(props.parentUUID) - const physicsWorld = Physics.useWorld(parentEntity) - if (!parentEntity || !physicsWorld) return null + if (!parentEntity) return null return } +const getNodeUUID = (node: GLTF.INode, documentID: string, nodeIndex: number) => + (node.extensions?.[UUIDComponent.jsonID] as EntityUUID) ?? (`${documentID}-${nodeIndex}` as EntityUUID) + +const isBoneNode = (json: GLTF.IGLTF, nodeIndex: number) => { + const skinDefs = json.skins || [] + for (let skinIndex = 0, skinLength = skinDefs.length; skinIndex < skinLength; skinIndex++) { + const joints = skinDefs[skinIndex].joints + for (let i = 0, il = joints.length; i < il; i++) { + if (joints[i] === nodeIndex) return true + } + } + return false +} + const NodeReactor = (props: { nodeIndex: number; childIndex: number; parentUUID: EntityUUID; documentID: string }) => { const documentState = useMutableState(GLTFDocumentState)[props.documentID] - const nodes = documentState.nodes! // as State + const nodes = documentState.nodes! const node = nodes[props.nodeIndex]! as State const parentEntity = UUIDComponent.useEntityByUUID(props.parentUUID) - const entity = useHookstate(() => { - const uuid = node.extensions.value?.[UUIDComponent.jsonID] as EntityUUID + const entityState = useHookstate(UndefinedEntity) + const entity = entityState.value + + useLayoutEffect(() => { + const uuid = getNodeUUID(node.get(NO_PROXY) as GLTF.IGLTF, props.documentID, props.nodeIndex) const entity = UUIDComponent.getOrCreateEntityByUUID(uuid) setComponent(entity, UUIDComponent, uuid) @@ -444,6 +659,9 @@ const NodeReactor = (props: { nodeIndex: number; childIndex: number; parentUUID: setComponent(entity, TransformComponent, { position, rotation, scale }) } + /** Always set visible extension if this is not an ECS node */ + if (!node.extensions.value?.[UUIDComponent.jsonID]) setComponent(entity, VisibleComponent) + // add all extensions for synchronous mount if (node.extensions.value) { for (const extension in node.extensions.value) { @@ -454,17 +672,18 @@ const NodeReactor = (props: { nodeIndex: number; childIndex: number; parentUUID: } if (!hasComponent(entity, Object3DComponent) && !hasComponent(entity, MeshComponent)) { - const obj3d = new Group() - obj3d.entity = entity - addObjectToGroup(entity, obj3d) - proxifyParentChildRelationships(obj3d) - setComponent(entity, Object3DComponent, obj3d) + if (isBoneNode(documentState.get(NO_PROXY) as GLTF.IGLTF, props.nodeIndex)) { + const bone = new Bone() + bone.name = node.name.value ?? 'Bone-' + props.nodeIndex + setComponent(entity, BoneComponent, bone) + addObjectToGroup(entity, bone) + proxifyParentChildRelationships(bone) + setComponent(entity, Object3DComponent, bone) + } } - return entity - }).value + entityState.set(entity) - useEffect(() => { return () => { //check if entity is in some other document if (hasComponent(entity, UUIDComponent)) { @@ -508,13 +727,43 @@ const NodeReactor = (props: { nodeIndex: number; childIndex: number; parentUUID: setComponent(entity, TransformComponent, { position, rotation, scale }) }, [entity, node.matrix]) + useLayoutEffect(() => { + if (!entity) return + + if (!node.translation.value) return + const position = new Vector3().fromArray(node.translation.value) + setComponent(entity, TransformComponent, { position }) + }, [entity, node.translation]) + + useLayoutEffect(() => { + if (!entity) return + + if (!node.rotation.value) return + const rotation = new Quaternion().fromArray(node.rotation.value) + setComponent(entity, TransformComponent, { rotation }) + }, [entity, node.rotation]) + + useLayoutEffect(() => { + if (!entity) return + + if (!node.scale.value) return + const scale = new Vector3().fromArray(node.scale.value) + setComponent(entity, TransformComponent, { scale }) + }, [entity, node.scale]) + if (!entity) return null return ( <> - {/* {node.mesh.value && ( + {typeof node.mesh.get(NO_PROXY) === 'number' && ( - )} */} + )} + {typeof node.skin.get(NO_PROXY) === 'number' && ( + + )} + {typeof node.camera.get(NO_PROXY) === 'number' && ( + + )} {node.extensions.value && Object.keys(node.extensions.get(NO_PROXY)!).map((extension) => ( { const documentState = useMutableState(GLTFDocumentState)[props.documentID] - const nodes = documentState.nodes! // as State + const nodes = documentState.nodes!.get(NO_PROXY)! const node = nodes[props.nodeIndex]! const extension = node.extensions![props.extension] @@ -558,49 +807,390 @@ const ExtensionReactor = (props: { entity: Entity; extension: string; nodeIndex: useLayoutEffect(() => { const Component = ComponentJSONIDMap.get(props.extension) if (!Component) return console.warn('no component found for extension', props.extension) - setComponent(props.entity, Component, extension.get(NO_PROXY_STEALTH)) + setComponent(props.entity, Component, extension) }, [extension]) return null } -// const MeshReactor = (props: { nodeIndex: number; documentID: string; entity: Entity }) => { -// const documentState = useHookstate(getMutableState(GLTFDocumentState)[props.documentID]) -// const nodes = documentState.nodes! as State -// const node = nodes[props.nodeIndex]! - -// const mesh = documentState.meshes![node.mesh.value!] as State - -// return ( -// <> -// {mesh.primitives.value.map((primitive, index) => ( -// -// ))} -// -// ) -// } - -// const PrimitiveReactor = (props: { primitiveIndex: number; nodeIndex: number; documentID: string; entity: Entity }) => { -// const documentState = useHookstate(getMutableState(GLTFDocumentState)[props.documentID]) -// const nodes = documentState.nodes! as State -// const node = nodes[props.nodeIndex]! - -// const primitive = documentState.meshes![node.mesh.value!].primitives[props.primitiveIndex] - -// useEffect(() => { -// /** TODO implement all mesh types */ -// }, [primitive]) - -// return null -// } - -/** - * TODO figure out how to support extensions that change the behaviour of these reactors - * - we pretty much have to add a new API for each dependency type, like how the GLTFLoader does - */ +const MeshReactor = (props: { nodeIndex: number; documentID: string; entity: Entity }) => { + const documentState = useMutableState(GLTFDocumentState)[props.documentID] + const nodes = documentState.nodes!.get(NO_PROXY)! + const node = nodes[props.nodeIndex]! + + const mesh = documentState.meshes.get(NO_PROXY)![node.mesh!] + + useEffect(() => { + setComponent(props.entity, VisibleComponent) + }, []) + + const isSinglePrimitive = mesh.primitives.length === 1 + + return ( + <> + {mesh.primitives.map((primitive, index) => ( + + ))} + + ) +} + +const SkinnedMeshReactor = (props: { nodeIndex: number; documentID: string; entity: Entity }) => { + const documentState = useHookstate(getMutableState(GLTFDocumentState)[props.documentID]) + const nodes = documentState.nodes!.get(NO_PROXY)! + const node = nodes[props.nodeIndex]! + + const skinnedMeshComponent = useOptionalComponent(props.entity, SkinnedMeshComponent) + + const skin = documentState.skins.get(NO_PROXY)![node.skin!] + + const options = getParserOptions(props.entity) + const inverseBindMatrices = GLTFLoaderFunctions.useLoadAccessor(options, skin.inverseBindMatrices) + + const jointNodeUUIDs = skin.joints.map((joint) => + getNodeUUID(nodes[joint] as GLTF.INode, props.documentID, joint) + ) as EntityUUID[] + /** @todo make reactive to edits */ + const jointEntityLoadedState = useHookstate(() => + Object.fromEntries(jointNodeUUIDs.map((uuid) => [uuid, UndefinedEntity])) + ) + + useEffect(() => { + if (!inverseBindMatrices || !skinnedMeshComponent) return + + const jointEntities = Object.values(jointEntityLoadedState.value) + if (jointEntities.includes(UndefinedEntity)) return + + const jointBones = jointEntities.map((entity) => getOptionalComponent(entity, BoneComponent)) + if (jointBones.includes(undefined)) return + + const bones = [] as Bone[] + const boneInverses = [] as Matrix4[] + + for (let i = 0, il = jointBones.length; i < il; i++) { + const jointNode = jointBones[i] + + if (jointNode) { + bones.push(jointNode) + + const mat = new Matrix4() + + if (inverseBindMatrices !== null) { + mat.fromArray(inverseBindMatrices.array, i * 16) + } + + boneInverses.push(mat) + } else { + // console.warn('THREE.GLTFLoader: Joint "%s" could not be found.', skinDef.joints[i]) + } + } + + const skeleton = new Skeleton(bones, boneInverses) + skinnedMeshComponent.skeleton.set(skeleton) + }, [jointEntityLoadedState, inverseBindMatrices, !!skinnedMeshComponent]) + + return ( + <> + {jointEntityLoadedState.keys.map((entityUUID: EntityUUID) => ( + + ))} + + ) +} + +/** @todo we can probably simplify this into a nested reactor in a 'useNodesLoaded' entity */ +const SkeletonNodeDependencyReactor = (props: { + entityUUID: EntityUUID + jointEntityLoadedState: State> +}) => { + const entity = UUIDComponent.useEntityByUUID(props.entityUUID) + const jointEntityLoadedState = props.jointEntityLoadedState + + useEffect(() => { + if (!entity) return + jointEntityLoadedState[props.entityUUID].set(entity) + return () => jointEntityLoadedState[props.entityUUID].set(UndefinedEntity) + }, [entity]) + + return null +} + +const CameraReactor = (props: { nodeIndex: number; documentID: string; entity: Entity }) => { + const documentState = useHookstate(getMutableState(GLTFDocumentState)[props.documentID]) + const nodes = documentState.nodes!.get(NO_PROXY)! + const node = nodes[props.nodeIndex]! + + const camera = documentState.cameras.get(NO_PROXY)![node.camera!] as GLTF.ICamera + + useEffect(() => { + if (camera.type === 'orthographic' || !camera.perspective) + return console.warn('Orthographic cameras not supported yet') + + const perspectiveCamera = camera.perspective + + setComponent(props.entity, CameraComponent, { + fov: MathUtils.radToDeg(perspectiveCamera.yfov), + aspect: perspectiveCamera.aspectRatio || 1, + near: perspectiveCamera.znear || 1, + far: perspectiveCamera.zfar || 2e6 + }) + }, [camera]) + + return null +} + +const PrimitiveReactor = (props: { + isSinglePrimitive: boolean + primitiveIndex: number + nodeIndex: number + documentID: string + entity: Entity +}) => { + const documentState = useHookstate(getMutableState(GLTFDocumentState)[props.documentID]) + + const nodes = documentState.nodes!.get(NO_PROXY)! + const node = nodes[props.nodeIndex]! as GLTF.INode + + const meshDef = documentState.meshes.get(NO_PROXY)![node.mesh!] + + const primitive = meshDef.primitives[props.primitiveIndex] as GLTF.IMeshPrimitive + + const options = getParserOptions(props.entity) + const geometry = GLTFLoaderFunctions.useLoadPrimitive(options, props.nodeIndex, props.primitiveIndex) + + /** @todo until we have multiple geometry to a single mesh entity support, we have to make children */ + const entity = useHookstate(() => { + if (props.isSinglePrimitive) return props.entity + + const uuid = (getNodeUUID(node, props.documentID, props.nodeIndex) + + '-primitive-' + + props.primitiveIndex) as EntityUUID + const entity = UUIDComponent.getOrCreateEntityByUUID(uuid) + + setComponent(entity, UUIDComponent, uuid) + setComponent(entity, SourceComponent, props.documentID) + + /** Ensure all base components are added for synchronous mount */ + setComponent(entity, EntityTreeComponent, { parentEntity: props.entity, childIndex: props.primitiveIndex }) + setComponent(entity, NameComponent, (node.name ?? 'Node-' + props.nodeIndex) + '_' + props.primitiveIndex) + setComponent(entity, TransformComponent) + setComponent(entity, VisibleComponent) + + return entity + }).value + + useEffect(() => { + return () => { + if (!props.isSinglePrimitive) removeEntity(entity) + } + }, []) + + useLayoutEffect(() => { + if (!geometry) return + + let mesh: Mesh | SkinnedMesh + if (typeof node.skin !== 'undefined') { + const skinnedMesh = new SkinnedMesh(geometry, new MeshBasicMaterial()) + mesh = skinnedMesh + skinnedMesh.skeleton = new Skeleton() + setComponent(entity, MeshComponent, skinnedMesh) + setComponent(entity, SkinnedMeshComponent, skinnedMesh) + } else { + mesh = new Mesh(geometry, new MeshBasicMaterial()) + setComponent(entity, MeshComponent, mesh) + } + + mesh.name = node.name ?? 'Node-' + props.nodeIndex + + /** @todo multiple primitive support */ + addObjectToGroup(entity, mesh) + proxifyParentChildRelationships(mesh) + + return () => { + removeComponent(entity, SkinnedMeshComponent) + removeComponent(entity, MeshComponent) + removeObjectFromGroup(entity, mesh) + } + }, [node.skin, geometry]) + + if (!geometry) return null + + return ( + <> + {typeof primitive.extensions === 'object' && ( + + )} + {typeof primitive.material === 'number' && ( + + )} + {primitive.targets && ( + + )} + + ) +} + +const PrimitiveExtensionReactor = (props: { + nodeIndex: number + primitiveIndex: number + documentID: string + entity: Entity +}) => { + const documentState = useHookstate(getMutableState(GLTFDocumentState)[props.documentID]) + + const nodes = documentState.nodes!.get(NO_PROXY)! + const node = nodes[props.nodeIndex]! + + const mesh = documentState.meshes.get(NO_PROXY)![node.mesh!] + + const primitive = mesh.primitives[props.primitiveIndex] + + const extensions = primitive.extensions + + useEffect(() => { + if (!extensions) return + + for (const extension in extensions) { + const Component = ComponentJSONIDMap.get(extension) + if (!Component) continue + setComponent(props.entity, Component, extensions[extension]) + } + }, [extensions]) + + return null +} + +const MaterialInstanceReactor = (props: { + nodeIndex: number + documentID: string + primitiveIndex: number + entity: Entity +}) => { + const documentState = useHookstate(getMutableState(GLTFDocumentState)[props.documentID]) + + const nodes = documentState.nodes!.get(NO_PROXY)! + const node = nodes[props.nodeIndex]! + + const mesh = documentState.meshes.get(NO_PROXY)![node.mesh!] + + const primitive = mesh.primitives[props.primitiveIndex] + + const materialUUID = (props.documentID + '-material-' + primitive.material!) as EntityUUID + const materialEntity = UUIDComponent.useEntityByUUID(materialUUID) + + useEffect(() => { + if (typeof primitive.material !== 'number' || !materialEntity) return + + setComponent(props.entity, MaterialInstanceComponent) + getMutableComponent(props.entity, MaterialInstanceComponent).uuid.merge([materialUUID]) + }, [materialEntity, primitive.material]) + + return null +} + +export const MorphTargetReactor = (props: { + documentID: string + entity: Entity + nodeIndex: number + primitiveIndex: number + targets: Record[] +}) => { + const documentState = useHookstate(getMutableState(GLTFDocumentState)[props.documentID]) + + const nodes = documentState.nodes!.get(NO_PROXY)! + const node = nodes[props.nodeIndex]! + + const meshDef = documentState.meshes.get(NO_PROXY)![node.mesh!] + + const options = getParserOptions(props.entity) + const morphTargets = GLTFLoaderFunctions.useLoadMorphTargets(options, props.targets) + + useEffect(() => { + if (!morphTargets) return + + const mesh = getOptionalMutableComponent(props.entity, MeshComponent) + if (!mesh) return + + if (morphTargets.POSITION) mesh.geometry.morphAttributes.position.set(morphTargets.POSITION) + if (morphTargets.NORMAL) mesh.geometry.morphAttributes.normal.set(morphTargets.NORMAL) + if (morphTargets.COLOR_0) mesh.geometry.morphAttributes.color.set(morphTargets.COLOR_0) + + mesh.geometry.morphTargetsRelative.set(true) + + mesh.get(NO_PROXY).updateMorphTargets() + + if (meshDef.weights) { + for (let i = 0, il = meshDef.weights.length; i < il; i++) { + mesh.morphTargetInfluences[i].set(meshDef.weights[i]) + } + } + }, [morphTargets]) + + return null +} + +export const AnimationReactor = (props: { + index: number + documentID: string + parentUUID: EntityUUID + animationState: State +}) => { + const entity = UUIDComponent.useEntityByUUID(props.parentUUID) + const options = getParserOptions(entity) + const animationTrack = GLTFLoaderFunctions.useLoadAnimation(options, props.index) + + useEffect(() => { + if (!animationTrack) return + props.animationState.merge([animationTrack]) + }, [animationTrack]) + + return null +} + +export const getParserOptions = (entity: Entity) => { + const gltfEntity = getAncestorWithComponent(entity, GLTFComponent) + const documentID = GLTFComponent.getInstanceID(gltfEntity) + const gltfComponent = getComponent(gltfEntity, GLTFComponent) + const document = getState(GLTFDocumentState)[documentID] + const gltfLoader = getState(AssetLoaderState).gltfLoader + return { + document, + documentID, + url: gltfComponent.src, + path: LoaderUtils.extractUrlBase(gltfComponent.src), + body: gltfComponent.body, + crossOrigin: gltfLoader.crossOrigin, + requestHeader: gltfLoader.requestHeader, + manager: gltfLoader.manager, + ktx2Loader: gltfLoader.ktx2Loader, + meshoptDecoder: gltfLoader.meshoptDecoder + } as GLTFParserOptions +} diff --git a/packages/engine/src/gltf/MaterialDefinitionComponent.tsx b/packages/engine/src/gltf/MaterialDefinitionComponent.tsx new file mode 100644 index 0000000000..48c341519f --- /dev/null +++ b/packages/engine/src/gltf/MaterialDefinitionComponent.tsx @@ -0,0 +1,1228 @@ +/* +CPAL-1.0 License + +The contents of this file are subject to the Common Public Attribution License +Version 1.0. (the "License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at +https://github.com/EtherealEngine/etherealengine/blob/dev/LICENSE. +The License is based on the Mozilla Public License Version 1.1, but Sections 14 +and 15 have been added to cover use of software over a computer network and +provide for limited attribution for the Original Developer. In addition, +Exhibit A has been modified to be consistent with Exhibit B. + +Software distributed under the License is distributed on an "AS IS" basis, +WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License for the +specific language governing rights and limitations under the License. + +The Original Code is Ethereal Engine. + +The Original Developer is the Initial Developer. The Initial Developer of the +Original Code is the Ethereal Engine team. + +All portions of the code written by the Ethereal Engine team are Copyright © 2021-2023 +Ethereal Engine. All Rights Reserved. +*/ + +import { + ComponentType, + EntityUUID, + UUIDComponent, + defineComponent, + getComponent, + setComponent, + useComponent, + useEntityContext +} from '@etherealengine/ecs' +import { NO_PROXY, startReactor, useImmediateEffect } from '@etherealengine/hyperflux' +import createReadableTexture from '@etherealengine/spatial/src/renderer/functions/createReadableTexture' +import { MaterialStateComponent } from '@etherealengine/spatial/src/renderer/materials/MaterialComponent' +import { GLTF } from '@gltf-transform/core' +import { useEffect, useLayoutEffect } from 'react' +import { + CanvasTexture, + Color, + LinearSRGBColorSpace, + Material, + MeshPhysicalMaterial, + MeshStandardMaterial, + SRGBColorSpace, + Texture, + Vector2 +} from 'three' +import { EXTENSIONS } from '../assets/loaders/gltf/GLTFExtensions' +import { GLTFLoaderFunctions } from './GLTFLoaderFunctions' +import { getParserOptions } from './GLTFState' + +export const MaterialDefinitionComponent = defineComponent({ + name: 'MaterialDefinitionComponent', + onInit: (entity) => { + return { + type: 'MeshStandardMaterial' + } as GLTF.IMaterial & { + type: 'MeshStandardMaterial' | 'MeshPhysicalMaterial' | 'MeshBasicMaterial' | (string & {}) + } + }, + + onSet: (entity, component, json) => { + if (!json) return + if (typeof json.type === 'string') component.type.set(json.type) + if (typeof json.pbrMetallicRoughness === 'object') component.pbrMetallicRoughness.set(json.pbrMetallicRoughness) + if (typeof json.normalTexture === 'object') component.normalTexture.set(json.normalTexture) + if (typeof json.occlusionTexture === 'object') component.occlusionTexture.set(json.occlusionTexture) + if (typeof json.emissiveTexture === 'object') component.emissiveTexture.set(json.emissiveTexture) + if (typeof json.emissiveFactor === 'object') component.emissiveFactor.set(json.emissiveFactor) + if (typeof json.alphaMode === 'string') component.alphaMode.set(json.alphaMode) + if (typeof json.alphaCutoff === 'number') component.alphaCutoff.set(json.alphaCutoff) + if (typeof json.doubleSided === 'boolean') component.doubleSided.set(json.doubleSided) + }, + + reactor: () => { + const entity = useEntityContext() + const component = useComponent(entity, MaterialDefinitionComponent) + const material = GLTFLoaderFunctions.useLoadMaterial( + getParserOptions(entity), + component.get(NO_PROXY) as ComponentType + ) + + useLayoutEffect(() => { + if (!entity || !material) return + const uuid = getComponent(entity, UUIDComponent) + material.uuid = uuid + setComponent(entity, MaterialStateComponent, { material }) + }, [material]) + + return null + } +}) + +declare module 'three/src/materials/MeshPhysicalMaterial' { + export interface MeshPhysicalMaterial { + setValues(parameters: MeshPhysicalMaterialParameters): void + } +} + +/** + * Unlit Materials Extension + * + * Specification: https://github.com/KhronosGroup/glTF/tree/master/extensions/2.0/Khronos/KHR_materials_unlit + */ +export const KHRUnlitExtensionComponent = defineComponent({ + name: 'KHRUnlitExtensionComponent', + jsonID: EXTENSIONS.KHR_MATERIALS_UNLIT, + + onInit(entity) { + return {} + }, + + toJSON(entity, component) { + return {} + }, + + reactor: () => { + const entity = useEntityContext() + + useEffect(() => { + setComponent(entity, MaterialDefinitionComponent, { type: 'MeshBasicMaterial' }) + }, []) + + return null + } +}) + +/** + * Materials Emissive Strength Extension + * + * Specification: https://github.com/KhronosGroup/glTF/blob/5768b3ce0ef32bc39cdf1bef10b948586635ead3/extensions/2.0/Khronos/KHR_materials_emissive_strength/README.md + */ +export const KHREmissiveStrengthExtensionComponent = defineComponent({ + name: 'KHREmissiveStrengthExtensionComponent', + jsonID: EXTENSIONS.KHR_MATERIALS_EMISSIVE_STRENGTH, + + onInit(entity) { + return {} as { + emissiveStrength?: number + } + }, + + onSet(entity, component, json) { + if (!json) return + if (typeof json.emissiveStrength === 'number') component.emissiveStrength.set(json.emissiveStrength) + }, + + toJSON(entity, component) { + return { + emissiveStrength: component.emissiveStrength.value + } + }, + + reactor: () => { + const entity = useEntityContext() + const component = useComponent(entity, KHREmissiveStrengthExtensionComponent) + const materialStateComponent = useComponent(entity, MaterialStateComponent) + + useEffect(() => { + if (typeof component.emissiveStrength.value !== 'number') return + const material = materialStateComponent.material.value as MeshStandardMaterial + material.setValues({ emissiveIntensity: component.emissiveStrength.value }) + }, [materialStateComponent.material.value.type, component.emissiveStrength.value]) + + return null + } +}) + +/** + * Clearcoat Materials Extension + * + * Specification: https://github.com/KhronosGroup/glTF/tree/master/extensions/2.0/Khronos/KHR_materials_clearcoat + */ +export const KHRClearcoatExtensionComponent = defineComponent({ + name: 'KHRClearcoatExtensionComponent', + jsonID: EXTENSIONS.KHR_MATERIALS_CLEARCOAT, + + onInit(entity) { + return {} as { + clearcoatFactor?: number + clearcoatTexture?: GLTF.ITextureInfo + clearcoatRoughnessFactor?: number + clearcoatRoughnessTexture?: GLTF.ITextureInfo + clearcoatNormalTexture?: GLTF.IMaterialNormalTextureInfo + } + }, + + onSet(entity, component, json) { + if (!json) return + if (typeof json.clearcoatFactor === 'number') component.clearcoatFactor.set(json.clearcoatFactor) + if (typeof json.clearcoatTexture === 'object') component.clearcoatTexture.set(json.clearcoatTexture) + if (typeof json.clearcoatRoughnessFactor === 'number') + component.clearcoatRoughnessFactor.set(json.clearcoatRoughnessFactor) + if (typeof json.clearcoatRoughnessTexture === 'object') + component.clearcoatRoughnessTexture.set(json.clearcoatRoughnessTexture) + if (typeof json.clearcoatNormalTexture === 'object') + component.clearcoatNormalTexture.set(json.clearcoatNormalTexture) + }, + + reactor: () => { + const entity = useEntityContext() + const component = useComponent(entity, KHRClearcoatExtensionComponent) + const materialStateComponent = useComponent(entity, MaterialStateComponent) + + useEffect(() => { + setComponent(entity, MaterialDefinitionComponent, { type: 'MeshPhysicalMaterial' }) + }, []) + + useEffect(() => { + if (!component.clearcoatFactor.value) return + const material = materialStateComponent.material.value as MeshPhysicalMaterial + material.setValues({ clearcoat: component.clearcoatFactor.value }) + material.needsUpdate = true + }, [materialStateComponent.material.value.type, component.clearcoatFactor.value]) + + useEffect(() => { + if (!component.clearcoatRoughnessFactor.value) return + const material = materialStateComponent.material.value as MeshPhysicalMaterial + material.setValues({ clearcoatRoughness: component.clearcoatRoughnessFactor.value }) + material.needsUpdate = true + }, [materialStateComponent.material.value.type, component.clearcoatRoughnessFactor.value]) + + const options = getParserOptions(entity) + const clearcoatMap = GLTFLoaderFunctions.useAssignTexture(options, component.clearcoatTexture.get(NO_PROXY)) + + useEffect(() => { + const material = materialStateComponent.material.value as MeshPhysicalMaterial + material.setValues({ clearcoatMap }) + material.needsUpdate = true + }, [materialStateComponent.material.value.type, clearcoatMap]) + + const clearcoatRoughnessMap = GLTFLoaderFunctions.useAssignTexture( + options, + component.clearcoatRoughnessTexture.get(NO_PROXY) + ) + + useEffect(() => { + const material = materialStateComponent.material.value as MeshPhysicalMaterial + material.setValues({ clearcoatRoughnessMap }) + material.needsUpdate = true + }, [materialStateComponent.material.value.type, clearcoatRoughnessMap]) + + const clearcoatNormalMap = GLTFLoaderFunctions.useAssignTexture( + options, + component.clearcoatNormalTexture.get(NO_PROXY) + ) + + useEffect(() => { + const material = materialStateComponent.material.value as MeshPhysicalMaterial + + if (component.clearcoatNormalTexture.value?.scale !== undefined) { + const scale = component.clearcoatNormalTexture.value.scale + material.setValues({ clearcoatNormalScale: new Vector2(scale, scale) }) + } + + material.setValues({ clearcoatNormalMap }) + material.needsUpdate = true + }, [materialStateComponent.material.value.type, clearcoatNormalMap]) + + return null + } +}) + +/** + * Iridescence Materials Extension + * + * Specification: https://github.com/KhronosGroup/glTF/tree/master/extensions/2.0/Khronos/KHR_materials_iridescence + */ +export const KHRIridescenceExtensionComponent = defineComponent({ + name: 'KHRIridescenceExtensionComponent', + jsonID: EXTENSIONS.KHR_MATERIALS_IRIDESCENCE, + + onInit(entity) { + return {} as { + iridescenceFactor?: number + iridescenceTexture?: GLTF.ITextureInfo + iridescenceIor?: number + iridescenceThicknessMinimum?: number + iridescenceThicknessMaximum?: number + iridescenceThicknessTexture?: GLTF.ITextureInfo + } + }, + + onSet(entity, component, json) { + if (!json) return + if (typeof json.iridescenceFactor === 'number') component.iridescenceFactor.set(json.iridescenceFactor) + if (typeof json.iridescenceTexture === 'object') component.iridescenceTexture.set(json.iridescenceTexture) + if (typeof json.iridescenceIor === 'number') component.iridescenceIor.set(json.iridescenceIor) + if (typeof json.iridescenceThicknessMinimum === 'number') + component.iridescenceThicknessMinimum.set(json.iridescenceThicknessMinimum) + if (typeof json.iridescenceThicknessMaximum === 'number') + component.iridescenceThicknessMaximum.set(json.iridescenceThicknessMaximum) + if (typeof json.iridescenceThicknessTexture === 'object') + component.iridescenceThicknessTexture.set(json.iridescenceThicknessTexture) + }, + + toJSON(entity, component) { + return { + iridescenceFactor: component.iridescenceFactor.value, + iridescenceTexture: component.iridescenceTexture.value, + iridescenceIor: component.iridescenceIor.value, + iridescenceThicknessMinimum: component.iridescenceThicknessMinimum.value, + iridescenceThicknessMaximum: component.iridescenceThicknessMaximum.value, + iridescenceThicknessTexture: component.iridescenceThicknessTexture.value + } + }, + + reactor: () => { + const entity = useEntityContext() + const component = useComponent(entity, KHRIridescenceExtensionComponent) + const materialStateComponent = useComponent(entity, MaterialStateComponent) + + useEffect(() => { + setComponent(entity, MaterialDefinitionComponent, { type: 'MeshPhysicalMaterial' }) + }, []) + + useEffect(() => { + if (!component.iridescenceFactor.value) return + const material = materialStateComponent.material.value as MeshPhysicalMaterial + material.setValues({ iridescence: component.iridescenceFactor.value }) + material.needsUpdate = true + }, [materialStateComponent.material.value.type, component.iridescenceFactor.value]) + + useEffect(() => { + if (!component.iridescenceIor.value) return + const material = materialStateComponent.material.value as MeshPhysicalMaterial + material.setValues({ iridescenceIOR: component.iridescenceIor.value }) + material.needsUpdate = true + }, [materialStateComponent.material.value.type, component.iridescenceIor.value]) + + const options = getParserOptions(entity) + const iridescenceMap = GLTFLoaderFunctions.useAssignTexture(options, component.iridescenceTexture.get(NO_PROXY)) + + useEffect(() => { + const material = materialStateComponent.material.value as MeshPhysicalMaterial + material.setValues({ iridescenceMap }) + material.needsUpdate = true + }, [materialStateComponent.material.value.type, iridescenceMap]) + + useEffect(() => { + const material = materialStateComponent.material.value as MeshPhysicalMaterial + material.setValues({ + iridescenceThicknessRange: [ + component.iridescenceThicknessMinimum.value ?? 100, + component.iridescenceThicknessMaximum.value ?? 400 + ] + }) + material.needsUpdate = true + }, [ + materialStateComponent.material.value.type, + component.iridescenceThicknessMinimum.value, + component.iridescenceThicknessMaximum.value + ]) + + const iridescenceThicknessMap = GLTFLoaderFunctions.useAssignTexture( + options, + component.iridescenceThicknessTexture.get(NO_PROXY) + ) + + useEffect(() => { + const material = materialStateComponent.material.value as MeshPhysicalMaterial + material.setValues({ iridescenceThicknessMap }) + material.needsUpdate = true + }, [materialStateComponent.material.value.type, iridescenceThicknessMap]) + + return null + } +}) + +/** + * Sheen Materials Extension + * + * Specification: https://github.com/KhronosGroup/glTF/tree/main/extensions/2.0/Khronos/KHR_materials_sheen + */ +export const KHRSheenExtensionComponent = defineComponent({ + name: 'KHRSheenExtensionComponent', + jsonID: EXTENSIONS.KHR_MATERIALS_SHEEN, + + onInit(entity) { + return {} as { + sheenColorFactor?: [number, number, number] + sheenRoughnessFactor?: number + sheenColorTexture?: GLTF.ITextureInfo + sheenRoughnessTexture?: GLTF.ITextureInfo + } + }, + + onSet(entity, component, json) { + if (!json) return + if (Array.isArray(json.sheenColorFactor)) component.sheenColorFactor.set(json.sheenColorFactor) + if (typeof json.sheenRoughnessFactor === 'number') component.sheenRoughnessFactor.set(json.sheenRoughnessFactor) + if (typeof json.sheenColorTexture === 'object') component.sheenColorTexture.set(json.sheenColorTexture) + if (typeof json.sheenRoughnessTexture === 'object') component.sheenRoughnessTexture.set(json.sheenRoughnessTexture) + }, + + toJSON(entity, component) { + return { + sheenColorFactor: component.sheenColorFactor.value, + sheenRoughnessFactor: component.sheenRoughnessFactor.value, + sheenColorTexture: component.sheenColorTexture.value, + sheenRoughnessTexture: component.sheenRoughnessTexture.value + } + }, + + reactor: () => { + const entity = useEntityContext() + const component = useComponent(entity, KHRSheenExtensionComponent) + const materialStateComponent = useComponent(entity, MaterialStateComponent) + + useEffect(() => { + setComponent(entity, MaterialDefinitionComponent, { type: 'MeshPhysicalMaterial' }) + const material = materialStateComponent.material.value as MeshPhysicalMaterial + material.setValues({ sheen: 1 }) + }, []) + + useEffect(() => { + if (!component.sheenColorFactor.value) return + const material = materialStateComponent.material.value as MeshPhysicalMaterial + material.setValues({ + sheenColor: new Color().setRGB( + component.sheenColorFactor.value[0], + component.sheenColorFactor.value[1], + component.sheenColorFactor.value[2], + LinearSRGBColorSpace + ) + }) + material.needsUpdate = true + }, [materialStateComponent.material.value.type, component.sheenColorFactor.value]) + + useEffect(() => { + if (!component.sheenRoughnessFactor.value) return + const material = materialStateComponent.material.value as MeshPhysicalMaterial + material.setValues({ sheenRoughness: component.sheenRoughnessFactor.value }) + material.needsUpdate = true + }, [materialStateComponent.material.value.type, component.sheenRoughnessFactor.value]) + + const options = getParserOptions(entity) + const sheenColorMap = GLTFLoaderFunctions.useAssignTexture(options, component.sheenColorTexture.get(NO_PROXY)) + + useEffect(() => { + if (sheenColorMap) sheenColorMap.colorSpace = SRGBColorSpace + const material = materialStateComponent.material.value as MeshPhysicalMaterial + material.setValues({ sheenColorMap }) + material.needsUpdate = true + }, [materialStateComponent.material.value.type, sheenColorMap]) + + const sheenRoughnessMap = GLTFLoaderFunctions.useAssignTexture( + options, + component.sheenRoughnessTexture.get(NO_PROXY) + ) + + useEffect(() => { + const material = materialStateComponent.material.value as MeshPhysicalMaterial + material.setValues({ sheenRoughnessMap }) + material.needsUpdate = true + }, [materialStateComponent.material.value.type, sheenRoughnessMap]) + + return null + } +}) + +/** + * Transmission Materials Extension + * + * Specification: https://github.com/KhronosGroup/glTF/tree/master/extensions/2.0/Khronos/KHR_materials_transmission + * Draft: https://github.com/KhronosGroup/glTF/pull/1698 + */ +export const KHRTransmissionExtensionComponent = defineComponent({ + name: 'KHRTransmissionExtensionComponent', + jsonID: EXTENSIONS.KHR_MATERIALS_TRANSMISSION, + + onInit(entity) { + return {} as { + transmissionFactor?: number + transmissionTexture?: GLTF.ITextureInfo + } + }, + + onSet(entity, component, json) { + if (!json) return + if (typeof json.transmissionFactor === 'number') component.transmissionFactor.set(json.transmissionFactor) + if (typeof json.transmissionTexture === 'object') component.transmissionTexture.set(json.transmissionTexture) + }, + + toJSON(entity, component) { + return { + transmissionFactor: component.transmissionFactor.value, + transmissionTexture: component.transmissionTexture.value + } + }, + + reactor: () => { + const entity = useEntityContext() + const component = useComponent(entity, KHRTransmissionExtensionComponent) + const materialStateComponent = useComponent(entity, MaterialStateComponent) + + useEffect(() => { + setComponent(entity, MaterialDefinitionComponent, { type: 'MeshPhysicalMaterial' }) + }, []) + + useEffect(() => { + if (!component.transmissionFactor.value) return + const material = materialStateComponent.material.value as MeshPhysicalMaterial + material.setValues({ transmission: component.transmissionFactor.value }) + material.needsUpdate = true + }, [materialStateComponent.material.value, component.transmissionFactor.value]) + + const options = getParserOptions(entity) + const transmissionMap = GLTFLoaderFunctions.useAssignTexture(options, component.transmissionTexture.get(NO_PROXY)) + + useEffect(() => { + const material = materialStateComponent.material.value as MeshPhysicalMaterial + material.setValues({ transmissionMap }) + material.needsUpdate = true + }, [materialStateComponent.material.value, transmissionMap]) + + return null + } +}) + +/** + * Materials Volume Extension + * + * Specification: https://github.com/KhronosGroup/glTF/tree/master/extensions/2.0/Khronos/KHR_materials_volume + */ +export const KHRVolumeExtensionComponent = defineComponent({ + name: 'KHRVolumeExtensionComponent', + jsonID: EXTENSIONS.KHR_MATERIALS_VOLUME, + + onInit(entity) { + return {} as { + thicknessFactor?: number + thicknessTexture?: GLTF.ITextureInfo + attenuationDistance?: number + attenuationColorFactor?: [number, number, number] + } + }, + + onSet(entity, component, json) { + if (!json) return + if (typeof json.thicknessFactor === 'number') component.thicknessFactor.set(json.thicknessFactor) + if (typeof json.thicknessTexture === 'object') component.thicknessTexture.set(json.thicknessTexture) + if (typeof json.attenuationDistance === 'number') component.attenuationDistance.set(json.attenuationDistance) + if (Array.isArray(json.attenuationColorFactor)) component.attenuationColorFactor.set(json.attenuationColorFactor) + }, + + toJSON(entity, component) { + return { + thicknessFactor: component.thicknessFactor.value, + thicknessTexture: component.thicknessTexture.value, + attenuationDistance: component.attenuationDistance.value, + attenuationColorFactor: component.attenuationColorFactor.value + } + }, + + reactor: () => { + const entity = useEntityContext() + const component = useComponent(entity, KHRVolumeExtensionComponent) + const materialStateComponent = useComponent(entity, MaterialStateComponent) + + useEffect(() => { + setComponent(entity, MaterialDefinitionComponent, { type: 'MeshPhysicalMaterial' }) + }, []) + + useEffect(() => { + const material = materialStateComponent.material.value as MeshPhysicalMaterial + material.setValues({ thickness: component.thicknessFactor.value ?? 0 }) + material.needsUpdate = true + }, [materialStateComponent.material.value.type, component.thicknessFactor.value]) + + useEffect(() => { + const material = materialStateComponent.material.value as MeshPhysicalMaterial + material.setValues({ attenuationDistance: component.attenuationDistance.value || Infinity }) + material.needsUpdate = true + }, [materialStateComponent.material.value.type, component.attenuationDistance.value]) + + useEffect(() => { + if (!component.attenuationColorFactor.value) return + const material = materialStateComponent.material.value as MeshPhysicalMaterial + material.setValues({ + attenuationColor: new Color().setRGB( + component.attenuationColorFactor.value[0], + component.attenuationColorFactor.value[1], + component.attenuationColorFactor.value[2], + LinearSRGBColorSpace + ) + }) + material.needsUpdate = true + }, [materialStateComponent.material.value.type, component.attenuationColorFactor.value]) + + const options = getParserOptions(entity) + const thicknessMap = GLTFLoaderFunctions.useAssignTexture(options, component.thicknessTexture.get(NO_PROXY)) + + useEffect(() => { + const material = materialStateComponent.material.value as MeshPhysicalMaterial + material.setValues({ thicknessMap }) + material.needsUpdate = true + }, [materialStateComponent.material.value.type, thicknessMap]) + + return null + } +}) + +/** + * Materials ior Extension + * + * Specification: https://github.com/KhronosGroup/glTF/tree/master/extensions/2.0/Khronos/KHR_materials_ior + */ +export const KHRIorExtensionComponent = defineComponent({ + name: 'KHRIorExtensionComponent', + jsonID: EXTENSIONS.KHR_MATERIALS_IOR, + + onInit(entity) { + return {} as { + ior?: number + } + }, + + onSet(entity, component, json) { + if (!json) return + if (typeof json.ior === 'number') component.ior.set(json.ior) + }, + + toJSON(entity, component) { + return { + ior: component.ior.value + } + }, + + reactor: () => { + const entity = useEntityContext() + const component = useComponent(entity, KHRIorExtensionComponent) + const materialStateComponent = useComponent(entity, MaterialStateComponent) + + useEffect(() => { + setComponent(entity, MaterialDefinitionComponent, { type: 'MeshPhysicalMaterial' }) + }, []) + + useEffect(() => { + const material = materialStateComponent.material.value as MeshPhysicalMaterial + material.setValues({ ior: component.ior.value ?? 1.5 }) + material.needsUpdate = true + }, [materialStateComponent.material.value.type, component.ior.value]) + + return null + } +}) + +/** + * Materials specular Extension + * + * Specification: https://github.com/KhronosGroup/glTF/tree/master/extensions/2.0/Khronos/KHR_materials_specular + */ +export const KHRSpecularExtensionComponent = defineComponent({ + name: 'KHRSpecularExtensionComponent', + jsonID: EXTENSIONS.KHR_MATERIALS_SPECULAR, + + onInit(entity) { + return {} as { + specularFactor?: number + specularTexture?: GLTF.ITextureInfo + specularColorFactor?: [number, number, number] + specularColorTexture?: GLTF.ITextureInfo + } + }, + + onSet(entity, component, json) { + if (!json) return + if (typeof json.specularFactor === 'number') component.specularFactor.set(json.specularFactor) + if (typeof json.specularTexture === 'object') component.specularTexture.set(json.specularTexture) + if (Array.isArray(json.specularColorFactor)) component.specularColorFactor.set(json.specularColorFactor) + if (typeof json.specularColorTexture === 'object') component.specularColorTexture.set(json.specularColorTexture) + }, + + toJSON(entity, component) { + return { + specularFactor: component.specularFactor.value, + specularTexture: component.specularTexture.value, + specularColorFactor: component.specularColorFactor.value, + specularColorTexture: component.specularColorTexture.value + } + }, + + reactor: () => { + const entity = useEntityContext() + const component = useComponent(entity, KHRSpecularExtensionComponent) + const materialStateComponent = useComponent(entity, MaterialStateComponent) + + useImmediateEffect(() => { + setComponent(entity, MaterialDefinitionComponent, { type: 'MeshPhysicalMaterial' }) + }, []) + + useEffect(() => { + const material = materialStateComponent.material.value as MeshPhysicalMaterial + material.setValues({ specularIntensity: component.specularFactor.value ?? 1.0 }) + material.needsUpdate = true + }, [materialStateComponent.material.type.value, component.specularFactor.value]) + + useEffect(() => { + const specularColorFactor = component.specularColorFactor.value ?? [1, 1, 1] + const material = materialStateComponent.material.value as MeshPhysicalMaterial + material.setValues({ + specularColor: new Color().setRGB( + specularColorFactor[0], + specularColorFactor[1], + specularColorFactor[2], + LinearSRGBColorSpace + ) + }) + material.needsUpdate = true + }, [materialStateComponent.material.type.value, component.specularColorFactor.value]) + + const options = getParserOptions(entity) + const specularMap = GLTFLoaderFunctions.useAssignTexture(options, component.specularTexture.get(NO_PROXY)) + + useEffect(() => { + const material = materialStateComponent.material.value as MeshPhysicalMaterial + material.setValues({ specularIntensityMap: specularMap }) + material.needsUpdate = true + }, [materialStateComponent.material.type.value, specularMap]) + + const specularColorMap = GLTFLoaderFunctions.useAssignTexture(options, component.specularColorTexture.get(NO_PROXY)) + + useEffect(() => { + const material = materialStateComponent.material.value as MeshPhysicalMaterial + material.setValues({ specularColorMap }) + material.needsUpdate = true + }, [materialStateComponent.material.type.value, specularColorMap]) + + return null + } +}) + +/** + * Materials bump Extension + * + * Specification: https://github.com/KhronosGroup/glTF/tree/master/extensions/2.0/Khronos/EXT_materials_bump + */ +export const EXTBumpExtensionComponent = defineComponent({ + name: 'EXTBumpExtensionComponent', + jsonID: EXTENSIONS.EXT_MATERIALS_BUMP, + + onInit(entity) { + return {} as { + bumpFactor?: number + bumpTexture?: GLTF.ITextureInfo + } + }, + + onSet(entity, component, json) { + if (!json) return + if (typeof json.bumpFactor === 'number') component.bumpFactor.set(json.bumpFactor) + if (typeof json.bumpTexture === 'object') component.bumpTexture.set(json.bumpTexture) + }, + + toJSON(entity, component) { + return { + bumpFactor: component.bumpFactor.value, + bumpTexture: component.bumpTexture.value + } + }, + + reactor: () => { + const entity = useEntityContext() + const component = useComponent(entity, EXTBumpExtensionComponent) + const materialStateComponent = useComponent(entity, MaterialStateComponent) + + useEffect(() => { + setComponent(entity, MaterialDefinitionComponent, { type: 'MeshPhysicalMaterial' }) + }, []) + + useEffect(() => { + const material = materialStateComponent.material.value as MeshPhysicalMaterial + material.setValues({ bumpScale: component.bumpFactor.value ?? 1.0 }) + material.needsUpdate = true + }, [materialStateComponent.material.value.type, component.bumpFactor.value]) + + const options = getParserOptions(entity) + const bumpMap = GLTFLoaderFunctions.useAssignTexture(options, component.bumpTexture.get(NO_PROXY)) + + useEffect(() => { + const material = materialStateComponent.material.value as MeshPhysicalMaterial + material.setValues({ bumpMap }) + material.needsUpdate = true + }, [materialStateComponent.material.value.type, bumpMap]) + + return null + } +}) + +/** + * Materials anisotropy Extension + * + * Specification: https://github.com/KhronosGroup/glTF/tree/master/extensions/2.0/Khronos/KHR_materials_anisotropy + */ +export const KHRAnisotropyExtensionComponent = defineComponent({ + name: 'KHRAnisotropyExtensionComponent', + jsonID: EXTENSIONS.KHR_MATERIALS_ANISOTROPY, + + onInit(entity) { + return {} as { + anisotropyStrength?: number + anisotropyRotation?: number + anisotropyTexture?: GLTF.ITextureInfo + } + }, + + onSet(entity, component, json) { + if (!json) return + if (typeof json.anisotropyStrength === 'number') component.anisotropyStrength.set(json.anisotropyStrength) + if (typeof json.anisotropyRotation === 'number') component.anisotropyRotation.set(json.anisotropyRotation) + if (typeof json.anisotropyTexture === 'object') component.anisotropyTexture.set(json.anisotropyTexture) + }, + + toJSON(entity, component) { + return { + anisotropyStrength: component.anisotropyStrength.value, + anisotropyRotation: component.anisotropyRotation.value, + anisotropyTexture: component.anisotropyTexture.value + } + }, + + reactor: () => { + const entity = useEntityContext() + const component = useComponent(entity, KHRAnisotropyExtensionComponent) + const materialStateComponent = useComponent(entity, MaterialStateComponent) + + useEffect(() => { + setComponent(entity, MaterialDefinitionComponent, { type: 'MeshPhysicalMaterial' }) + }, []) + + useEffect(() => { + const material = materialStateComponent.material.value as MeshPhysicalMaterial + material.setValues({ anisotropy: component.anisotropyStrength.value ?? 0.0 }) + material.needsUpdate = true + }, [materialStateComponent.material.value.type, component.anisotropyStrength.value]) + + useEffect(() => { + const material = materialStateComponent.material.value as MeshPhysicalMaterial + material.setValues({ anisotropyRotation: component.anisotropyRotation.value ?? 0.0 }) + material.needsUpdate = true + }, [materialStateComponent.material.value.type, component.anisotropyRotation.value]) + + const options = getParserOptions(entity) + const anisotropyMap = GLTFLoaderFunctions.useAssignTexture(options, component.anisotropyTexture.get(NO_PROXY)) + + useEffect(() => { + const material = materialStateComponent.material.value as MeshPhysicalMaterial + material.setValues({ anisotropyMap }) + material.needsUpdate = true + }, [materialStateComponent.material.value.type, anisotropyMap]) + + return null + } +}) + +/** + * Texture Transform Extension + * + * Specification: https://github.com/KhronosGroup/glTF/tree/master/extensions/2.0/Khronos/KHR_texture_transform + */ +export class GLTFTextureTransformExtension { + name = EXTENSIONS.KHR_TEXTURE_TRANSFORM + + extendTexture(texture, transform) { + if ( + (transform.texCoord === undefined || transform.texCoord === texture.channel) && + transform.offset === undefined && + transform.rotation === undefined && + transform.scale === undefined + ) { + // See https://github.com/mrdoob/three.js/issues/21819. + return texture + } + + texture = texture.clone() + + if (transform.texCoord !== undefined) { + texture.channel = transform.texCoord + } + + if (transform.offset !== undefined) { + texture.offset.fromArray(transform.offset) + } + + if (transform.rotation !== undefined) { + texture.rotation = transform.rotation + } + + if (transform.scale !== undefined) { + texture.repeat.fromArray(transform.scale) + } + + texture.needsUpdate = true + + return texture + } +} + +type GLTFTextureTransformExtensionType = { + texCoord?: number + offset?: [number, number] + rotation?: number + scale?: [number, number] +} + +export const KHRTextureTransformExtensionComponent = defineComponent({ + name: 'KHRTextureTransformExtensionComponent', + jsonID: EXTENSIONS.KHR_TEXTURE_TRANSFORM, + + /** static function */ + extendTexture: (texture: Texture, transform: GLTFTextureTransformExtensionType) => { + if ( + (transform.texCoord === undefined || transform.texCoord === texture.channel) && + transform.offset === undefined && + transform.rotation === undefined && + transform.scale === undefined + ) { + // See https://github.com/mrdoob/three.js/issues/21819. + return texture + } + + /** @todo this throws hookstate 109... */ + // texture = texture.clone() + + if (transform.texCoord !== undefined) { + texture.channel = transform.texCoord + } + + if (transform.offset !== undefined) { + texture.offset.fromArray(transform.offset) + } + + if (transform.rotation !== undefined) { + texture.rotation = transform.rotation + } + + if (transform.scale !== undefined) { + texture.repeat.fromArray(transform.scale) + } + + texture.needsUpdate = true + + return texture + } +}) + +export const MozillaHubsLightMapComponent = defineComponent({ + name: 'MozillaHubsLightMapComponent', + jsonID: 'MOZ_lightmap', + + onInit(entity) { + return {} as { + index: 1 + intensity: 1.0 + } + }, + + onSet(entity, component, json) { + if (!json) return + if (typeof json.index === 'number') component.index.set(json.index) + if (typeof json.intensity === 'number') component.intensity.set(json.intensity) + }, + + toJSON(entity, component) { + return { + index: component.index.value, + intensity: component.intensity.value + } + }, + + reactor: () => { + const entity = useEntityContext() + const component = useComponent(entity, MozillaHubsLightMapComponent) + const materialStateComponent = useComponent(entity, MaterialStateComponent) + + useEffect(() => { + const material = materialStateComponent.material.value as MeshPhysicalMaterial + const materialDefinitionComponent = getComponent(entity, MaterialDefinitionComponent) + // Multiply by pi for MeshBasicMaterial shading + const lightMapIntensity = + component.intensity.value * (materialDefinitionComponent.type === 'basic' ? Math.PI : 1.0) + + material.setValues({ lightMapIntensity }) + material.needsUpdate = true + }, [component.intensity.value]) + + const options = getParserOptions(entity) + const lightMap = GLTFLoaderFunctions.useAssignTexture(options, component.get(NO_PROXY)) + + useEffect(() => { + if (!lightMap) return + + const material = materialStateComponent.material.value as MeshPhysicalMaterial + + lightMap.channel = 1 + material.lightMap = lightMap + + material.setValues({ lightMap: lightMap }) + material.needsUpdate = true + }, [lightMap]) + + return null + } +}) + +/** + * @deprecated - use KHR_materials_ior and KHR_materials_specular instead + */ +export const KHRMaterialsPBRSpecularGlossinessComponent = defineComponent({ + name: 'KHRMaterialsPBRSpecularGlossinessComponent', + jsonID: 'KHR_materials_pbrSpecularGlossiness', + + onInit(entity) { + return {} as { + diffuseFactor?: [number, number, number, number] + diffuseTexture?: GLTF.ITextureInfo + specularFactor?: [number, number, number] + glossinessFactor?: number + specularGlossinessTexture?: GLTF.ITextureInfo + } + }, + + onSet(entity, component, json) { + if (!json) return + if (Array.isArray(json.diffuseFactor)) component.diffuseFactor.set(json.diffuseFactor) + if (typeof json.diffuseTexture === 'object') component.diffuseTexture.set(json.diffuseTexture) + if (Array.isArray(json.specularFactor)) component.specularFactor.set(json.specularFactor) + if (typeof json.glossinessFactor === 'number') component.glossinessFactor.set(json.glossinessFactor) + if (typeof json.specularGlossinessTexture === 'object') + component.specularGlossinessTexture.set(json.specularGlossinessTexture) + }, + + toJSON(entity, component) { + return { + diffuseFactor: component.diffuseFactor.value, + diffuseTexture: component.diffuseTexture.value, + specularFactor: component.specularFactor.value, + glossinessFactor: component.glossinessFactor.value, + specularGlossinessTexture: component.specularGlossinessTexture.value + } + }, + + reactor: () => { + const entity = useEntityContext() + const component = useComponent(entity, KHRMaterialsPBRSpecularGlossinessComponent) + const materialStateComponent = useComponent(entity, MaterialStateComponent) + + useEffect(() => { + setComponent(entity, MaterialDefinitionComponent, { type: 'MeshStandardMaterial' }) + console.warn( + 'KHR_materials_pbrSpecularGlossiness is deprecated. Use KHR_materials_ior and KHR_materials_specular instead.' + ) + }, []) + + useEffect(() => { + const material = materialStateComponent.material.value as MeshStandardMaterial + material.setValues({ + color: new Color().fromArray(component.diffuseFactor.value ?? [1, 1, 1, 1]), + opacity: component.diffuseFactor.value ? component.diffuseFactor.value[3] : 1 + }) + material.needsUpdate = true + }, [materialStateComponent.material.value.type, component.diffuseFactor.value]) + + useEffect(() => { + const material = materialStateComponent.material.value as MeshStandardMaterial + material.setValues({ + roughness: 1 - (component.glossinessFactor.value ?? 1) + }) + material.needsUpdate = true + }, [materialStateComponent.material.value.type, component.glossinessFactor.value]) + + const options = getParserOptions(entity) + const map = GLTFLoaderFunctions.useAssignTexture(options, component.diffuseTexture.get(NO_PROXY)) + + useEffect(() => { + const material = materialStateComponent.material.value as MeshStandardMaterial + material.setValues({ map }) + material.needsUpdate = true + }, [materialStateComponent.material.value.type, map]) + + const specularGlossinessMap = GLTFLoaderFunctions.useAssignTexture( + options, + component.specularGlossinessTexture.get(NO_PROXY) + ) + + useEffect(() => { + if (!specularGlossinessMap) return + + const abortController = new AbortController() + + invertGlossinessMap(specularGlossinessMap).then((invertedMap) => { + if (abortController.signal.aborted) return + + const material = materialStateComponent.material.value as MeshStandardMaterial + material.setValues({ roughnessMap: invertedMap }) + material.needsUpdate = true + }) + + return () => { + abortController.abort() + } + }, [materialStateComponent.material.value.type, specularGlossinessMap]) + + return null + } +}) + +const invertGlossinessMap = async (glossinessMap: Texture) => { + const mapData: Texture = (await createReadableTexture(glossinessMap, { canvas: true })) as Texture + const canvas = mapData.image as HTMLCanvasElement + const ctx = canvas.getContext('2d')! + ctx.globalCompositeOperation = 'difference' + ctx.fillStyle = 'white' + ctx.fillRect(0, 0, canvas.width, canvas.height) + ctx.globalCompositeOperation = 'source-over' + const invertedTexture = new CanvasTexture(canvas) + return invertedTexture +} + +export type MaterialExtensionPluginType = { id: string; uniforms: { [key: string]: any } } + +export const EEMaterialComponent = defineComponent({ + name: 'EEMaterialComponent', + jsonID: 'EE_material', + + onInit(entity) { + return {} as { + uuid: EntityUUID + name: string + prototype: string + args: { + [field: string]: { + type: string + contents: any + } + } + plugins: MaterialExtensionPluginType[] + } + }, + + onSet(entity, component, json) { + if (!json) return + if (typeof json.uuid === 'string') component.uuid.set(json.uuid) + if (typeof json.name === 'string') component.name.set(json.name) + if (typeof json.prototype === 'string') component.prototype.set(json.prototype) + if (typeof json.args === 'object') component.args.set(json.args) + if (Array.isArray(json.plugins)) component.plugins.set(json.plugins) + }, + + toJSON(entity, component) { + return { + uuid: component.uuid.value, + name: component.name.value, + prototype: component.prototype.value, + args: component.args.value, + plugins: component.plugins.value + } + }, + + reactor: () => { + const entity = useEntityContext() + const component = useComponent(entity, EEMaterialComponent) + const materialStateComponent = useComponent(entity, MaterialStateComponent) + + useEffect(() => { + setComponent(entity, MaterialDefinitionComponent, { type: component.prototype.value }) + }, [component.prototype.value]) + + useEffect(() => { + const options = getParserOptions(entity) + const resultProperties = {} as Record + const texturePromises = Object.fromEntries( + Object.entries(component.args.value).filter(([k, v]) => v.type === 'texture' && v.contents) + ) + + const reactor = startReactor(() => { + for (const [k, v] of Object.entries(component.args.value)) { + if (v.type === 'texture') { + if (v.contents) { + const texture = GLTFLoaderFunctions.useAssignTexture(options, v.contents) + useEffect(() => { + if (!texture) return + if (k === 'map') texture.colorSpace = SRGBColorSpace + resultProperties[k] = texture + delete texturePromises[k] + if (Object.keys(texturePromises).length === 0) { + const material = materialStateComponent.material.value as Material + material.setValues(resultProperties) + material.needsUpdate = true + reactor.stop() + } + }, [texture]) + } else { + useEffect(() => { + resultProperties[k] = null + }, []) + } + } else if (v.type === 'color') { + useEffect(() => { + if (v.contents !== null && !(v.contents as Color)?.isColor) { + resultProperties[k] = new Color(v.contents) + } else { + resultProperties[k] = v.contents + } + }, []) + } else { + useEffect(() => { + resultProperties[k] = v.contents + }, []) + } + } + + return null + }) + + return () => { + reactor.stop() + } + }, [materialStateComponent.material.type.value, component.args.value]) + + return null + } +}) diff --git a/packages/engine/src/gltf/MeshExtensionComponents.tsx b/packages/engine/src/gltf/MeshExtensionComponents.tsx new file mode 100644 index 0000000000..6b62ce3c1e --- /dev/null +++ b/packages/engine/src/gltf/MeshExtensionComponents.tsx @@ -0,0 +1,287 @@ +/* +CPAL-1.0 License + +The contents of this file are subject to the Common Public Attribution License +Version 1.0. (the "License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at +https://github.com/EtherealEngine/etherealengine/blob/dev/LICENSE. +The License is based on the Mozilla Public License Version 1.1, but Sections 14 +and 15 have been added to cover use of software over a computer network and +provide for limited attribution for the Original Developer. In addition, +Exhibit A has been modified to be consistent with Exhibit B. + +Software distributed under the License is distributed on an "AS IS" basis, +WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License for the +specific language governing rights and limitations under the License. + +The Original Code is Ethereal Engine. + +The Original Developer is the Initial Developer. The Initial Developer of the +Original Code is the Ethereal Engine team. + +All portions of the code written by the Ethereal Engine team are Copyright © 2021-2023 +Ethereal Engine. All Rights Reserved. +*/ + +import { + Entity, + defineComponent, + removeComponent, + setComponent, + useComponent, + useEntityContext, + useOptionalComponent +} from '@etherealengine/ecs' +import { NO_PROXY, State, none, useHookstate } from '@etherealengine/hyperflux' +import { DirectionalLightComponent, PointLightComponent, SpotLightComponent } from '@etherealengine/spatial' +import { addObjectToGroup, removeObjectFromGroup } from '@etherealengine/spatial/src/renderer/components/GroupComponent' +import { MeshComponent } from '@etherealengine/spatial/src/renderer/components/MeshComponent' +import React, { useEffect } from 'react' +import { + BufferAttribute, + Color, + InstancedBufferAttribute, + InstancedMesh, + Matrix4, + Mesh, + Object3D, + Quaternion, + Vector3 +} from 'three' +import { InstancingComponent } from '../scene/components/InstancingComponent' +import { GLTFLoaderFunctions } from './GLTFLoaderFunctions' +import { getParserOptions } from './GLTFState' + +export type KHRPunctualLight = { + color?: [number, number, number] + intensity?: number + range?: number + type: 'directional' | 'point' | 'spot' + spot?: { + innerConeAngle?: number + outerConeAngle?: number + } +} + +export const KHRLightsPunctualComponent = defineComponent({ + name: 'KHRLightsPunctualComponent', + jsonID: 'KHR_lights_punctual', + + onInit(entity) { + return {} as { + light?: number + } + }, + + onSet(entity, component, json) { + if (!json) return + if (typeof json.light === 'number') component.light.set(json.light) + }, + + reactor: () => { + const entity = useEntityContext() + const component = useComponent(entity, KHRLightsPunctualComponent) + + const options = getParserOptions(entity) + const json = options.document + const extensions: { + lights?: KHRPunctualLight[] + } = (json.extensions && json.extensions[KHRLightsPunctualComponent.jsonID]) || {} + const lightDefs = extensions.lights + const lightDef = lightDefs && component.light.value ? lightDefs[component.light.value] : undefined + + useEffect(() => { + return () => { + removeComponent(entity, DirectionalLightComponent) + removeComponent(entity, SpotLightComponent) + removeComponent(entity, PointLightComponent) + } + }, [lightDef?.type]) + + useEffect(() => { + if (!lightDef) return + + if (lightDef.type !== 'directional') return + + const color = lightDef.color + ? new Color().setRGB(lightDef.color[0], lightDef.color[1], lightDef.color[2]) + : undefined + const intensity = typeof lightDef.intensity === 'number' ? lightDef.intensity : undefined + + setComponent(entity, DirectionalLightComponent, { + color, + intensity + }) + }, [lightDef]) + + useEffect(() => { + if (!lightDef) return + + if (lightDef.type !== 'spot') return + + const color = lightDef.color + ? new Color().setRGB(lightDef.color[0], lightDef.color[1], lightDef.color[2]) + : undefined + + const intensity = typeof lightDef.intensity === 'number' ? lightDef.intensity : undefined + const range = typeof lightDef.range === 'number' ? lightDef.range : undefined + const innerConeAngle = typeof lightDef.spot?.innerConeAngle === 'number' ? lightDef.spot.innerConeAngle : 0 + const outerConeAngle = + typeof lightDef.spot?.outerConeAngle === 'number' ? lightDef.spot.outerConeAngle : Math.PI / 4.0 + + const penumbra = 1.0 - innerConeAngle / outerConeAngle + const angle = outerConeAngle + + setComponent(entity, SpotLightComponent, { + color, + intensity, + decay: 2, + range, + angle, + penumbra + }) + }, [lightDef]) + + useEffect(() => { + if (!lightDef) return + + if (lightDef.type !== 'point') return + + const color = lightDef.color + ? new Color().setRGB(lightDef.color[0], lightDef.color[1], lightDef.color[2]) + : undefined + const intensity = typeof lightDef.intensity === 'number' ? lightDef.intensity : undefined + const range = typeof lightDef.range === 'number' ? lightDef.range : undefined + + setComponent(entity, PointLightComponent, { + color, + intensity, + decay: 2, + range + }) + }, [lightDef]) + + return null + } +}) + +export const EXTMeshGPUInstancingComponent = defineComponent({ + name: 'EXTMeshGPUInstancingComponent', + jsonID: 'EXT_mesh_gpu_instancing', + + onInit(entity) { + return {} as { + attributes: Record + } + }, + + onSet(entity, component, json) { + if (!json) return + if (json.attributes) component.attributes.set(json.attributes) + }, + + reactor: () => { + const entity = useEntityContext() + const component = useComponent(entity, EXTMeshGPUInstancingComponent) + const meshComponent = useOptionalComponent(entity, MeshComponent) + + const attributes = component.attributes.get(NO_PROXY) + + const accessorsState = useHookstate({} as Record) + + useEffect(() => { + if (!meshComponent || !attributes) return + // ensure all accessors are loaded + if (Object.keys(attributes).length === 0 || Object.keys(attributes).length !== accessorsState.keys.length) return + processGPUInstancing(entity, accessorsState.get(NO_PROXY), meshComponent.get(NO_PROXY)! as Mesh) + }, [accessorsState, !!meshComponent]) + + return ( + <> + {attributes && + Object.entries(attributes).map(([attribute, accessorIndex]) => ( + + ))} + + ) + } +}) + +const MeshGPUInstancingAttributeReactor = (props: { + accessorsState: State> + entity: Entity + attribute: string + accessorIndex: number +}) => { + const { accessorsState, entity, accessorIndex } = props + const options = getParserOptions(entity) + const accessor = GLTFLoaderFunctions.useLoadAccessor(options, accessorIndex) + + useEffect(() => { + if (!accessor) return + accessorsState[props.attribute].set(accessor) + return () => { + accessorsState[props.attribute].set(none) + } + }, [accessor]) + + return null +} + +const processGPUInstancing = (entity: Entity, attributes: Record, mesh: Mesh) => { + // get any attribute to get the count + const attribute0 = Object.values(attributes)[0] + const count = attribute0.count // All attribute counts should be same + + // For Working + const m = new Matrix4() + const p = new Vector3() + const q = new Quaternion() + const s = new Vector3(1, 1, 1) + + const instancedMesh = new InstancedMesh(mesh.geometry, mesh.material, count) + for (let i = 0; i < count; i++) { + if (attributes.TRANSLATION) { + p.fromBufferAttribute(attributes.TRANSLATION, i) + } + if (attributes.ROTATION) { + q.fromBufferAttribute(attributes.ROTATION, i) + } + if (attributes.SCALE) { + s.fromBufferAttribute(attributes.SCALE, i) + } + // @TODO: Support _ID and others + instancedMesh.setMatrixAt(i, m.compose(p, q, s)) + } + + for (const attributeName in attributes) { + if (attributeName === '_COLOR_0') { + const attr = attributes[attributeName] + instancedMesh.instanceColor = new InstancedBufferAttribute(attr.array, attr.itemSize, attr.normalized) + } else if (attributeName !== 'TRANSLATION' && attributeName !== 'ROTATION' && attributeName !== 'SCALE') { + mesh.geometry.setAttribute(attributeName, attributes[attributeName]) + } + } + + // Just in case + Object3D.prototype.copy.call(instancedMesh, mesh) + + instancedMesh.frustumCulled = false + instancedMesh.instanceMatrix.needsUpdate = true + + /** @todo we really should tidy this up, and change it such that the mesh component itself handles adding and removing from group */ + removeObjectFromGroup(entity, mesh) + removeComponent(entity, MeshComponent) + setComponent(entity, MeshComponent, instancedMesh) + addObjectToGroup(entity, instancedMesh) + + setComponent(entity, InstancingComponent, { + instanceMatrix: instancedMesh.instanceMatrix + }) +} diff --git a/packages/engine/src/scene/components/ModelComponent.tsx b/packages/engine/src/scene/components/ModelComponent.tsx index fd8297b382..1dc9d7104c 100644 --- a/packages/engine/src/scene/components/ModelComponent.tsx +++ b/packages/engine/src/scene/components/ModelComponent.tsx @@ -23,35 +23,31 @@ All portions of the code written by the Ethereal Engine team are Copyright © 20 Ethereal Engine. All Rights Reserved. */ -import { QueryReactor, UUIDComponent } from '@etherealengine/ecs' +import { UUIDComponent } from '@etherealengine/ecs' import { defineComponent, getComponent, getOptionalComponent, hasComponent, setComponent, - useComponent, - useOptionalComponent + useComponent } from '@etherealengine/ecs/src/ComponentFunctions' import { Engine } from '@etherealengine/ecs/src/Engine' -import { Entity, EntityUUID } from '@etherealengine/ecs/src/Entity' +import { EntityUUID } from '@etherealengine/ecs/src/Entity' import { useEntityContext } from '@etherealengine/ecs/src/EntityFunctions' import { NO_PROXY, dispatchAction, getMutableState, getState, none, useHookstate } from '@etherealengine/hyperflux' import { CameraComponent } from '@etherealengine/spatial/src/camera/components/CameraComponent' import { RendererComponent } from '@etherealengine/spatial/src/renderer/WebGLRendererSystem' import { GroupComponent, addObjectToGroup } from '@etherealengine/spatial/src/renderer/components/GroupComponent' -import { MeshComponent } from '@etherealengine/spatial/src/renderer/components/MeshComponent' import { ObjectLayerMaskComponent } from '@etherealengine/spatial/src/renderer/components/ObjectLayerComponent' import { ObjectLayers } from '@etherealengine/spatial/src/renderer/constants/ObjectLayers' import { EntityTreeComponent, iterateEntityNode, - removeEntityNodeRecursively, - useAncestorWithComponent + removeEntityNodeRecursively } from '@etherealengine/spatial/src/transform/components/EntityTree' import { VRM } from '@pixiv/three-vrm' -import { Not } from 'bitecs' -import React, { FC, useEffect } from 'react' +import { useEffect } from 'react' import { AnimationMixer, Group, Scene } from 'three' import { useGLTF } from '../../assets/functions/resourceLoaderHooks' import { GLTF } from '../../assets/loaders/gltf/GLTFLoader' @@ -62,7 +58,7 @@ import { GLTFSnapshotState, GLTFSourceState } from '../../gltf/GLTFState' import { SceneJsonType, convertSceneJSONToGLTF } from '../../gltf/convertJsonToGLTF' import { addError, removeError } from '../functions/ErrorFunctions' import { parseGLTFModel, proxifyParentChildRelationships } from '../functions/loadGLTFModel' -import { getModelSceneID, useModelSceneID } from '../functions/loaders/ModelFunctions' +import { getModelSceneID } from '../functions/loaders/ModelFunctions' import { SourceComponent } from './SourceComponent' /** @@ -70,7 +66,7 @@ import { SourceComponent } from './SourceComponent' */ export const ModelComponent = defineComponent({ name: 'ModelComponent', - jsonID: 'EE_model', + // jsonID: 'EE_model', onInit: (entity) => { return { @@ -245,43 +241,3 @@ function ModelReactor() { return null } - -/** - * Returns true if the entity has a model component or a mesh component that is not a child of model - * @param entity - * @returns {boolean} - */ -export const useHasModelOrIndependentMesh = (entity: Entity) => { - const hasModel = !!useOptionalComponent(entity, ModelComponent) - const isChildOfModel = !!useAncestorWithComponent(entity, ModelComponent) - const hasMesh = !!useOptionalComponent(entity, MeshComponent) - - return hasModel || (hasMesh && !isChildOfModel) -} - -export const MeshOrModelQuery = (props: { ChildReactor: FC<{ entity: Entity; rootEntity: Entity }> }) => { - const ModelReactor = () => { - const entity = useEntityContext() - const sceneInstanceID = useModelSceneID(entity) - const childEntities = useHookstate(SourceComponent.entitiesBySourceState[sceneInstanceID]) - return ( - <> - {childEntities.value?.map((childEntity) => ( - - ))} - - ) - } - - const MeshReactor = () => { - const entity = useEntityContext() - return - } - - return ( - <> - - - - ) -} diff --git a/packages/engine/src/scene/components/ObjectGridSnapComponent.ts b/packages/engine/src/scene/components/ObjectGridSnapComponent.ts index b4a1d15acb..b279edcd84 100644 --- a/packages/engine/src/scene/components/ObjectGridSnapComponent.ts +++ b/packages/engine/src/scene/components/ObjectGridSnapComponent.ts @@ -47,7 +47,7 @@ import { TransformComponent } from '@etherealengine/spatial/src/transform/compon import { computeTransformMatrix } from '@etherealengine/spatial/src/transform/systems/TransformSystem' import { useEffect } from 'react' import { Box3, BufferGeometry, ColorRepresentation, LineBasicMaterial, Matrix4, Mesh, Quaternion, Vector3 } from 'three' -import { ModelComponent } from './ModelComponent' +import { GLTFComponent } from '../../gltf/GLTFComponent' function createBBoxGridGeometry(matrixWorld: Matrix4, bbox: Box3, density: number): BufferGeometry { const lineSegmentList: Vector3[] = [] @@ -183,11 +183,12 @@ export const ObjectGridSnapComponent = defineComponent({ reactor: () => { const entity = useEntityContext() const engineState = useState(getMutableState(EngineState)) - const modelComponent = useComponent(entity, ModelComponent) + const modelComponent = useComponent(entity, GLTFComponent) + const modelLoaded = modelComponent.progress.value === 100 const snapComponent = useComponent(entity, ObjectGridSnapComponent) useEffect(() => { - if (!modelComponent.scene.value) return + if (!modelLoaded) return const originalPosition = new Vector3() const originalRotation = new Quaternion() const originalScale = new Vector3() @@ -227,7 +228,7 @@ export const ObjectGridSnapComponent = defineComponent({ iterateEntityNode(entity, computeTransformMatrix, (childEntity) => hasComponent(childEntity, TransformComponent)) //set bounding box in component snapComponent.bbox.set(bbox) - }, [modelComponent.scene]) + }, [modelLoaded]) useEffect(() => { if (!engineState.isEditing.value) return diff --git a/packages/engine/src/scene/functions/loadGLTFModel.ts b/packages/engine/src/scene/functions/loadGLTFModel.ts index eff4b293ee..356bc123fd 100644 --- a/packages/engine/src/scene/functions/loadGLTFModel.ts +++ b/packages/engine/src/scene/functions/loadGLTFModel.ts @@ -50,8 +50,8 @@ import { EntityTreeComponent } from '@etherealengine/spatial/src/transform/compo import { computeTransformMatrix } from '@etherealengine/spatial/src/transform/systems/TransformSystem' import { ColliderComponent } from '@etherealengine/spatial/src/physics/components/ColliderComponent' -import { BoneComponent } from '../../avatar/components/BoneComponent' -import { SkinnedMeshComponent } from '../../avatar/components/SkinnedMeshComponent' +import { BoneComponent } from '@etherealengine/spatial/src/renderer/components/BoneComponent' +import { SkinnedMeshComponent } from '@etherealengine/spatial/src/renderer/components/SkinnedMeshComponent' import { GLTFLoadedComponent } from '../components/GLTFLoadedComponent' import { InstancingComponent } from '../components/InstancingComponent' import { ModelComponent } from '../components/ModelComponent' diff --git a/packages/engine/src/scene/functions/loaders/ModelFunctions.ts b/packages/engine/src/scene/functions/loaders/ModelFunctions.ts index ee1227323b..ebd4398143 100644 --- a/packages/engine/src/scene/functions/loaders/ModelFunctions.ts +++ b/packages/engine/src/scene/functions/loaders/ModelFunctions.ts @@ -27,12 +27,7 @@ import { DracoOptions } from '@gltf-transform/functions' import { Material, Texture } from 'three' import { UUIDComponent } from '@etherealengine/ecs' -import { - getComponent, - getOptionalComponent, - hasComponent, - useOptionalComponent -} from '@etherealengine/ecs/src/ComponentFunctions' +import { getComponent, getOptionalComponent, hasComponent } from '@etherealengine/ecs/src/ComponentFunctions' import { Entity } from '@etherealengine/ecs/src/Entity' import { MeshComponent } from '@etherealengine/spatial/src/renderer/components/MeshComponent' import { iterateEntityNode } from '@etherealengine/spatial/src/transform/components/EntityTree' @@ -53,13 +48,6 @@ export function getModelSceneID(entity: Entity): string { return getComponent(entity, UUIDComponent) + '-' + getComponent(entity, ModelComponent).src } -export function useModelSceneID(entity: Entity): string { - const uuid = useOptionalComponent(entity, UUIDComponent)?.value - const model = useOptionalComponent(entity, ModelComponent)?.value - if (!uuid || !model) return '' - return uuid + '-' + model.src -} - export function getModelResources(entity: Entity, defaultParms: ModelTransformParameters): ResourceTransforms { const model = getOptionalComponent(entity, ModelComponent) if (!model?.scene) return { geometries: [], images: [] } diff --git a/packages/engine/src/scene/functions/migrateOldColliders.ts b/packages/engine/src/scene/functions/migrateOldColliders.ts index 6be7a9d019..258b09554b 100644 --- a/packages/engine/src/scene/functions/migrateOldColliders.ts +++ b/packages/engine/src/scene/functions/migrateOldColliders.ts @@ -27,17 +27,17 @@ import { ColliderComponent } from '@etherealengine/spatial/src/physics/component import { RigidBodyComponent } from '@etherealengine/spatial/src/physics/components/RigidBodyComponent' import { TriggerComponent } from '@etherealengine/spatial/src/physics/components/TriggerComponent' -import { ModelComponent } from '../components/ModelComponent' import { ComponentJsonType, EntityJsonType } from '../types/SceneTypes' const oldColliderJSONID = 'collider' +const oldModelJSONID = 'EE_model' /** * Converts old ColliderComponent to RigidbodyComponent, new ColliderComponent and TriggerComponent */ export const migrateOldColliders = (oldJSON: EntityJsonType) => { /** models need to be manually converted in the studio */ - const hasModel = Object.values(oldJSON.components).some((comp) => comp.name === ModelComponent.jsonID) + const hasModel = Object.values(oldJSON.components).some((comp) => comp.name === oldModelJSONID) if (hasModel) return const newComponents = [] as ComponentJsonType[] diff --git a/packages/engine/src/scene/materials/functions/materialSourcingFunctions.ts b/packages/engine/src/scene/materials/functions/materialSourcingFunctions.ts index 7eee24fcbb..73f6f99248 100644 --- a/packages/engine/src/scene/materials/functions/materialSourcingFunctions.ts +++ b/packages/engine/src/scene/materials/functions/materialSourcingFunctions.ts @@ -26,12 +26,12 @@ Ethereal Engine. All Rights Reserved. import { Entity, EntityUUID, getComponent, hasComponent } from '@etherealengine/ecs' import { MaterialInstanceComponent } from '@etherealengine/spatial/src/renderer/materials/MaterialComponent' +import { GLTFComponent } from '../../../gltf/GLTFComponent' import { SourceComponent } from '../../components/SourceComponent' -import { getModelSceneID } from '../../functions/loaders/ModelFunctions' /**Gets all materials used by child and self entity */ export const getMaterialsFromScene = (source: Entity) => { - const sceneInstanceID = getModelSceneID(source) + const sceneInstanceID = GLTFComponent.getInstanceID(source) const childEntities = SourceComponent.entitiesBySource[sceneInstanceID] ?? ([] as Entity[]) childEntities.push(source) const materials = {} as Record diff --git a/packages/engine/src/scene/materials/systems/MaterialLibrarySystem.tsx b/packages/engine/src/scene/materials/systems/MaterialLibrarySystem.tsx index fb427ab80f..9fa9c29dd7 100644 --- a/packages/engine/src/scene/materials/systems/MaterialLibrarySystem.tsx +++ b/packages/engine/src/scene/materials/systems/MaterialLibrarySystem.tsx @@ -23,21 +23,9 @@ All portions of the code written by the Ethereal Engine team are Copyright © 20 Ethereal Engine. All Rights Reserved. */ -import React, { ReactElement, useEffect } from 'react' +import { useEffect } from 'react' -import { - EntityUUID, - getComponent, - getOptionalComponent, - PresentationSystemGroup, - QueryReactor, - removeEntity, - setComponent, - UndefinedEntity, - useComponent, - useEntityContext, - useOptionalComponent -} from '@etherealengine/ecs' +import { PresentationSystemGroup, UndefinedEntity } from '@etherealengine/ecs' import { defineSystem } from '@etherealengine/ecs/src/SystemFunctions' import { MaterialPrototypeDefinition, @@ -45,22 +33,13 @@ import { } from '@etherealengine/spatial/src/renderer/materials/MaterialComponent' import { createAndAssignMaterial, - createMaterialPrototype, - materialPrototypeMatches, - setMeshMaterial, - updateMaterialPrototype + createMaterialPrototype } from '@etherealengine/spatial/src/renderer/materials/materialFunctions' -import { MeshComponent } from '@etherealengine/spatial/src/renderer/components/MeshComponent' -import { - MaterialInstanceComponent, - MaterialStateComponent -} from '@etherealengine/spatial/src/renderer/materials/MaterialComponent' -import { isArray } from 'lodash' -import { Material, MeshBasicMaterial } from 'three' -import { SourceComponent } from '../../components/SourceComponent' +import { MaterialStateComponent } from '@etherealengine/spatial/src/renderer/materials/MaterialComponent' +import { MeshBasicMaterial } from 'three' -const reactor = (): ReactElement => { +const reactor = () => { useEffect(() => { MaterialPrototypeDefinitions.map((prototype: MaterialPrototypeDefinition, uuid) => createMaterialPrototype(prototype) @@ -70,68 +49,6 @@ const reactor = (): ReactElement => { createAndAssignMaterial(UndefinedEntity, fallbackMaterial) }, []) - return ( - <> - - - - - ) -} - -const MeshReactor = () => { - const entity = useEntityContext() - const materialComponent = useOptionalComponent(entity, MaterialInstanceComponent) - const meshComponent = useComponent(entity, MeshComponent) - - const createAndSourceMaterial = (material: Material) => { - const materialEntity = createAndAssignMaterial(entity, material) - const source = getOptionalComponent(entity, SourceComponent) - if (source) setComponent(materialEntity, SourceComponent, source) - } - - useEffect(() => { - if (materialComponent) return - const material = meshComponent.material.value as Material - if (!isArray(material)) createAndSourceMaterial(material) - else for (const mat of material) createAndSourceMaterial(mat) - }, []) - return null -} - -const MaterialEntityReactor = () => { - const entity = useEntityContext() - const materialComponent = useComponent(entity, MaterialStateComponent) - useEffect(() => { - if (!materialComponent.instances.value!) return - for (const sourceEntity of materialComponent.instances.value) { - const sourceComponent = getComponent(sourceEntity, SourceComponent) - if (!SourceComponent.entitiesBySource[sourceComponent]) return - for (const entity of SourceComponent.entitiesBySource[getComponent(sourceEntity, SourceComponent)]) { - const uuid = getOptionalComponent(entity, MaterialInstanceComponent)?.uuid as EntityUUID[] | undefined - if (uuid) setMeshMaterial(entity, uuid) - } - } - }, [materialComponent.material]) - - useEffect(() => { - if (materialComponent.prototypeEntity.value && !materialPrototypeMatches(entity)) updateMaterialPrototype(entity) - }, [materialComponent.prototypeEntity]) - - useEffect(() => { - if (materialComponent.instances.value?.length === 0) removeEntity(entity) - }, [materialComponent.instances]) - - return null -} - -const MaterialInstanceReactor = () => { - const entity = useEntityContext() - const materialComponent = useComponent(entity, MaterialInstanceComponent) - const uuid = materialComponent.uuid - useEffect(() => { - if (uuid.value) setMeshMaterial(entity, uuid.value as EntityUUID[]) - }, [materialComponent.uuid]) return null } diff --git a/packages/engine/src/scene/systems/SceneObjectSystem.tsx b/packages/engine/src/scene/systems/SceneObjectSystem.tsx index 25d82268b0..3b88e3d7b9 100644 --- a/packages/engine/src/scene/systems/SceneObjectSystem.tsx +++ b/packages/engine/src/scene/systems/SceneObjectSystem.tsx @@ -73,12 +73,11 @@ import { MaterialStateComponent } from '@etherealengine/spatial/src/renderer/materials/MaterialComponent' import { createAndAssignMaterial } from '@etherealengine/spatial/src/renderer/materials/materialFunctions' +import { GLTFComponent } from '../../gltf/GLTFComponent' import { EnvmapComponent } from '../components/EnvmapComponent' -import { ModelComponent } from '../components/ModelComponent' import { ShadowComponent } from '../components/ShadowComponent' import { SourceComponent } from '../components/SourceComponent' import { UpdatableCallback, UpdatableComponent } from '../components/UpdatableComponent' -import { getModelSceneID, useModelSceneID } from '../functions/loaders/ModelFunctions' const disposeMaterial = (material: Material) => { for (const [key, val] of Object.entries(material) as [string, Texture][]) { @@ -168,8 +167,8 @@ function SceneObjectReactor(props: { entity: Entity; obj: Object3D }) { const forceBasicMaterials = useHookstate(renderState.forceBasicMaterials) useEffect(() => { - const source = hasComponent(entity, ModelComponent) - ? getModelSceneID(entity) + const source = hasComponent(entity, GLTFComponent) + ? GLTFComponent.getInstanceID(entity) : getOptionalComponent(entity, SourceComponent) return () => { ResourceManager.unloadObj(obj, source) @@ -210,8 +209,8 @@ const execute = () => { const ModelEntityReactor = () => { const entity = useEntityContext() - const modelSceneID = useModelSceneID(entity) - const childEntities = useHookstate(SourceComponent.entitiesBySourceState[modelSceneID]) + const modelInstanceID = GLTFComponent.useInstanceID(entity) + const childEntities = useHookstate(SourceComponent.entitiesBySourceState[modelInstanceID]) return ( <> @@ -275,7 +274,7 @@ const ChildReactor = (props: { entity: Entity; parentEntity: Entity }) => { const reactor = () => { return ( <> - + ) diff --git a/packages/engine/src/scene/systems/ShadowSystem.tsx b/packages/engine/src/scene/systems/ShadowSystem.tsx index 63a498a5c5..175cbf0751 100644 --- a/packages/engine/src/scene/systems/ShadowSystem.tsx +++ b/packages/engine/src/scene/systems/ShadowSystem.tsx @@ -91,8 +91,8 @@ import { EngineState } from '@etherealengine/spatial/src/EngineState' import { RenderModes } from '@etherealengine/spatial/src/renderer/constants/RenderModes' import { createDisposable } from '@etherealengine/spatial/src/resources/resourceHooks' import { useTexture } from '../../assets/functions/resourceLoaderHooks' +import { useHasModelOrIndependentMesh } from '../../gltf/GLTFComponent' import { DropShadowComponent } from '../components/DropShadowComponent' -import { useHasModelOrIndependentMesh } from '../components/ModelComponent' import { RenderSettingsComponent } from '../components/RenderSettingsComponent' import { ShadowComponent } from '../components/ShadowComponent' import { SceneObjectSystem } from './SceneObjectSystem' diff --git a/packages/engine/src/avatar/components/BoneComponent.ts b/packages/spatial/src/renderer/components/BoneComponent.ts similarity index 100% rename from packages/engine/src/avatar/components/BoneComponent.ts rename to packages/spatial/src/renderer/components/BoneComponent.ts diff --git a/packages/spatial/src/renderer/components/SkinnedMeshComponent.ts b/packages/spatial/src/renderer/components/SkinnedMeshComponent.ts new file mode 100644 index 0000000000..5e704b34ba --- /dev/null +++ b/packages/spatial/src/renderer/components/SkinnedMeshComponent.ts @@ -0,0 +1,173 @@ +/* +CPAL-1.0 License + +The contents of this file are subject to the Common Public Attribution License +Version 1.0. (the "License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at +https://github.com/EtherealEngine/etherealengine/blob/dev/LICENSE. +The License is based on the Mozilla Public License Version 1.1, but Sections 14 +and 15 have been added to cover use of software over a computer network and +provide for limited attribution for the Original Developer. In addition, +Exhibit A has been modified to be consistent with Exhibit B. + +Software distributed under the License is distributed on an "AS IS" basis, +WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License for the +specific language governing rights and limitations under the License. + +The Original Code is Ethereal Engine. + +The Original Developer is the Initial Developer. The Initial Developer of the +Original Code is the Ethereal Engine team. + +All portions of the code written by the Ethereal Engine team are Copyright © 2021-2023 +Ethereal Engine. All Rights Reserved. +*/ + +import { + BufferGeometry, + Color, + Float32BufferAttribute, + LineBasicMaterial, + LineSegments, + Matrix4, + SkinnedMesh, + Vector3 +} from 'three' + +import { createEntity, removeEntity, useEntityContext } from '@etherealengine/ecs' +import { + defineComponent, + getComponent, + getOptionalComponent, + hasComponent, + setComponent, + useComponent, + useOptionalComponent +} from '@etherealengine/ecs/src/ComponentFunctions' +import { getMutableState, useHookstate } from '@etherealengine/hyperflux' +import { NameComponent } from '@etherealengine/spatial/src/common/NameComponent' +import { RendererState } from '@etherealengine/spatial/src/renderer/RendererState' +import { addObjectToGroup } from '@etherealengine/spatial/src/renderer/components/GroupComponent' +import { VisibleComponent, setVisibleComponent } from '@etherealengine/spatial/src/renderer/components/VisibleComponent' +import { ComputedTransformComponent } from '@etherealengine/spatial/src/transform/components/ComputedTransformComponent' +import { EntityTreeComponent } from '@etherealengine/spatial/src/transform/components/EntityTree' +import { useEffect } from 'react' +import { TransformComponent } from '../../transform/components/TransformComponent' +import { ObjectLayers } from '../constants/ObjectLayers' +import { BoneComponent } from './BoneComponent' +import { setObjectLayers } from './ObjectLayerComponent' + +export const SkinnedMeshComponent = defineComponent({ + name: 'SkinnedMeshComponent', + + onInit: (entity) => null! as SkinnedMesh, + + onSet: (entity, component, mesh: SkinnedMesh) => { + if (!mesh || !mesh.isSkinnedMesh) throw new Error('SkinnedMeshComponent: Invalid skinned mesh') + component.set(mesh) + }, + + reactor: function () { + const entity = useEntityContext() + const component = useComponent(entity, SkinnedMeshComponent) + const debugEnabled = useHookstate(getMutableState(RendererState).avatarDebug) + const visible = useOptionalComponent(entity, VisibleComponent) + + useEffect(() => { + if (!visible?.value || !debugEnabled.value) return + + const root = getComponent(entity, SkinnedMeshComponent) + const bones = root.skeleton.bones //getBoneList(entity) + + const geometry = new BufferGeometry() + + const vertices = [] as number[] + const colors = [] as number[] + + const color1 = new Color(0, 0, 1) + const color2 = new Color(0, 1, 0) + + for (let i = 0; i < bones.length; i++) { + const bone = bones[i] + + const boneParentEntity = getComponent(bone.entity, EntityTreeComponent).parentEntity + const boneParentComponent = hasComponent(boneParentEntity, BoneComponent) + + if (boneParentComponent) { + vertices.push(0, 0, 0) + vertices.push(0, 0, 0) + colors.push(color1.r, color1.g, color1.b) + colors.push(color2.r, color2.g, color2.b) + } + } + + geometry.setAttribute('position', new Float32BufferAttribute(vertices, 3)) + geometry.setAttribute('color', new Float32BufferAttribute(colors, 3)) + + const material = new LineBasicMaterial({ + vertexColors: true, + depthTest: false, + depthWrite: false, + toneMapped: false, + transparent: true + }) + + const line = new LineSegments(geometry, material) + + line.frustumCulled = false + line.name = `Skinned Mesh Helper For: ${entity}` + + const helperEntity = createEntity() + setVisibleComponent(helperEntity, true) + addObjectToGroup(helperEntity, line) + setComponent(helperEntity, NameComponent, line.name) + setObjectLayers(line, ObjectLayers.AvatarHelper) + setComponent(helperEntity, EntityTreeComponent, { parentEntity: entity }) + + setComponent(helperEntity, ComputedTransformComponent, { + referenceEntities: [entity], + computeFunction: () => { + const position = geometry.getAttribute('position') + + _matrixWorldInv.copy(root.matrixWorld).invert() + + for (let i = 0, j = 0; i < bones.length; i++) { + const bone = bones[i] + + const boneParentEntity = getComponent(bone.entity, EntityTreeComponent).parentEntity + const boneParentComponent = getOptionalComponent(boneParentEntity, BoneComponent) + + if (boneParentComponent) { + _boneMatrix.multiplyMatrices(_matrixWorldInv, bone.matrixWorld) + _vector.setFromMatrixPosition(_boneMatrix) + position.setXYZ(j, _vector.x, _vector.y, _vector.z) + + _boneMatrix.multiplyMatrices( + _matrixWorldInv, + getComponent(boneParentEntity, TransformComponent).matrixWorld + ) + _vector.setFromMatrixPosition(_boneMatrix) + position.setXYZ(j + 1, _vector.x, _vector.y, _vector.z) + + j += 2 + } + } + + geometry.getAttribute('position').needsUpdate = true + } + }) + + return () => { + removeEntity(helperEntity) + geometry.dispose() + material.dispose() + } + }, [visible, debugEnabled, component.skeleton.value]) + + return null + } +}) + +const _vector = /*@__PURE__*/ new Vector3() +const _boneMatrix = /*@__PURE__*/ new Matrix4() +const _matrixWorldInv = /*@__PURE__*/ new Matrix4() diff --git a/packages/spatial/src/renderer/materials/MaterialComponent.ts b/packages/spatial/src/renderer/materials/MaterialComponent.tsx similarity index 75% rename from packages/spatial/src/renderer/materials/MaterialComponent.ts rename to packages/spatial/src/renderer/materials/MaterialComponent.tsx index ada1a7f4cb..365d14eb2a 100644 --- a/packages/spatial/src/renderer/materials/MaterialComponent.ts +++ b/packages/spatial/src/renderer/materials/MaterialComponent.tsx @@ -31,15 +31,19 @@ import { defineComponent, defineQuery, getComponent, - getMutableComponent + getMutableComponent, + useComponent, + useEntityContext } from '@etherealengine/ecs' import { Entity, EntityUUID, UndefinedEntity } from '@etherealengine/ecs/src/Entity' import { PluginType } from '@etherealengine/spatial/src/common/functions/OnBeforeCompilePlugin' +import React, { useEffect } from 'react' import { v4 as uuidv4 } from 'uuid' +import { MeshComponent } from '../components/MeshComponent' import { NoiseOffsetPlugin } from './constants/plugins/NoiseOffsetPlugin' import { TransparencyDitheringPlugin } from './constants/plugins/TransparencyDitheringComponent' -import { setMeshMaterial } from './materialFunctions' +import { materialPrototypeMatches, setMeshMaterial, updateMaterialPrototype } from './materialFunctions' import MeshBasicMaterial from './prototypes/MeshBasicMaterial.mat' import MeshLambertMaterial from './prototypes/MeshLambertMaterial.mat' import MeshMatcapMaterial from './prototypes/MeshMatcapMaterial.mat' @@ -115,6 +119,17 @@ export const MaterialStateComponent = defineComponent({ for (const instanceEntity of materialComponent.instances) { setMeshMaterial(instanceEntity, getComponent(instanceEntity, MaterialInstanceComponent).uuid) } + }, + + reactor: () => { + const entity = useEntityContext() + const materialComponent = useComponent(entity, MaterialStateComponent) + + useEffect(() => { + if (materialComponent.prototypeEntity.value && !materialPrototypeMatches(entity)) updateMaterialPrototype(entity) + }, [materialComponent.prototypeEntity]) + + return null } }) @@ -136,9 +151,53 @@ export const MaterialInstanceComponent = defineComponent({ if (materialComponent.instances.value) materialComponent.instances.set(materialComponent.instances.value.filter((instance) => instance !== entity)) } + }, + reactor: () => { + const entity = useEntityContext() + const materialComponent = useComponent(entity, MaterialInstanceComponent) + + if (materialComponent.uuid.value.length === 0) return null + + if (materialComponent.uuid.value.length > 1) + return ( + <> + {materialComponent.uuid.value.map((uuid, index) => ( + + ))} + + ) + + return ( + + ) } }) +const MaterialInstanceSubReactor = (props: { array: boolean; uuid: EntityUUID; entity: Entity; index: number }) => { + const { uuid, entity, index } = props + const materialStateEntity = UUIDComponent.useEntityByUUID(uuid) + const materialStateComponent = useComponent(materialStateEntity, MaterialStateComponent) + const meshComponent = useComponent(entity, MeshComponent) + + useEffect(() => { + const material = getComponent(materialStateEntity, MaterialStateComponent).material + if (props.array) { + if (!Array.isArray(meshComponent.material.value)) meshComponent.material.set([]) + meshComponent.material[index].set(material) + } else { + meshComponent.material.set(material) + } + }, [materialStateComponent.material]) + + return null +} + export const MaterialPrototypeComponent = defineComponent({ name: 'MaterialPrototypeComponent', onInit: (entity) => { diff --git a/packages/spatial/src/threejsPatches.ts b/packages/spatial/src/threejsPatches.ts index 604b6907e3..1fe4b217a7 100644 --- a/packages/spatial/src/threejsPatches.ts +++ b/packages/spatial/src/threejsPatches.ts @@ -24,13 +24,29 @@ Ethereal Engine. All Rights Reserved. */ import * as THREE from 'three' -import { Euler, Matrix4, Object3D, Quaternion, Scene, SkinnedMesh, Vector2, Vector3, Vector4 } from 'three' +import { + Euler, + Matrix4, + Object3D, + PropertyBinding, + Quaternion, + Scene, + SkinnedMesh, + Vector2, + Vector3, + Vector4 +} from 'three' import { isClient } from '@etherealengine/common/src/utils/getEnvironment' import { Object3DUtils } from '@etherealengine/common/src/utils/Object3DUtils' -import { Entity } from '@etherealengine/ecs' +import { Entity, getComponent, getOptionalComponent, hasComponent } from '@etherealengine/ecs' import { overrideOnBeforeCompile } from './common/functions/OnBeforeCompilePlugin' +import { BoneComponent } from './renderer/components/BoneComponent' +import { MeshComponent } from './renderer/components/MeshComponent' +import { Object3DComponent } from './renderer/components/Object3DComponent' +import { SkinnedMeshComponent } from './renderer/components/SkinnedMeshComponent' +import { EntityTreeComponent } from './transform/components/EntityTree' //@ts-ignore Vector3.prototype.toJSON = function () { @@ -214,6 +230,88 @@ SkinnedMesh.prototype.applyBoneTransform = function (index, vector) { return vector.applyMatrix4(this.bindMatrixInverse) } +PropertyBinding.findNode = function (root: SkinnedMesh, nodeName: string | number) { + if ( + nodeName === undefined || + nodeName === '' || + nodeName === '.' || + nodeName === -1 || + nodeName === root.name || + nodeName === root.uuid + ) { + return root + } + + // search into skeleton bones. + if (root.skeleton) { + const bone = root.skeleton.getBoneByName(nodeName as string) + + if (bone !== undefined) { + return bone + } + } + + const entity = root.entity + if (entity) { + if (!hasComponent(entity, EntityTreeComponent)) return null + + const children = getComponent(entity, EntityTreeComponent).children + + // search into node subtree. + const searchEntitySubtree = function (children: Entity[]) { + for (let i = 0; i < children.length; i++) { + const entity = children[i] + const childNode = + getOptionalComponent(entity, BoneComponent) ?? + getOptionalComponent(entity, MeshComponent) ?? + getOptionalComponent(entity, SkinnedMeshComponent) ?? + getOptionalComponent(entity, Object3DComponent)! + + if (childNode && (childNode.name === nodeName || childNode.uuid === nodeName)) { + return childNode + } + + const result = searchEntitySubtree(getComponent(entity, EntityTreeComponent).children) + + if (result) return result + } + + return null + } + + const subTreeNode = searchEntitySubtree(children) + + if (subTreeNode) { + return subTreeNode + } + } + + // fallback to three hierarchy for non-ecs hierarchy (normalize vrm rigs) + const searchNodeSubtree = function (children) { + for (let i = 0; i < children.length; i++) { + const childNode = children[i] + + if (childNode.name === nodeName || childNode.uuid === nodeName) { + return childNode + } + + const result = searchNodeSubtree(childNode.children) + + if (result) return result + } + + return null + } + + const subTreeNode = searchNodeSubtree(root.children) + + if (subTreeNode) { + return subTreeNode + } + + return null +} + overrideOnBeforeCompile() globalThis.THREE = { ...THREE } as any diff --git a/packages/spatial/src/transform/components/EntityTree.tsx b/packages/spatial/src/transform/components/EntityTree.tsx index 41feecb691..43dafcfa8b 100644 --- a/packages/spatial/src/transform/components/EntityTree.tsx +++ b/packages/spatial/src/transform/components/EntityTree.tsx @@ -291,6 +291,34 @@ export function getAncestorWithComponent( return result } +const inlineMatchesQuery = (entity: Entity, components: ComponentType[]) => + components.map((c) => hasComponent(entity, c)).filter((c) => !!c).length === components.length + +/** + * Returns the closest ancestor of an entity that has the given component by walking up the entity tree + * @param entity Entity to start from + * @param component Component to search for + * @param closest (default true) - whether to return the closest ancestor or the furthest ancestor + * @param includeSelf (default true) - whether to include the entity itself in the search + * @returns + */ +export function getAncestorWithComponents( + entity: Entity, + components: ComponentType, + closest = true, + includeSelf = true +): Entity { + let result = UndefinedEntity + if (includeSelf && closest && inlineMatchesQuery(entity, components)) return entity + traverseEntityNodeParent(entity, (parent) => { + if (closest && result) return + if (inlineMatchesQuery(parent, components)) { + result = parent + } + }) + return result +} + /** * Finds the index of an entity tree node using entity. * This function is useful for node which is not contained in array but can have same entity as one of array elements @@ -376,6 +404,7 @@ export function useTreeQuery(entity: Entity) { /** * Returns the closest ancestor of an entity that has a component + * @deprecated use useAncestorWithComponents instead * @todo maybe extend this to be a list of components? * @todo maybe extend this or write an alternative to get the furthest ancestor with component? * @param entity @@ -419,6 +448,52 @@ export function useAncestorWithComponent(entity: Entity, component: ComponentTyp return result.value } +/** + * Returns the closest ancestor of an entity that has a component + * @todo maybe extend this to be a list of components? + * @todo maybe extend this or write an alternative to get the furthest ancestor with component? + * @param entity + * @param component + * @param closest + * @returns + */ +export function useAncestorWithComponents(entity: Entity, components: ComponentType[]) { + const result = useHookstate(() => getAncestorWithComponents(entity, components)) + + useImmediateEffect(() => { + let unmounted = false + const ParentSubReactor = (props: { entity: Entity }) => { + const tree = useOptionalComponent(props.entity, EntityTreeComponent) + const matchesQuery = + components.map((c) => useOptionalComponent(props.entity, c)).filter((c) => !!c).length === components.length + + useLayoutEffect(() => { + if (!matchesQuery) return + result.set(props.entity) + return () => { + if (!unmounted) result.set(UndefinedEntity) + } + }, [tree?.parentEntity?.value, matchesQuery]) + + if (matchesQuery) return null + + if (!tree?.parentEntity?.value) return null + + return + } + + const root = startReactor(function useQueryReactor() { + return + }) + return () => { + unmounted = true + root.stop() + } + }, [entity, components.map((c) => c.name).join(',')]) + + return result.value +} + /** * @todo - return an array of entities that have the component *