diff --git a/RealityMixer.xcodeproj/project.pbxproj b/RealityMixer.xcodeproj/project.pbxproj index f8da171..526942b 100644 --- a/RealityMixer.xcodeproj/project.pbxproj +++ b/RealityMixer.xcodeproj/project.pbxproj @@ -44,9 +44,6 @@ A328D8B4253C8C4000B90E62 /* MixedRealityConnectionViewController.swift in Sources */ = {isa = PBXBuildFile; fileRef = A328D8B3253C8C4000B90E62 /* MixedRealityConnectionViewController.swift */; }; A328D8B9253C8C4300B90E62 /* Assets.xcassets in Resources */ = {isa = PBXBuildFile; fileRef = A328D8B8253C8C4300B90E62 /* Assets.xcassets */; }; A328D8BC253C8C4300B90E62 /* LaunchScreen.storyboard in Resources */ = {isa = PBXBuildFile; fileRef = A328D8BA253C8C4300B90E62 /* LaunchScreen.storyboard */; }; - A328D960253C959500B90E62 /* libbz2.tbd in Frameworks */ = {isa = PBXBuildFile; fileRef = A328D95F253C958D00B90E62 /* libbz2.tbd */; }; - A328D963253C95A000B90E62 /* libz.tbd in Frameworks */ = {isa = PBXBuildFile; fileRef = A328D962253C959B00B90E62 /* libz.tbd */; }; - A328D966253C95AD00B90E62 /* libiconv.tbd in Frameworks */ = {isa = PBXBuildFile; fileRef = A328D965253C95A700B90E62 /* libiconv.tbd */; }; A328D969253C95B700B90E62 /* VideoToolbox.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = A328D968253C95B700B90E62 /* VideoToolbox.framework */; }; A328D96B253C95BD00B90E62 /* AudioToolbox.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = A328D96A253C95BD00B90E62 /* AudioToolbox.framework */; }; A328D96D253C95C300B90E62 /* CoreMedia.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = A328D96C253C95C200B90E62 /* CoreMedia.framework */; }; @@ -56,8 +53,6 @@ A328D98E253CF2EE00B90E62 /* ARKit.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = A328D98D253CF2EE00B90E62 /* ARKit.framework */; }; A328D990253CF2F400B90E62 /* SceneKit.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = A328D98F253CF2F400B90E62 /* SceneKit.framework */; }; A32A3D0226CB0A9A0093FAC5 /* AudioManager.swift in Sources */ = {isa = PBXBuildFile; fileRef = A32A3D0126CB0A990093FAC5 /* AudioManager.swift */; }; - A338B35A265ADFFE00099827 /* UserDefaultsStorage.swift in Sources */ = {isa = PBXBuildFile; fileRef = A338B359265ADFFE00099827 /* UserDefaultsStorage.swift */; }; - A338B35D265AE1E300099827 /* ChromaKeyConfiguration.swift in Sources */ = {isa = PBXBuildFile; fileRef = A338B35C265AE1E300099827 /* ChromaKeyConfiguration.swift */; }; A338B360265B250100099827 /* ChromaKeyConfigurationViewController.swift in Sources */ = {isa = PBXBuildFile; fileRef = A338B35E265B250100099827 /* ChromaKeyConfigurationViewController.swift */; }; A338B361265B250100099827 /* ChromaKeyConfigurationViewController.xib in Resources */ = {isa = PBXBuildFile; fileRef = A338B35F265B250100099827 /* ChromaKeyConfigurationViewController.xib */; }; A338B363265B25E500099827 /* ARKitHelpers.swift in Sources */ = {isa = PBXBuildFile; fileRef = A338B362265B25E500099827 /* ARKitHelpers.swift */; }; @@ -68,18 +63,49 @@ A3798AE125D9F6CA00B3BFBC /* CameraPoseSender.swift in Sources */ = {isa = PBXBuildFile; fileRef = A3798AE025D9F6CA00B3BFBC /* CameraPoseSender.swift */; }; A3837F6D263A239F0053951E /* next.mp3 in Resources */ = {isa = PBXBuildFile; fileRef = A3837F6B263A239F0053951E /* next.mp3 */; }; A3837F6E263A239F0053951E /* reset.mp3 in Resources */ = {isa = PBXBuildFile; fileRef = A3837F6C263A239F0053951E /* reset.mp3 */; }; - A39033ED256859A8004CB638 /* Shaders.swift in Sources */ = {isa = PBXBuildFile; fileRef = A39033EC256859A8004CB638 /* Shaders.swift */; }; A3A50C0725F4945700413E01 /* SwiftlyAttributedStrings in Frameworks */ = {isa = PBXBuildFile; productRef = A3A50C0625F4945700413E01 /* SwiftlyAttributedStrings */; }; A3A50C0C25F57AB700413E01 /* ProjectionPickerViewController.swift in Sources */ = {isa = PBXBuildFile; fileRef = A3A50C0A25F57AB700413E01 /* ProjectionPickerViewController.swift */; }; A3A50C0D25F57AB700413E01 /* ProjectionPickerViewController.xib in Resources */ = {isa = PBXBuildFile; fileRef = A3A50C0B25F57AB700413E01 /* ProjectionPickerViewController.xib */; }; A3A50C4125F5B3B100413E01 /* SwiftSocket in Frameworks */ = {isa = PBXBuildFile; productRef = A3A50C4025F5B3B100413E01 /* SwiftSocket */; }; - A3EEFFBF26890AE300A0DC19 /* ChromaKeyMaskBuilder.swift in Sources */ = {isa = PBXBuildFile; fileRef = A3EEFFBE26890AE200A0DC19 /* ChromaKeyMaskBuilder.swift */; }; EF2C4C0D28739194001AA8DE /* CaptureFrame.swift in Sources */ = {isa = PBXBuildFile; fileRef = EF2C4C0928739194001AA8DE /* CaptureFrame.swift */; }; EF2C4C0E28739194001AA8DE /* OculusCapture.swift in Sources */ = {isa = PBXBuildFile; fileRef = EF2C4C0A28739194001AA8DE /* OculusCapture.swift */; }; EF2C4C0F28739194001AA8DE /* CapturePayload.swift in Sources */ = {isa = PBXBuildFile; fileRef = EF2C4C0B28739194001AA8DE /* CapturePayload.swift */; }; EF2C4C1028739194001AA8DE /* VideoDecoder.swift in Sources */ = {isa = PBXBuildFile; fileRef = EF2C4C0C28739194001AA8DE /* VideoDecoder.swift */; }; + EFC0DFDF2874C70E0066F6F3 /* RealityMixerKit.h in Headers */ = {isa = PBXBuildFile; fileRef = EFC0DFDE2874C70E0066F6F3 /* RealityMixerKit.h */; settings = {ATTRIBUTES = (Public, ); }; }; + EFC0DFE22874C70E0066F6F3 /* RealityMixerKit.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = EFC0DFDC2874C70E0066F6F3 /* RealityMixerKit.framework */; }; + EFC0DFE32874C70E0066F6F3 /* RealityMixerKit.framework in Embed Frameworks */ = {isa = PBXBuildFile; fileRef = EFC0DFDC2874C70E0066F6F3 /* RealityMixerKit.framework */; settings = {ATTRIBUTES = (CodeSignOnCopy, RemoveHeadersOnCopy, ); }; }; + EFC0DFFA2874CA900066F6F3 /* RealityMixerMacApp.swift in Sources */ = {isa = PBXBuildFile; fileRef = EFC0DFF92874CA900066F6F3 /* RealityMixerMacApp.swift */; }; + EFC0DFFE2874CA900066F6F3 /* Assets.xcassets in Resources */ = {isa = PBXBuildFile; fileRef = EFC0DFFD2874CA900066F6F3 /* Assets.xcassets */; }; + EFC0E0062874CACF0066F6F3 /* RealityMixerKit.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = EFC0DFDC2874C70E0066F6F3 /* RealityMixerKit.framework */; }; + EFC0E00A2874CD890066F6F3 /* RealityMixerMac.xcconfig in Resources */ = {isa = PBXBuildFile; fileRef = EFC0E0092874CD890066F6F3 /* RealityMixerMac.xcconfig */; }; + EFC0E00C2874CFD60066F6F3 /* RealityMixerKit.framework in Embed Frameworks */ = {isa = PBXBuildFile; fileRef = EFC0DFDC2874C70E0066F6F3 /* RealityMixerKit.framework */; settings = {ATTRIBUTES = (CodeSignOnCopy, RemoveHeadersOnCopy, ); }; }; + EFC0E0122874DDF60066F6F3 /* CameraSelectionScreen.swift in Sources */ = {isa = PBXBuildFile; fileRef = EFC0E0112874DDF60066F6F3 /* CameraSelectionScreen.swift */; }; + EFC0E0152874DE230066F6F3 /* CameraSelectionViewModel.swift in Sources */ = {isa = PBXBuildFile; fileRef = EFC0E0142874DE230066F6F3 /* CameraSelectionViewModel.swift */; }; + EFD65FD92874F1A200344323 /* MainWindow.swift in Sources */ = {isa = PBXBuildFile; fileRef = EFD65FD82874F1A200344323 /* MainWindow.swift */; }; + EFD65FDC2874FAAA00344323 /* CameraSession.swift in Sources */ = {isa = PBXBuildFile; fileRef = EFD65FDB2874FAAA00344323 /* CameraSession.swift */; }; + EFD65FDE2874FDA100344323 /* CameraTestScreen.swift in Sources */ = {isa = PBXBuildFile; fileRef = EFD65FDD2874FDA100344323 /* CameraTestScreen.swift */; }; + EFD65FE02874FE3400344323 /* CameraTestViewModel.swift in Sources */ = {isa = PBXBuildFile; fileRef = EFD65FDF2874FE3400344323 /* CameraTestViewModel.swift */; }; + EFD65FE32875E56000344323 /* ChromaKeyConfiguration.swift in Sources */ = {isa = PBXBuildFile; fileRef = A338B35C265AE1E300099827 /* ChromaKeyConfiguration.swift */; }; + EFD65FE42875E56300344323 /* UserDefaultsStorage.swift in Sources */ = {isa = PBXBuildFile; fileRef = A338B359265ADFFE00099827 /* UserDefaultsStorage.swift */; }; + EFD65FE72875EA1C00344323 /* SceneKitHelpers.swift in Sources */ = {isa = PBXBuildFile; fileRef = EFD65FE62875EA1C00344323 /* SceneKitHelpers.swift */; }; + EFD65FEA2875EE8E00344323 /* ChromaKeyMaskBuilder.swift in Sources */ = {isa = PBXBuildFile; fileRef = A3EEFFBE26890AE200A0DC19 /* ChromaKeyMaskBuilder.swift */; }; + EFD65FEB2875EEC800344323 /* Shaders.swift in Sources */ = {isa = PBXBuildFile; fileRef = A39033EC256859A8004CB638 /* Shaders.swift */; }; + EFD65FEF2876333300344323 /* ChromaKeyConfigurationScreen.swift in Sources */ = {isa = PBXBuildFile; fileRef = EFD65FEE2876333300344323 /* ChromaKeyConfigurationScreen.swift */; }; + EFD65FF12876344500344323 /* ChromaKeyConfigurationViewModel.swift in Sources */ = {isa = PBXBuildFile; fileRef = EFD65FF02876344500344323 /* ChromaKeyConfigurationViewModel.swift */; }; + EFD65FF5287650EF00344323 /* ChromaKeyConfigurationViewController.swift in Sources */ = {isa = PBXBuildFile; fileRef = EFD65FF3287650EF00344323 /* ChromaKeyConfigurationViewController.swift */; }; + EFD65FF6287650EF00344323 /* ChromaKeyConfigurationViewController.xib in Resources */ = {isa = PBXBuildFile; fileRef = EFD65FF4287650EF00344323 /* ChromaKeyConfigurationViewController.xib */; }; /* End PBXBuildFile section */ +/* Begin PBXContainerItemProxy section */ + EFC0DFE02874C70E0066F6F3 /* PBXContainerItemProxy */ = { + isa = PBXContainerItemProxy; + containerPortal = A328D8A4253C8C4000B90E62 /* Project object */; + proxyType = 1; + remoteGlobalIDString = EFC0DFDB2874C70E0066F6F3; + remoteInfo = RealityMixerKit; + }; +/* End PBXContainerItemProxy section */ + /* Begin PBXCopyFilesBuildPhase section */ A36BAB3025F059B00006FDD8 /* Embed Frameworks */ = { isa = PBXCopyFilesBuildPhase; @@ -87,6 +113,18 @@ dstPath = ""; dstSubfolderSpec = 10; files = ( + EFC0DFE32874C70E0066F6F3 /* RealityMixerKit.framework in Embed Frameworks */, + ); + name = "Embed Frameworks"; + runOnlyForDeploymentPostprocessing = 0; + }; + EFC0E00B2874CFC70066F6F3 /* Embed Frameworks */ = { + isa = PBXCopyFilesBuildPhase; + buildActionMask = 2147483647; + dstPath = ""; + dstSubfolderSpec = 10; + files = ( + EFC0E00C2874CFD60066F6F3 /* RealityMixerKit.framework in Embed Frameworks */, ); name = "Embed Frameworks"; runOnlyForDeploymentPostprocessing = 0; @@ -169,6 +207,24 @@ EF2C4C0A28739194001AA8DE /* OculusCapture.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = OculusCapture.swift; sourceTree = ""; }; EF2C4C0B28739194001AA8DE /* CapturePayload.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = CapturePayload.swift; sourceTree = ""; }; EF2C4C0C28739194001AA8DE /* VideoDecoder.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = VideoDecoder.swift; sourceTree = ""; }; + EFC0DFDC2874C70E0066F6F3 /* RealityMixerKit.framework */ = {isa = PBXFileReference; explicitFileType = wrapper.framework; includeInIndex = 0; path = RealityMixerKit.framework; sourceTree = BUILT_PRODUCTS_DIR; }; + EFC0DFDE2874C70E0066F6F3 /* RealityMixerKit.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = RealityMixerKit.h; sourceTree = ""; }; + EFC0DFF72874CA900066F6F3 /* RealityMixerMac.app */ = {isa = PBXFileReference; explicitFileType = wrapper.application; includeInIndex = 0; path = RealityMixerMac.app; sourceTree = BUILT_PRODUCTS_DIR; }; + EFC0DFF92874CA900066F6F3 /* RealityMixerMacApp.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = RealityMixerMacApp.swift; sourceTree = ""; }; + EFC0DFFD2874CA900066F6F3 /* Assets.xcassets */ = {isa = PBXFileReference; lastKnownFileType = folder.assetcatalog; path = Assets.xcassets; sourceTree = ""; }; + EFC0E0022874CA900066F6F3 /* RealityMixerMac.entitlements */ = {isa = PBXFileReference; lastKnownFileType = text.plist.entitlements; path = RealityMixerMac.entitlements; sourceTree = ""; }; + EFC0E0092874CD890066F6F3 /* RealityMixerMac.xcconfig */ = {isa = PBXFileReference; lastKnownFileType = text.xcconfig; path = RealityMixerMac.xcconfig; sourceTree = ""; }; + EFC0E0112874DDF60066F6F3 /* CameraSelectionScreen.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = CameraSelectionScreen.swift; sourceTree = ""; }; + EFC0E0142874DE230066F6F3 /* CameraSelectionViewModel.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = CameraSelectionViewModel.swift; sourceTree = ""; }; + EFD65FD82874F1A200344323 /* MainWindow.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = MainWindow.swift; sourceTree = ""; }; + EFD65FDB2874FAAA00344323 /* CameraSession.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = CameraSession.swift; sourceTree = ""; }; + EFD65FDD2874FDA100344323 /* CameraTestScreen.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = CameraTestScreen.swift; sourceTree = ""; }; + EFD65FDF2874FE3400344323 /* CameraTestViewModel.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = CameraTestViewModel.swift; sourceTree = ""; }; + EFD65FE62875EA1C00344323 /* SceneKitHelpers.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = SceneKitHelpers.swift; sourceTree = ""; }; + EFD65FEE2876333300344323 /* ChromaKeyConfigurationScreen.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ChromaKeyConfigurationScreen.swift; sourceTree = ""; }; + EFD65FF02876344500344323 /* ChromaKeyConfigurationViewModel.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ChromaKeyConfigurationViewModel.swift; sourceTree = ""; }; + EFD65FF3287650EF00344323 /* ChromaKeyConfigurationViewController.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ChromaKeyConfigurationViewController.swift; sourceTree = ""; }; + EFD65FF4287650EF00344323 /* ChromaKeyConfigurationViewController.xib */ = {isa = PBXFileReference; lastKnownFileType = file.xib; path = ChromaKeyConfigurationViewController.xib; sourceTree = ""; }; /* End PBXFileReference section */ /* Begin PBXFrameworksBuildPhase section */ @@ -176,16 +232,29 @@ isa = PBXFrameworksBuildPhase; buildActionMask = 2147483647; files = ( - A328D960253C959500B90E62 /* libbz2.tbd in Frameworks */, A3A50C0725F4945700413E01 /* SwiftlyAttributedStrings in Frameworks */, A3A50C4125F5B3B100413E01 /* SwiftSocket in Frameworks */, + EFC0DFE22874C70E0066F6F3 /* RealityMixerKit.framework in Frameworks */, A328D98E253CF2EE00B90E62 /* ARKit.framework in Frameworks */, A328D990253CF2F400B90E62 /* SceneKit.framework in Frameworks */, A328D96B253C95BD00B90E62 /* AudioToolbox.framework in Frameworks */, A328D96D253C95C300B90E62 /* CoreMedia.framework in Frameworks */, - A328D966253C95AD00B90E62 /* libiconv.tbd in Frameworks */, A328D969253C95B700B90E62 /* VideoToolbox.framework in Frameworks */, - A328D963253C95A000B90E62 /* libz.tbd in Frameworks */, + ); + runOnlyForDeploymentPostprocessing = 0; + }; + EFC0DFD92874C70E0066F6F3 /* Frameworks */ = { + isa = PBXFrameworksBuildPhase; + buildActionMask = 2147483647; + files = ( + ); + runOnlyForDeploymentPostprocessing = 0; + }; + EFC0DFF42874CA900066F6F3 /* Frameworks */ = { + isa = PBXFrameworksBuildPhase; + buildActionMask = 2147483647; + files = ( + EFC0E0062874CACF0066F6F3 /* RealityMixerKit.framework in Frameworks */, ); runOnlyForDeploymentPostprocessing = 0; }; @@ -206,7 +275,6 @@ A3798AE025D9F6CA00B3BFBC /* CameraPoseSender.swift */, A319DCB0264891CF00CDAE58 /* ARConfigurationFactory.swift */, A338B362265B25E500099827 /* ARKitHelpers.swift */, - A3EEFFBE26890AE200A0DC19 /* ChromaKeyMaskBuilder.swift */, A30B22E92693D9F2005EB78F /* CameraPermissionHelper.swift */, ); path = Misc; @@ -330,6 +398,8 @@ A32285B62558163B00FF3200 /* Instructions.md */, A37850BC256C433C00C046FA /* Compatibility.md */, A328D8AE253C8C4000B90E62 /* RealityMixer */, + EFC0DFDD2874C70E0066F6F3 /* RealityMixerKit */, + EFC0DFF82874CA900066F6F3 /* RealityMixerMac */, A328D8AD253C8C4000B90E62 /* Products */, A328D95E253C958D00B90E62 /* Frameworks */, ); @@ -339,6 +409,8 @@ isa = PBXGroup; children = ( A328D8AC253C8C4000B90E62 /* RealityMixer.app */, + EFC0DFDC2874C70E0066F6F3 /* RealityMixerKit.framework */, + EFC0DFF72874CA900066F6F3 /* RealityMixerMac.app */, ); name = Products; sourceTree = ""; @@ -399,8 +471,6 @@ children = ( A32226FA262C5229004629CC /* MixedRealityConfiguration.swift */, A32226FB262C5229004629CC /* NetworkConfiguration.swift */, - A338B359265ADFFE00099827 /* UserDefaultsStorage.swift */, - A338B35C265AE1E300099827 /* ChromaKeyConfiguration.swift */, ); path = Configuration; sourceTree = ""; @@ -449,7 +519,6 @@ A39033EB2568599B004CB638 /* Shaders */ = { isa = PBXGroup; children = ( - A39033EC256859A8004CB638 /* Shaders.swift */, ); path = Shaders; sourceTree = ""; @@ -465,8 +534,157 @@ path = MRC; sourceTree = ""; }; + EFC0DFDD2874C70E0066F6F3 /* RealityMixerKit */ = { + isa = PBXGroup; + children = ( + EFD65FE92875ED9600344323 /* Shaders */, + EFD65FE82875ED8200344323 /* ChromaKey */, + EFD65FE52875E9AB00344323 /* Helpers */, + EFD65FE22875E54C00344323 /* Configuration */, + EFC0DFDE2874C70E0066F6F3 /* RealityMixerKit.h */, + ); + path = RealityMixerKit; + sourceTree = ""; + }; + EFC0DFF82874CA900066F6F3 /* RealityMixerMac */ = { + isa = PBXGroup; + children = ( + EFD65FE12875E3E000344323 /* ChromaKey */, + EFD65FDA2874FAA100344323 /* Camera */, + EFC0E00F2874DDBE0066F6F3 /* Main */, + EFC0E0082874CB610066F6F3 /* Resources */, + EFC0DFF92874CA900066F6F3 /* RealityMixerMacApp.swift */, + EFC0E0022874CA900066F6F3 /* RealityMixerMac.entitlements */, + EFC0E0092874CD890066F6F3 /* RealityMixerMac.xcconfig */, + ); + path = RealityMixerMac; + sourceTree = ""; + }; + EFC0E0082874CB610066F6F3 /* Resources */ = { + isa = PBXGroup; + children = ( + EFC0DFFD2874CA900066F6F3 /* Assets.xcassets */, + ); + path = Resources; + sourceTree = ""; + }; + EFC0E00F2874DDBE0066F6F3 /* Main */ = { + isa = PBXGroup; + children = ( + EFC0E0132874DE150066F6F3 /* ViewModels */, + EFC0E0102874DDDB0066F6F3 /* Screens */, + ); + path = Main; + sourceTree = ""; + }; + EFC0E0102874DDDB0066F6F3 /* Screens */ = { + isa = PBXGroup; + children = ( + EFC0E0112874DDF60066F6F3 /* CameraSelectionScreen.swift */, + EFD65FD82874F1A200344323 /* MainWindow.swift */, + EFD65FDD2874FDA100344323 /* CameraTestScreen.swift */, + ); + path = Screens; + sourceTree = ""; + }; + EFC0E0132874DE150066F6F3 /* ViewModels */ = { + isa = PBXGroup; + children = ( + EFC0E0142874DE230066F6F3 /* CameraSelectionViewModel.swift */, + EFD65FDF2874FE3400344323 /* CameraTestViewModel.swift */, + ); + path = ViewModels; + sourceTree = ""; + }; + EFD65FDA2874FAA100344323 /* Camera */ = { + isa = PBXGroup; + children = ( + EFD65FDB2874FAAA00344323 /* CameraSession.swift */, + ); + path = Camera; + sourceTree = ""; + }; + EFD65FE12875E3E000344323 /* ChromaKey */ = { + isa = PBXGroup; + children = ( + EFD65FF2287650D100344323 /* ViewControllers */, + EFD65FEC2876331700344323 /* ViewModels */, + EFD65FED2876331D00344323 /* Screens */, + ); + path = ChromaKey; + sourceTree = ""; + }; + EFD65FE22875E54C00344323 /* Configuration */ = { + isa = PBXGroup; + children = ( + A338B359265ADFFE00099827 /* UserDefaultsStorage.swift */, + A338B35C265AE1E300099827 /* ChromaKeyConfiguration.swift */, + ); + path = Configuration; + sourceTree = ""; + }; + EFD65FE52875E9AB00344323 /* Helpers */ = { + isa = PBXGroup; + children = ( + EFD65FE62875EA1C00344323 /* SceneKitHelpers.swift */, + ); + path = Helpers; + sourceTree = ""; + }; + EFD65FE82875ED8200344323 /* ChromaKey */ = { + isa = PBXGroup; + children = ( + A3EEFFBE26890AE200A0DC19 /* ChromaKeyMaskBuilder.swift */, + ); + path = ChromaKey; + sourceTree = ""; + }; + EFD65FE92875ED9600344323 /* Shaders */ = { + isa = PBXGroup; + children = ( + A39033EC256859A8004CB638 /* Shaders.swift */, + ); + path = Shaders; + sourceTree = ""; + }; + EFD65FEC2876331700344323 /* ViewModels */ = { + isa = PBXGroup; + children = ( + EFD65FF02876344500344323 /* ChromaKeyConfigurationViewModel.swift */, + ); + path = ViewModels; + sourceTree = ""; + }; + EFD65FED2876331D00344323 /* Screens */ = { + isa = PBXGroup; + children = ( + EFD65FEE2876333300344323 /* ChromaKeyConfigurationScreen.swift */, + ); + path = Screens; + sourceTree = ""; + }; + EFD65FF2287650D100344323 /* ViewControllers */ = { + isa = PBXGroup; + children = ( + EFD65FF3287650EF00344323 /* ChromaKeyConfigurationViewController.swift */, + EFD65FF4287650EF00344323 /* ChromaKeyConfigurationViewController.xib */, + ); + path = ViewControllers; + sourceTree = ""; + }; /* End PBXGroup section */ +/* Begin PBXHeadersBuildPhase section */ + EFC0DFD72874C70E0066F6F3 /* Headers */ = { + isa = PBXHeadersBuildPhase; + buildActionMask = 2147483647; + files = ( + EFC0DFDF2874C70E0066F6F3 /* RealityMixerKit.h in Headers */, + ); + runOnlyForDeploymentPostprocessing = 0; + }; +/* End PBXHeadersBuildPhase section */ + /* Begin PBXNativeTarget section */ A328D8AB253C8C4000B90E62 /* RealityMixer */ = { isa = PBXNativeTarget; @@ -480,6 +698,7 @@ buildRules = ( ); dependencies = ( + EFC0DFE12874C70E0066F6F3 /* PBXTargetDependency */, ); name = RealityMixer; packageProductDependencies = ( @@ -490,19 +709,62 @@ productReference = A328D8AC253C8C4000B90E62 /* RealityMixer.app */; productType = "com.apple.product-type.application"; }; + EFC0DFDB2874C70E0066F6F3 /* RealityMixerKit */ = { + isa = PBXNativeTarget; + buildConfigurationList = EFC0DFE62874C70E0066F6F3 /* Build configuration list for PBXNativeTarget "RealityMixerKit" */; + buildPhases = ( + EFC0DFD72874C70E0066F6F3 /* Headers */, + EFC0DFD82874C70E0066F6F3 /* Sources */, + EFC0DFD92874C70E0066F6F3 /* Frameworks */, + EFC0DFDA2874C70E0066F6F3 /* Resources */, + ); + buildRules = ( + ); + dependencies = ( + ); + name = RealityMixerKit; + productName = RealityMixerKit; + productReference = EFC0DFDC2874C70E0066F6F3 /* RealityMixerKit.framework */; + productType = "com.apple.product-type.framework"; + }; + EFC0DFF62874CA900066F6F3 /* RealityMixerMac */ = { + isa = PBXNativeTarget; + buildConfigurationList = EFC0E0032874CA900066F6F3 /* Build configuration list for PBXNativeTarget "RealityMixerMac" */; + buildPhases = ( + EFC0DFF32874CA900066F6F3 /* Sources */, + EFC0DFF42874CA900066F6F3 /* Frameworks */, + EFC0DFF52874CA900066F6F3 /* Resources */, + EFC0E00B2874CFC70066F6F3 /* Embed Frameworks */, + ); + buildRules = ( + ); + dependencies = ( + ); + name = RealityMixerMac; + productName = RealityMixerMac; + productReference = EFC0DFF72874CA900066F6F3 /* RealityMixerMac.app */; + productType = "com.apple.product-type.application"; + }; /* End PBXNativeTarget section */ /* Begin PBXProject section */ A328D8A4253C8C4000B90E62 /* Project object */ = { isa = PBXProject; attributes = { - LastSwiftUpdateCheck = 1200; + LastSwiftUpdateCheck = 1400; LastUpgradeCheck = 1200; TargetAttributes = { A328D8AB253C8C4000B90E62 = { CreatedOnToolsVersion = 12.0; LastSwiftMigration = 1200; }; + EFC0DFDB2874C70E0066F6F3 = { + CreatedOnToolsVersion = 14.0; + LastSwiftMigration = 1400; + }; + EFC0DFF62874CA900066F6F3 = { + CreatedOnToolsVersion = 14.0; + }; }; }; buildConfigurationList = A328D8A7253C8C4000B90E62 /* Build configuration list for PBXProject "RealityMixer" */; @@ -523,6 +785,8 @@ projectRoot = ""; targets = ( A328D8AB253C8C4000B90E62 /* RealityMixer */, + EFC0DFDB2874C70E0066F6F3 /* RealityMixerKit */, + EFC0DFF62874CA900066F6F3 /* RealityMixerMac */, ); }; /* End PBXProject section */ @@ -555,6 +819,23 @@ ); runOnlyForDeploymentPostprocessing = 0; }; + EFC0DFDA2874C70E0066F6F3 /* Resources */ = { + isa = PBXResourcesBuildPhase; + buildActionMask = 2147483647; + files = ( + ); + runOnlyForDeploymentPostprocessing = 0; + }; + EFC0DFF52874CA900066F6F3 /* Resources */ = { + isa = PBXResourcesBuildPhase; + buildActionMask = 2147483647; + files = ( + EFC0E00A2874CD890066F6F3 /* RealityMixerMac.xcconfig in Resources */, + EFD65FF6287650EF00344323 /* ChromaKeyConfigurationViewController.xib in Resources */, + EFC0DFFE2874CA900066F6F3 /* Assets.xcassets in Resources */, + ); + runOnlyForDeploymentPostprocessing = 0; + }; /* End PBXResourcesBuildPhase section */ /* Begin PBXSourcesBuildPhase section */ @@ -574,18 +855,14 @@ EF2C4C0D28739194001AA8DE /* CaptureFrame.swift in Sources */, A36C916526377A3000D64107 /* AvatarProtocol.swift in Sources */, A319DCB1264891CF00CDAE58 /* ARConfigurationFactory.swift in Sources */, - A338B35A265ADFFE00099827 /* UserDefaultsStorage.swift in Sources */, A324160F2554D4AD008D4B90 /* CalibrationBuilder.swift in Sources */, - A338B35D265AE1E300099827 /* ChromaKeyConfiguration.swift in Sources */, A32416402555DBA9008D4B90 /* Double+ScientificString.swift in Sources */, EF2C4C0E28739194001AA8DE /* OculusCapture.swift in Sources */, A338B360265B250100099827 /* ChromaKeyConfigurationViewController.swift in Sources */, A324162E2554E213008D4B90 /* OculusCalibration.swift in Sources */, A32A3D0226CB0A9A0093FAC5 /* AudioManager.swift in Sources */, A36C916726377C0900D64107 /* RobotAvatar.swift in Sources */, - A3EEFFBF26890AE300A0DC19 /* ChromaKeyMaskBuilder.swift in Sources */, A324163D2554E520008D4B90 /* CGPoint+Extensions.swift in Sources */, - A39033ED256859A8004CB638 /* Shaders.swift in Sources */, A3A50C0C25F57AB700413E01 /* ProjectionPickerViewController.swift in Sources */, A32416022554D296008D4B90 /* Math.swift in Sources */, A32416052554D2FE008D4B90 /* Parser.swift in Sources */, @@ -606,8 +883,45 @@ ); runOnlyForDeploymentPostprocessing = 0; }; + EFC0DFD82874C70E0066F6F3 /* Sources */ = { + isa = PBXSourcesBuildPhase; + buildActionMask = 2147483647; + files = ( + EFD65FE32875E56000344323 /* ChromaKeyConfiguration.swift in Sources */, + EFD65FEB2875EEC800344323 /* Shaders.swift in Sources */, + EFD65FE72875EA1C00344323 /* SceneKitHelpers.swift in Sources */, + EFD65FE42875E56300344323 /* UserDefaultsStorage.swift in Sources */, + EFD65FEA2875EE8E00344323 /* ChromaKeyMaskBuilder.swift in Sources */, + ); + runOnlyForDeploymentPostprocessing = 0; + }; + EFC0DFF32874CA900066F6F3 /* Sources */ = { + isa = PBXSourcesBuildPhase; + buildActionMask = 2147483647; + files = ( + EFD65FEF2876333300344323 /* ChromaKeyConfigurationScreen.swift in Sources */, + EFD65FE02874FE3400344323 /* CameraTestViewModel.swift in Sources */, + EFD65FF12876344500344323 /* ChromaKeyConfigurationViewModel.swift in Sources */, + EFD65FDC2874FAAA00344323 /* CameraSession.swift in Sources */, + EFD65FD92874F1A200344323 /* MainWindow.swift in Sources */, + EFD65FDE2874FDA100344323 /* CameraTestScreen.swift in Sources */, + EFC0E0152874DE230066F6F3 /* CameraSelectionViewModel.swift in Sources */, + EFC0DFFA2874CA900066F6F3 /* RealityMixerMacApp.swift in Sources */, + EFC0E0122874DDF60066F6F3 /* CameraSelectionScreen.swift in Sources */, + EFD65FF5287650EF00344323 /* ChromaKeyConfigurationViewController.swift in Sources */, + ); + runOnlyForDeploymentPostprocessing = 0; + }; /* End PBXSourcesBuildPhase section */ +/* Begin PBXTargetDependency section */ + EFC0DFE12874C70E0066F6F3 /* PBXTargetDependency */ = { + isa = PBXTargetDependency; + target = EFC0DFDB2874C70E0066F6F3 /* RealityMixerKit */; + targetProxy = EFC0DFE02874C70E0066F6F3 /* PBXContainerItemProxy */; + }; +/* End PBXTargetDependency section */ + /* Begin PBXVariantGroup section */ A328D8BA253C8C4300B90E62 /* LaunchScreen.storyboard */ = { isa = PBXVariantGroup; @@ -740,6 +1054,7 @@ isa = XCBuildConfiguration; baseConfigurationReference = 3E1B214525678AF60084847C /* RealityMixer.xcconfig */; buildSettings = { + ALWAYS_EMBED_SWIFT_STANDARD_LIBRARIES = YES; ASSETCATALOG_COMPILER_APPICON_NAME = AppIcon; ASSETCATALOG_COMPILER_GLOBAL_ACCENT_COLOR_NAME = AccentColor; CLANG_ENABLE_MODULES = YES; @@ -767,6 +1082,7 @@ isa = XCBuildConfiguration; baseConfigurationReference = 3E1B214525678AF60084847C /* RealityMixer.xcconfig */; buildSettings = { + ALWAYS_EMBED_SWIFT_STANDARD_LIBRARIES = YES; ASSETCATALOG_COMPILER_APPICON_NAME = AppIcon; ASSETCATALOG_COMPILER_GLOBAL_ACCENT_COLOR_NAME = AccentColor; CLANG_ENABLE_MODULES = YES; @@ -789,6 +1105,147 @@ }; name = Release; }; + EFC0DFE42874C70E0066F6F3 /* Debug */ = { + isa = XCBuildConfiguration; + buildSettings = { + CLANG_CXX_LANGUAGE_STANDARD = "gnu++17"; + CLANG_ENABLE_MODULES = YES; + CODE_SIGN_STYLE = Automatic; + CURRENT_PROJECT_VERSION = 1; + DEFINES_MODULE = YES; + DYLIB_COMPATIBILITY_VERSION = 1; + DYLIB_CURRENT_VERSION = 1; + DYLIB_INSTALL_NAME_BASE = "@rpath"; + GENERATE_INFOPLIST_FILE = YES; + INFOPLIST_KEY_NSHumanReadableCopyright = ""; + INSTALL_PATH = "$(LOCAL_LIBRARY_DIR)/Frameworks"; + IPHONEOS_DEPLOYMENT_TARGET = 14.0; + LD_RUNPATH_SEARCH_PATHS = ( + "$(inherited)", + "@executable_path/Frameworks", + "@loader_path/Frameworks", + ); + MACOSX_DEPLOYMENT_TARGET = 12.0; + MARKETING_VERSION = 1.0; + PRODUCT_BUNDLE_IDENTIFIER = com.bluenose.RealityMixerKit; + PRODUCT_NAME = "$(TARGET_NAME:c99extidentifier)"; + SKIP_INSTALL = YES; + SUPPORTED_PLATFORMS = "iphoneos iphonesimulator macosx"; + SUPPORTS_MACCATALYST = YES; + SUPPORTS_MAC_DESIGNED_FOR_IPHONE_IPAD = YES; + SWIFT_EMIT_LOC_STRINGS = YES; + SWIFT_OPTIMIZATION_LEVEL = "-Onone"; + SWIFT_VERSION = 5.0; + TARGETED_DEVICE_FAMILY = "1,2"; + VERSIONING_SYSTEM = "apple-generic"; + VERSION_INFO_PREFIX = ""; + }; + name = Debug; + }; + EFC0DFE52874C70E0066F6F3 /* Release */ = { + isa = XCBuildConfiguration; + buildSettings = { + CLANG_CXX_LANGUAGE_STANDARD = "gnu++17"; + CLANG_ENABLE_MODULES = YES; + CODE_SIGN_STYLE = Automatic; + CURRENT_PROJECT_VERSION = 1; + DEFINES_MODULE = YES; + DYLIB_COMPATIBILITY_VERSION = 1; + DYLIB_CURRENT_VERSION = 1; + DYLIB_INSTALL_NAME_BASE = "@rpath"; + GENERATE_INFOPLIST_FILE = YES; + INFOPLIST_KEY_NSHumanReadableCopyright = ""; + INSTALL_PATH = "$(LOCAL_LIBRARY_DIR)/Frameworks"; + IPHONEOS_DEPLOYMENT_TARGET = 14.0; + LD_RUNPATH_SEARCH_PATHS = ( + "$(inherited)", + "@executable_path/Frameworks", + "@loader_path/Frameworks", + ); + MACOSX_DEPLOYMENT_TARGET = 12.0; + MARKETING_VERSION = 1.0; + PRODUCT_BUNDLE_IDENTIFIER = com.bluenose.RealityMixerKit; + PRODUCT_NAME = "$(TARGET_NAME:c99extidentifier)"; + SKIP_INSTALL = YES; + SUPPORTED_PLATFORMS = "iphoneos iphonesimulator macosx"; + SUPPORTS_MACCATALYST = YES; + SUPPORTS_MAC_DESIGNED_FOR_IPHONE_IPAD = YES; + SWIFT_EMIT_LOC_STRINGS = YES; + SWIFT_VERSION = 5.0; + TARGETED_DEVICE_FAMILY = "1,2"; + VERSIONING_SYSTEM = "apple-generic"; + VERSION_INFO_PREFIX = ""; + }; + name = Release; + }; + EFC0E0042874CA900066F6F3 /* Debug */ = { + isa = XCBuildConfiguration; + baseConfigurationReference = EFC0E0092874CD890066F6F3 /* RealityMixerMac.xcconfig */; + buildSettings = { + ALWAYS_EMBED_SWIFT_STANDARD_LIBRARIES = YES; + ASSETCATALOG_COMPILER_APPICON_NAME = AppIcon; + ASSETCATALOG_COMPILER_GLOBAL_ACCENT_COLOR_NAME = AccentColor; + CLANG_CXX_LANGUAGE_STANDARD = "gnu++17"; + CODE_SIGN_ENTITLEMENTS = RealityMixerMac/RealityMixerMac.entitlements; + "CODE_SIGN_IDENTITY[sdk=macosx*]" = "Apple Development"; + CODE_SIGN_STYLE = Automatic; + COMBINE_HIDPI_IMAGES = YES; + CURRENT_PROJECT_VERSION = 1; + DEVELOPMENT_ASSET_PATHS = ""; + DEVELOPMENT_TEAM = ZTJ55TM37B; + ENABLE_HARDENED_RUNTIME = YES; + ENABLE_PREVIEWS = YES; + GENERATE_INFOPLIST_FILE = YES; + INFOPLIST_KEY_NSCameraUsageDescription = "This app requires camera access for Mixed Reality"; + INFOPLIST_KEY_NSHumanReadableCopyright = ""; + LD_RUNPATH_SEARCH_PATHS = ( + "$(inherited)", + "@executable_path/../Frameworks", + ); + MACOSX_DEPLOYMENT_TARGET = 12.4; + MARKETING_VERSION = 1.0; + PRODUCT_BUNDLE_IDENTIFIER = "$(ORGANIZATION_IDENTIFIER).RealityMixerMac"; + PRODUCT_NAME = "$(TARGET_NAME)"; + SDKROOT = macosx; + SWIFT_EMIT_LOC_STRINGS = YES; + SWIFT_VERSION = 5.0; + }; + name = Debug; + }; + EFC0E0052874CA900066F6F3 /* Release */ = { + isa = XCBuildConfiguration; + baseConfigurationReference = EFC0E0092874CD890066F6F3 /* RealityMixerMac.xcconfig */; + buildSettings = { + ALWAYS_EMBED_SWIFT_STANDARD_LIBRARIES = YES; + ASSETCATALOG_COMPILER_APPICON_NAME = AppIcon; + ASSETCATALOG_COMPILER_GLOBAL_ACCENT_COLOR_NAME = AccentColor; + CLANG_CXX_LANGUAGE_STANDARD = "gnu++17"; + CODE_SIGN_ENTITLEMENTS = RealityMixerMac/RealityMixerMac.entitlements; + "CODE_SIGN_IDENTITY[sdk=macosx*]" = "Apple Development"; + CODE_SIGN_STYLE = Automatic; + COMBINE_HIDPI_IMAGES = YES; + CURRENT_PROJECT_VERSION = 1; + DEVELOPMENT_ASSET_PATHS = ""; + DEVELOPMENT_TEAM = ZTJ55TM37B; + ENABLE_HARDENED_RUNTIME = YES; + ENABLE_PREVIEWS = YES; + GENERATE_INFOPLIST_FILE = YES; + INFOPLIST_KEY_NSCameraUsageDescription = "This app requires camera access for Mixed Reality"; + INFOPLIST_KEY_NSHumanReadableCopyright = ""; + LD_RUNPATH_SEARCH_PATHS = ( + "$(inherited)", + "@executable_path/../Frameworks", + ); + MACOSX_DEPLOYMENT_TARGET = 12.4; + MARKETING_VERSION = 1.0; + PRODUCT_BUNDLE_IDENTIFIER = "$(ORGANIZATION_IDENTIFIER).RealityMixerMac"; + PRODUCT_NAME = "$(TARGET_NAME)"; + SDKROOT = macosx; + SWIFT_EMIT_LOC_STRINGS = YES; + SWIFT_VERSION = 5.0; + }; + name = Release; + }; /* End XCBuildConfiguration section */ /* Begin XCConfigurationList section */ @@ -810,6 +1267,24 @@ defaultConfigurationIsVisible = 0; defaultConfigurationName = Release; }; + EFC0DFE62874C70E0066F6F3 /* Build configuration list for PBXNativeTarget "RealityMixerKit" */ = { + isa = XCConfigurationList; + buildConfigurations = ( + EFC0DFE42874C70E0066F6F3 /* Debug */, + EFC0DFE52874C70E0066F6F3 /* Release */, + ); + defaultConfigurationIsVisible = 0; + defaultConfigurationName = Release; + }; + EFC0E0032874CA900066F6F3 /* Build configuration list for PBXNativeTarget "RealityMixerMac" */ = { + isa = XCConfigurationList; + buildConfigurations = ( + EFC0E0042874CA900066F6F3 /* Debug */, + EFC0E0052874CA900066F6F3 /* Release */, + ); + defaultConfigurationIsVisible = 0; + defaultConfigurationName = Release; + }; /* End XCConfigurationList section */ /* Begin XCRemoteSwiftPackageReference section */ diff --git a/RealityMixer/Capture/Configuration/ChromaKeyConfiguration.swift b/RealityMixer/Capture/Configuration/ChromaKeyConfiguration.swift deleted file mode 100644 index 2636f5f..0000000 --- a/RealityMixer/Capture/Configuration/ChromaKeyConfiguration.swift +++ /dev/null @@ -1,99 +0,0 @@ -// -// ChromaKeyConfiguration.swift -// RealityMixer -// -// Created by Fabio de Albuquerque Dela Antonio on 23/05/2021. -// - -import UIKit - -struct ChromaKeyConfiguration: Codable { - - struct Color: Codable { - let red: Float // 0 .. 1 - let green: Float // 0 .. 1 - let blue: Float // 0 .. 1 - - init(uiColor: UIColor) { - var red: CGFloat = 0 - var green: CGFloat = 0 - var blue: CGFloat = 0 - uiColor.getRed(&red, green: &green, blue: &blue, alpha: nil) - self.red = Float(red) - self.green = Float(green) - self.blue = Float(blue) - } - - var uiColor: UIColor { - UIColor(red: CGFloat(red), green: CGFloat(green), blue: CGFloat(blue), alpha: 1) - } - } - - let color: Color - let sensitivity: Float // 0 .. 1 - let smoothness: Float // 0 .. 1 -} - -// MARK: - Codable - -extension ChromaKeyConfiguration.Color { - - enum CodingKeys: String, CodingKey { - case red - case green - case blue - } - - init(from decoder: Decoder) throws { - let values = try decoder.container(keyedBy: CodingKeys.self) - self.red = max(0.0, min(1.0, try values.decode(Float.self, forKey: .red))) - self.green = max(0.0, min(1.0, try values.decode(Float.self, forKey: .green))) - self.blue = max(0.0, min(1.0, try values.decode(Float.self, forKey: .blue))) - } - - func encode(to encoder: Encoder) throws { - var container = encoder.container(keyedBy: CodingKeys.self) - try container.encode(red, forKey: .red) - try container.encode(green, forKey: .green) - try container.encode(blue, forKey: .blue) - } -} - -// MARK: - Storage - -final class ChromaKeyConfigurationStorage { - private let storage = UserDefaultsStorage(key: "ChromaConfiguration") - - func save(configuration: ChromaKeyConfiguration) throws { - try storage.save(configuration) - } - - var configuration: ChromaKeyConfiguration? { - storage.object - } -} - -final class ChromaKeyMaskStorage { - private let manager = FileManager.default - - var maskURL: URL? { - manager - .urls(for: .documentDirectory, in: .userDomainMask) - .first? - .appendingPathComponent("mask.png") - } - - func update(mask: UIImage?) throws { - guard let maskURL = maskURL else { return } - - if let mask = mask { - try mask.pngData()?.write(to: maskURL) - } else { - try manager.removeItem(at: maskURL) - } - } - - func load() -> UIImage? { - maskURL.flatMap({ try? Data(contentsOf: $0) }).flatMap(UIImage.init(data:)) - } -} diff --git a/RealityMixer/Capture/Configuration/MixedRealityConfiguration.swift b/RealityMixer/Capture/Configuration/MixedRealityConfiguration.swift index a6ffe3b..2251ca2 100644 --- a/RealityMixer/Capture/Configuration/MixedRealityConfiguration.swift +++ b/RealityMixer/Capture/Configuration/MixedRealityConfiguration.swift @@ -6,6 +6,7 @@ // import Foundation +import RealityMixerKit struct MixedRealityConfiguration: Codable, Equatable { diff --git a/RealityMixer/Capture/Configuration/NetworkConfiguration.swift b/RealityMixer/Capture/Configuration/NetworkConfiguration.swift index 95aa540..856c3c6 100644 --- a/RealityMixer/Capture/Configuration/NetworkConfiguration.swift +++ b/RealityMixer/Capture/Configuration/NetworkConfiguration.swift @@ -6,6 +6,7 @@ // import Foundation +import RealityMixerKit struct NetworkConfiguration: Codable { let address: String diff --git a/RealityMixer/Capture/Misc/ARKitHelpers.swift b/RealityMixer/Capture/Misc/ARKitHelpers.swift index 2ce32ff..1d18edd 100644 --- a/RealityMixer/Capture/Misc/ARKitHelpers.swift +++ b/RealityMixer/Capture/Misc/ARKitHelpers.swift @@ -6,6 +6,7 @@ // import ARKit +import RealityMixerKit struct ARKitHelpers { @@ -19,79 +20,7 @@ struct ARKitHelpers { return CGSize(width: CGFloat(width), height: CGFloat(height)) } - static func makePlane(size: CGSize, distance: Float) -> SCNNode { - let plane = SCNPlane(width: size.width, height: size.height) - plane.cornerRadius = 0 - plane.firstMaterial?.lightingModel = .constant - plane.firstMaterial?.diffuse.contents = UIColor(red: 0, green: 0, blue: 0, alpha: 1) - - let planeNode = SCNNode(geometry: plane) - planeNode.position = .init(0, 0, -distance) - return planeNode - } - static func makePlaneNodeForDistance(_ distance: Float, frame: ARFrame) -> SCNNode { - makePlane(size: planeSizeForDistance(distance, frame: frame), distance: distance) - } - - @discardableResult - static func create(textureCache: inout CVMetalTextureCache?, for sceneView: SCNView) -> Bool { - guard let metalDevice = sceneView.device else { - return false - } - - return create(textureCache: &textureCache, forDevice: metalDevice) - } - - @discardableResult - static func create(textureCache: inout CVMetalTextureCache?, forDevice metalDevice: MTLDevice) -> Bool { - let result = CVMetalTextureCacheCreate( - kCFAllocatorDefault, - nil, - metalDevice, - nil, - &textureCache - ) - - return result == kCVReturnSuccess - } - - static func texture( - from pixelBuffer: CVPixelBuffer, - format: MTLPixelFormat, - planeIndex: Int, - textureCache: CVMetalTextureCache? - ) -> MTLTexture? { - guard let textureCache = textureCache, - planeIndex >= 0, planeIndex < CVPixelBufferGetPlaneCount(pixelBuffer) //, -// CVPixelBufferGetPixelFormatType(pixelBuffer) == kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange - else { - return nil - } - - var texture: MTLTexture? - - let width = CVPixelBufferGetWidthOfPlane(pixelBuffer, planeIndex) - let height = CVPixelBufferGetHeightOfPlane(pixelBuffer, planeIndex) - - var textureRef : CVMetalTexture? - - let result = CVMetalTextureCacheCreateTextureFromImage( - nil, - textureCache, - pixelBuffer, - nil, - format, - width, - height, - planeIndex, - &textureRef - ) - - if result == kCVReturnSuccess, let textureRef = textureRef { - texture = CVMetalTextureGetTexture(textureRef) - } - - return texture + SceneKitHelpers.makePlane(size: planeSizeForDistance(distance, frame: frame), distance: distance) } } diff --git a/RealityMixer/Capture/ViewControllers/ChromaKeyConfigurationViewController.swift b/RealityMixer/Capture/ViewControllers/ChromaKeyConfigurationViewController.swift index 4c6574c..a7e0524 100644 --- a/RealityMixer/Capture/ViewControllers/ChromaKeyConfigurationViewController.swift +++ b/RealityMixer/Capture/ViewControllers/ChromaKeyConfigurationViewController.swift @@ -7,6 +7,7 @@ import UIKit import ARKit +import RealityMixerKit final class ChromaKeyConfigurationViewController: UIViewController { @@ -85,7 +86,7 @@ final class ChromaKeyConfigurationViewController: UIViewController { sceneView.scene = scene sceneView.session.delegate = self - ARKitHelpers.create(textureCache: &textureCache, for: sceneView) + SceneKitHelpers.create(textureCache: &textureCache, for: sceneView) } private func configureSliders() { @@ -112,7 +113,7 @@ final class ChromaKeyConfigurationViewController: UIViewController { private func configureBackgroundPlane(with frame: ARFrame) { let distance: Float = 100 let backgroundPlaneSize = ARKitHelpers.planeSizeForDistance(distance, frame: frame) - let backgroundPlaneNode = ARKitHelpers.makePlane(size: backgroundPlaneSize, distance: distance) + let backgroundPlaneNode = SceneKitHelpers.makePlane(size: backgroundPlaneSize, distance: distance) let planeMaterial = backgroundPlaneNode.geometry?.firstMaterial planeMaterial?.lightingModel = .constant @@ -139,8 +140,8 @@ final class ChromaKeyConfigurationViewController: UIViewController { } func updatePlaneImage(with pixelBuffer: CVPixelBuffer) { - let luma = ARKitHelpers.texture(from: pixelBuffer, format: .r8Unorm, planeIndex: 0, textureCache: textureCache) - let chroma = ARKitHelpers.texture(from: pixelBuffer, format: .rg8Unorm, planeIndex: 1, textureCache: textureCache) + let luma = SceneKitHelpers.texture(from: pixelBuffer, format: .r8Unorm, planeIndex: 0, textureCache: textureCache) + let chroma = SceneKitHelpers.texture(from: pixelBuffer, format: .rg8Unorm, planeIndex: 1, textureCache: textureCache) planeNode?.geometry?.firstMaterial?.transparent.contents = luma planeNode?.geometry?.firstMaterial?.diffuse.contents = chroma diff --git a/RealityMixer/Capture/ViewControllers/MixedRealityConnectionViewController.swift b/RealityMixer/Capture/ViewControllers/MixedRealityConnectionViewController.swift index 2be22f4..047ef30 100644 --- a/RealityMixer/Capture/ViewControllers/MixedRealityConnectionViewController.swift +++ b/RealityMixer/Capture/ViewControllers/MixedRealityConnectionViewController.swift @@ -7,6 +7,7 @@ import UIKit import SwiftSocket +import RealityMixerKit final class MixedRealityConnectionViewController: UIViewController { diff --git a/RealityMixer/Capture/ViewControllers/MixedRealityViewController.swift b/RealityMixer/Capture/ViewControllers/MixedRealityViewController.swift index af789d5..89909f3 100644 --- a/RealityMixer/Capture/ViewControllers/MixedRealityViewController.swift +++ b/RealityMixer/Capture/ViewControllers/MixedRealityViewController.swift @@ -9,6 +9,7 @@ import UIKit import ARKit import AVFoundation import SwiftSocket +import RealityMixerKit final class MixedRealityViewController: UIViewController { private let client: TCPClient @@ -115,10 +116,10 @@ final class MixedRealityViewController: UIViewController { sceneView.session.delegate = self if case .visible = configuration.backgroundLayerOptions.visibility { - sceneView.pointOfView?.addChildNode(ARKitHelpers.makePlane(size: .init(width: 9999, height: 9999), distance: 120)) + sceneView.pointOfView?.addChildNode(SceneKitHelpers.makePlane(size: .init(width: 9999, height: 9999), distance: 120)) } - ARKitHelpers.create(textureCache: &textureCache, for: sceneView) + SceneKitHelpers.create(textureCache: &textureCache, for: sceneView) } private func configureTap() { @@ -233,8 +234,8 @@ final class MixedRealityViewController: UIViewController { } private func updateForegroundBackground(with pixelBuffer: CVPixelBuffer) { - let luma = ARKitHelpers.texture(from: pixelBuffer, format: .r8Unorm, planeIndex: 0, textureCache: textureCache) - let chroma = ARKitHelpers.texture(from: pixelBuffer, format: .rg8Unorm, planeIndex: 1, textureCache: textureCache) + let luma = SceneKitHelpers.texture(from: pixelBuffer, format: .r8Unorm, planeIndex: 0, textureCache: textureCache) + let chroma = SceneKitHelpers.texture(from: pixelBuffer, format: .rg8Unorm, planeIndex: 1, textureCache: textureCache) backgroundNode?.geometry?.firstMaterial?.transparent.contents = luma backgroundNode?.geometry?.firstMaterial?.diffuse.contents = chroma @@ -245,8 +246,8 @@ final class MixedRealityViewController: UIViewController { private func updateMiddle(with pixelBuffer: CVPixelBuffer) { guard case .greenScreen = configuration.captureMode else { return } - let luma = ARKitHelpers.texture(from: pixelBuffer, format: .r8Unorm, planeIndex: 0, textureCache: textureCache) - let chroma = ARKitHelpers.texture(from: pixelBuffer, format: .rg8Unorm, planeIndex: 1, textureCache: textureCache) + let luma = SceneKitHelpers.texture(from: pixelBuffer, format: .r8Unorm, planeIndex: 0, textureCache: textureCache) + let chroma = SceneKitHelpers.texture(from: pixelBuffer, format: .rg8Unorm, planeIndex: 1, textureCache: textureCache) middlePlaneNode?.geometry?.firstMaterial?.transparent.contents = luma middlePlaneNode?.geometry?.firstMaterial?.diffuse.contents = chroma diff --git a/RealityMixer/Capture/Misc/ChromaKeyMaskBuilder.swift b/RealityMixerKit/ChromaKey/ChromaKeyMaskBuilder.swift similarity index 56% rename from RealityMixer/Capture/Misc/ChromaKeyMaskBuilder.swift rename to RealityMixerKit/ChromaKey/ChromaKeyMaskBuilder.swift index 532ad14..d808ba8 100644 --- a/RealityMixer/Capture/Misc/ChromaKeyMaskBuilder.swift +++ b/RealityMixerKit/ChromaKey/ChromaKeyMaskBuilder.swift @@ -5,27 +5,31 @@ // Created by Fabio de Albuquerque Dela Antonio on 27/06/2021. // -import UIKit +#if os(iOS) import ARKit +#elseif os(macOS) +import AppKit +#endif + import SceneKit import CoreImage.CIFilterBuiltins // Consider replacing this with Metal (MTLComputeCommandEncoder) // to reduce the SceneKit overhead. -final class ChromaKeyMaskBuilder { - - static func buildMask(for frame: ARFrame, chromaConfiguration: ChromaKeyConfiguration) -> UIImage? { - - let capturedImage = frame.capturedImage - let imageSize = frame.camera.imageResolution +public final class ChromaKeyMaskBuilder { + static func buildMask( + for capturedImage: CVPixelBuffer, + chromaConfiguration: ChromaKeyConfiguration + ) -> CIImage? { guard let device = MTLCreateSystemDefaultDevice() else { return nil } + let imageSize = SceneKitHelpers.size(from: capturedImage) let renderer = SCNRenderer(device: device, options: nil) var cache: CVMetalTextureCache? - ARKitHelpers.create(textureCache: &cache, forDevice: device) + SceneKitHelpers.create(textureCache: &cache, forDevice: device) guard let textureCache = cache else { return nil } @@ -47,7 +51,7 @@ final class ChromaKeyMaskBuilder { } }() - let planeNode = ARKitHelpers.makePlane(size: planeSize, distance: 1) + let planeNode = SceneKitHelpers.makePlane(size: planeSize, distance: 1) planeNode.geometry?.firstMaterial?.transparencyMode = .rgbZero @@ -67,8 +71,8 @@ final class ChromaKeyMaskBuilder { forKey: "sensitivity" ) - let luma = ARKitHelpers.texture(from: capturedImage, format: .r8Unorm, planeIndex: 0, textureCache: textureCache) - let chroma = ARKitHelpers.texture(from: capturedImage, format: .rg8Unorm, planeIndex: 1, textureCache: textureCache) + let luma = SceneKitHelpers.texture(from: capturedImage, format: .r8Unorm, planeIndex: 0, textureCache: textureCache) + let chroma = SceneKitHelpers.texture(from: capturedImage, format: .rg8Unorm, planeIndex: 1, textureCache: textureCache) planeNode.geometry?.firstMaterial?.transparent.contents = luma planeNode.geometry?.firstMaterial?.diffuse.contents = chroma @@ -77,12 +81,40 @@ final class ChromaKeyMaskBuilder { renderer.scene = scene let initialMask = renderer.snapshot(atTime: 0, with: imageSize, antialiasingMode: SCNAntialiasingMode.none) - // Eroding the white area +#if os(macOS) + let ciImage = initialMask.ciImage() +#elseif os(iOS) let ciImage = CIImage(image: initialMask) +#endif + + // Eroding the white area let filter = CIFilter.morphologyMinimum() filter.inputImage = ciImage filter.radius = 10 - return filter.outputImage.flatMap(uiImage(from:)) + return filter.outputImage + } + +#if os(macOS) + public static func buildMask( + for capturedImage: CVPixelBuffer, + chromaConfiguration: ChromaKeyConfiguration + ) -> NSImage? { + let ciImage: CIImage? = buildMask(for: capturedImage, chromaConfiguration: chromaConfiguration) + return ciImage.flatMap(nsImage(from:)) + } + + static func nsImage(from inputImage: CIImage) -> NSImage { + let rep = NSCIImageRep(ciImage: inputImage) + let nsImage = NSImage(size: rep.size) + nsImage.addRepresentation(rep) + return nsImage + } + +#elseif os(iOS) + public static func buildMask(for frame: ARFrame, chromaConfiguration: ChromaKeyConfiguration) -> UIImage? { + let capturedImage = frame.capturedImage + let ciImage = buildMask(for: capturedImage, chromaConfiguration: chromaConfiguration) + return ciImage.flatMap(uiImage(from:)) } static func uiImage(from inputImage: CIImage) -> UIImage? { @@ -94,4 +126,24 @@ final class ChromaKeyMaskBuilder { return UIImage(cgImage: cgImage) } +#endif +} + + +#if os(macOS) + +private extension NSImage { + + func ciImage() -> CIImage? { + guard let data = self.tiffRepresentation, + let bitmap = NSBitmapImageRep(data: data) + else { + return nil + } + + let ci = CIImage(bitmapImageRep: bitmap) + return ci + } } + +#endif diff --git a/RealityMixerKit/Configuration/ChromaKeyConfiguration.swift b/RealityMixerKit/Configuration/ChromaKeyConfiguration.swift new file mode 100644 index 0000000..f403598 --- /dev/null +++ b/RealityMixerKit/Configuration/ChromaKeyConfiguration.swift @@ -0,0 +1,174 @@ +// +// ChromaKeyConfiguration.swift +// RealityMixer +// +// Created by Fabio de Albuquerque Dela Antonio on 23/05/2021. +// + +#if os(macOS) +import AppKit +#elseif os(iOS) +import UIKit +#endif + +public struct ChromaKeyConfiguration: Codable { + + public struct Color: Codable { + public let red: Float // 0 .. 1 + public let green: Float // 0 .. 1 + public let blue: Float // 0 .. 1 + +#if os(macOS) + public init(nsColor: NSColor) { + var red: CGFloat = 0 + var green: CGFloat = 0 + var blue: CGFloat = 0 + nsColor.getRed(&red, green: &green, blue: &blue, alpha: nil) + self.red = Float(red) + self.green = Float(green) + self.blue = Float(blue) + } + + public var nsColor: NSColor { + NSColor(red: CGFloat(red), green: CGFloat(green), blue: CGFloat(blue), alpha: 1) + } +#elseif os(iOS) + public init(uiColor: UIColor) { + var red: CGFloat = 0 + var green: CGFloat = 0 + var blue: CGFloat = 0 + uiColor.getRed(&red, green: &green, blue: &blue, alpha: nil) + self.red = Float(red) + self.green = Float(green) + self.blue = Float(blue) + } + + public var uiColor: UIColor { + UIColor(red: CGFloat(red), green: CGFloat(green), blue: CGFloat(blue), alpha: 1) + } +#endif + } + + public let color: Color + public let sensitivity: Float // 0 .. 1 + public let smoothness: Float // 0 .. 1 + + public init(color: Color, sensitivity: Float, smoothness: Float) { + self.color = color + self.sensitivity = sensitivity + self.smoothness = smoothness + } +} + +// MARK: - Codable + +public extension ChromaKeyConfiguration.Color { + + enum CodingKeys: String, CodingKey { + case red + case green + case blue + } + + init(from decoder: Decoder) throws { + let values = try decoder.container(keyedBy: CodingKeys.self) + self.red = max(0.0, min(1.0, try values.decode(Float.self, forKey: .red))) + self.green = max(0.0, min(1.0, try values.decode(Float.self, forKey: .green))) + self.blue = max(0.0, min(1.0, try values.decode(Float.self, forKey: .blue))) + } + + func encode(to encoder: Encoder) throws { + var container = encoder.container(keyedBy: CodingKeys.self) + try container.encode(red, forKey: .red) + try container.encode(green, forKey: .green) + try container.encode(blue, forKey: .blue) + } +} + +// MARK: - Storage + +public final class ChromaKeyConfigurationStorage { + private let storage: UserDefaultsStorage + + public init(_ deviceId: String? = nil) { + if let deviceId = deviceId { + self.storage = .init(key: "ChromaConfiguration-\(deviceId)") + } else { + self.storage = .init(key: "ChromaConfiguration") + } + } + + public func save(configuration: ChromaKeyConfiguration) throws { + try storage.save(configuration) + } + + public var configuration: ChromaKeyConfiguration? { + storage.object + } +} + +public final class ChromaKeyMaskStorage { + private let manager = FileManager.default + private let maskFileName: String + + public init(_ deviceId: String? = nil) { + if let deviceId = deviceId { + let sanitized = deviceId.components(separatedBy: CharacterSet.alphanumerics.inverted).joined() + self.maskFileName = "\(sanitized)-mask.png" + } else { + self.maskFileName = "mask.png" + } + } + + var maskURL: URL? { + manager + .urls(for: .documentDirectory, in: .userDomainMask) + .first? + .appendingPathComponent(maskFileName) + } + +#if os(macOS) + public func update(mask: NSImage?) throws { + guard let maskURL = maskURL else { return } + + if let mask = mask { + try mask.pngData()?.write(to: maskURL) + } else { + try manager.removeItem(at: maskURL) + } + } + + public func load() -> NSImage? { + maskURL.flatMap(NSImage.init(contentsOf:)) + } + +#elseif os(iOS) + public func update(mask: UIImage?) throws { + guard let maskURL = maskURL else { return } + + if let mask = mask { + try mask.pngData()?.write(to: maskURL) + } else { + try manager.removeItem(at: maskURL) + } + } + + public func load() -> UIImage? { + maskURL.flatMap({ try? Data(contentsOf: $0) }).flatMap(UIImage.init(data:)) + } +#endif +} + +#if os(macOS) + +private extension NSImage { + + func pngData() -> Data? { + let cgImage = self.cgImage(forProposedRect: nil, context: nil, hints: nil) + let bitmapRep = cgImage.flatMap(NSBitmapImageRep.init(cgImage:)) + bitmapRep?.size = self.size + return bitmapRep?.representation(using: .png, properties: [:]) + } +} + +#endif diff --git a/RealityMixer/Capture/Configuration/UserDefaultsStorage.swift b/RealityMixerKit/Configuration/UserDefaultsStorage.swift similarity index 75% rename from RealityMixer/Capture/Configuration/UserDefaultsStorage.swift rename to RealityMixerKit/Configuration/UserDefaultsStorage.swift index 572facf..faef284 100644 --- a/RealityMixer/Capture/Configuration/UserDefaultsStorage.swift +++ b/RealityMixerKit/Configuration/UserDefaultsStorage.swift @@ -7,22 +7,22 @@ import Foundation -final class UserDefaultsStorage { +public final class UserDefaultsStorage { private let defaults: UserDefaults private let key: String - init(defaults: UserDefaults = .standard, key: String) { + public init(defaults: UserDefaults = .standard, key: String) { self.defaults = defaults self.key = key } - func save(_ object: T) throws { + public func save(_ object: T) throws { let data = try JSONEncoder().encode(object) let string = data.base64EncodedString() defaults.setValue(string, forKey: key) } - var object: T? { + public var object: T? { defaults.string(forKey: key) .flatMap({ Data(base64Encoded: $0) }) .flatMap({ try? JSONDecoder().decode(T.self, from: $0) }) diff --git a/RealityMixerKit/Helpers/SceneKitHelpers.swift b/RealityMixerKit/Helpers/SceneKitHelpers.swift new file mode 100644 index 0000000..05ad173 --- /dev/null +++ b/RealityMixerKit/Helpers/SceneKitHelpers.swift @@ -0,0 +1,94 @@ +// +// SceneKitHelpers.swift +// RealityMixerKit +// +// Created by Fabio Dela Antonio on 06/07/2022. +// + +import SceneKit + +public struct SceneKitHelpers { + + public static func makePlane(size: CGSize, distance: Float) -> SCNNode { + let plane = SCNPlane(width: size.width, height: size.height) + plane.cornerRadius = 0 + plane.firstMaterial?.lightingModel = .constant + +#if os(macOS) + plane.firstMaterial?.diffuse.contents = NSColor(red: 0, green: 0, blue: 0, alpha: 1) +#elseif os(iOS) + plane.firstMaterial?.diffuse.contents = UIColor(red: 0, green: 0, blue: 0, alpha: 1) +#endif + + let planeNode = SCNNode(geometry: plane) + planeNode.position = .init(0, 0, -distance) + return planeNode + } + + @discardableResult + public static func create(textureCache: inout CVMetalTextureCache?, for sceneView: SCNView) -> Bool { + guard let metalDevice = sceneView.device else { + return false + } + + return create(textureCache: &textureCache, forDevice: metalDevice) + } + + @discardableResult + public static func create(textureCache: inout CVMetalTextureCache?, forDevice metalDevice: MTLDevice) -> Bool { + let result = CVMetalTextureCacheCreate( + kCFAllocatorDefault, + nil, + metalDevice, + nil, + &textureCache + ) + + return result == kCVReturnSuccess + } + + public static func texture( + from pixelBuffer: CVPixelBuffer, + format: MTLPixelFormat, + planeIndex: Int, + textureCache: CVMetalTextureCache? + ) -> MTLTexture? { + guard let textureCache = textureCache, + planeIndex >= 0, planeIndex < CVPixelBufferGetPlaneCount(pixelBuffer) //, +// CVPixelBufferGetPixelFormatType(pixelBuffer) == kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange + else { + return nil + } + + var texture: MTLTexture? + + let width = CVPixelBufferGetWidthOfPlane(pixelBuffer, planeIndex) + let height = CVPixelBufferGetHeightOfPlane(pixelBuffer, planeIndex) + + var textureRef : CVMetalTexture? + + let result = CVMetalTextureCacheCreateTextureFromImage( + nil, + textureCache, + pixelBuffer, + nil, + format, + width, + height, + planeIndex, + &textureRef + ) + + if result == kCVReturnSuccess, let textureRef = textureRef { + texture = CVMetalTextureGetTexture(textureRef) + } + + return texture + } + + public static func size(from capturedImage: CVPixelBuffer) -> CGSize { + let width = CVPixelBufferGetWidthOfPlane(capturedImage, 0) + let height = CVPixelBufferGetHeightOfPlane(capturedImage, 0) + return CGSize(width: CGFloat(width), height: CGFloat(height)) + } +} diff --git a/RealityMixerKit/RealityMixerKit.h b/RealityMixerKit/RealityMixerKit.h new file mode 100644 index 0000000..21c7fe2 --- /dev/null +++ b/RealityMixerKit/RealityMixerKit.h @@ -0,0 +1,18 @@ +// +// RealityMixerKit.h +// RealityMixerKit +// +// Created by Fabio Dela Antonio on 05/07/2022. +// + +#import + +//! Project version number for RealityMixerKit. +FOUNDATION_EXPORT double RealityMixerKitVersionNumber; + +//! Project version string for RealityMixerKit. +FOUNDATION_EXPORT const unsigned char RealityMixerKitVersionString[]; + +// In this header, you should import all the public headers of your framework using statements like #import + + diff --git a/RealityMixer/Capture/Shaders/Shaders.swift b/RealityMixerKit/Shaders/Shaders.swift similarity index 93% rename from RealityMixer/Capture/Shaders/Shaders.swift rename to RealityMixerKit/Shaders/Shaders.swift index dc0d414..cb2ffd4 100644 --- a/RealityMixer/Capture/Shaders/Shaders.swift +++ b/RealityMixerKit/Shaders/Shaders.swift @@ -7,7 +7,7 @@ import Foundation -struct Shaders { +public struct Shaders { static let yCrCbToRGB = """ float BT709_nonLinearNormToLinear(float normV) { @@ -74,7 +74,7 @@ struct Shaders { """ - static let backgroundSurface = """ + public static let backgroundSurface = """ \(yCrCbToRGB) #pragma body @@ -110,7 +110,7 @@ struct Shaders { } // FIXME: avoid converting from yCbCr to RGB and then back to yCbCr... - static func surfaceChromaKey() -> String { + public static func surfaceChromaKey() -> String { """ \(yCrCbToRGB) \(smoothChromaKey) @@ -141,7 +141,7 @@ struct Shaders { """ } - static func surfaceChromaKeyConfiguration() -> String { + public static func surfaceChromaKeyConfiguration() -> String { """ \(yCrCbToRGB) \(smoothChromaKey) @@ -194,7 +194,7 @@ struct Shaders { """ } - static let backgroundSurfaceWithBlackChromaKey = """ + public static let backgroundSurfaceWithBlackChromaKey = """ \(yCrCbToRGB) #pragma body @@ -214,8 +214,8 @@ struct Shaders { """ - static let backgroundSurfaceWithGreenChromaKey = backgroundSurfaceChromaKey(red: 0, green: 1, blue: 0) - static let backgroundSurfaceWithMagentaChromaKey = backgroundSurfaceChromaKey(red: 1, green: 0, blue: 1) + public static let backgroundSurfaceWithGreenChromaKey = backgroundSurfaceChromaKey(red: 0, green: 1, blue: 0) + public static let backgroundSurfaceWithMagentaChromaKey = backgroundSurfaceChromaKey(red: 1, green: 0, blue: 1) static let foregroundSurfaceShared = """ \(yCrCbToRGB) @@ -230,7 +230,7 @@ struct Shaders { _surface.diffuse = yCbCrToRGB(luma, chroma); """ - static let foregroundSurface = """ + public static let foregroundSurface = """ \(foregroundSurfaceShared) vec2 alphaCoords = vec2((_surface.transparentTexcoord.x * 0.25) + 0.75, _surface.transparentTexcoord.y); @@ -249,7 +249,7 @@ struct Shaders { """ // Consider setting the smoothness - static let magentaForegroundSurface = """ + public static let magentaForegroundSurface = """ \(smoothChromaKey) \(foregroundSurfaceShared) diff --git a/RealityMixerMac/Camera/CameraSession.swift b/RealityMixerMac/Camera/CameraSession.swift new file mode 100644 index 0000000..66241d0 --- /dev/null +++ b/RealityMixerMac/Camera/CameraSession.swift @@ -0,0 +1,76 @@ +// +// CameraSession.swift +// RealityMixerMac +// +// Created by Fabio Dela Antonio on 06/07/2022. +// + +import Foundation +import AVFoundation + +enum CameraSessionError: Error { + case notAuthorized + case unableToAddVideoInput +} + +protocol CameraSessionDelegate: AnyObject { + func cameraSession(_ cameraSession: CameraSession, didReceiveFrame: CMSampleBuffer) +} + +final class CameraSession: NSObject { + + let device: AVCaptureDevice + private var captureSession: AVCaptureSession + weak var delegate: CameraSessionDelegate? + + init(device: AVCaptureDevice, delegate: CameraSessionDelegate) throws { + + guard case .authorized = AVCaptureDevice.authorizationStatus(for: .video) else { + throw CameraSessionError.notAuthorized + } + + let captureSession = AVCaptureSession() + self.captureSession = captureSession + self.delegate = delegate + self.device = device + + super.init() + + let videoInput = try AVCaptureDeviceInput(device: device) + + if captureSession.canAddInput(videoInput) { + captureSession.addInput(videoInput) + } else { + throw CameraSessionError.unableToAddVideoInput + } + + let videoOutput = AVCaptureVideoDataOutput() + videoOutput.setSampleBufferDelegate(self, queue: .main) + + if captureSession.canAddOutput(videoOutput) { + captureSession.addOutput(videoOutput) + videoOutput.videoSettings = [kCVPixelBufferPixelFormatTypeKey as String: kCVPixelFormatType_420YpCbCr8BiPlanarFullRange] + } else { + throw CameraSessionError.unableToAddVideoInput + } + } + + func startRunning() { + captureSession.startRunning() + } + + deinit { + captureSession.stopRunning() + } +} + +extension CameraSession: AVCaptureVideoDataOutputSampleBufferDelegate { + + func captureOutput( + _ output: AVCaptureOutput, + didOutput sampleBuffer: CMSampleBuffer, + from connection: AVCaptureConnection + ) { + delegate?.cameraSession(self, didReceiveFrame: sampleBuffer) + } +} diff --git a/RealityMixerMac/ChromaKey/Screens/ChromaKeyConfigurationScreen.swift b/RealityMixerMac/ChromaKey/Screens/ChromaKeyConfigurationScreen.swift new file mode 100644 index 0000000..defe08d --- /dev/null +++ b/RealityMixerMac/ChromaKey/Screens/ChromaKeyConfigurationScreen.swift @@ -0,0 +1,50 @@ +//// +//// ChromaKeyConfigurationScreen.swift +//// RealityMixerMac +//// +//// Created by Fabio Dela Antonio on 06/07/2022. +//// +// +//import SwiftUI +//import SceneKit +// +//protocol ChromaKeyConfigurationViewModelProtocol: ObservableObject { +// var state: ChromaKeyState { get } +// func onAppear() +// func navigateBack() +//} +// +//struct ChromaKeyConfigurationScreen: View { +// @StateObject var viewModel: ViewModel +// +// @ViewBuilder var content: some View { +// switch viewModel.state { +// case .notAuthorized: +// Text("Reality Mixer is no longer authorized to access your camera.") +// case .error: +// Text("Reality Mixer was not able to access your camera.") +// case .ready(let details): +// SceneView( +// scene: details.scene, +// pointOfView: details.pointOfView, +// options: [], +// delegate: details.rendererDelegate.delegate +// ) +// case .loading: +// ProgressView() +// .progressViewStyle(CircularProgressViewStyle()) +// } +// } +// +// var body: some View { +// VStack(spacing: 20) { +// content +// Button(action: viewModel.navigateBack) { +// Text("Go back") +// } +// } +// .padding() +// .frame(maxWidth: .infinity, maxHeight: .infinity) +// .onAppear(perform: viewModel.onAppear) +// } +//} diff --git a/RealityMixerMac/ChromaKey/ViewControllers/ChromaKeyConfigurationViewController.swift b/RealityMixerMac/ChromaKey/ViewControllers/ChromaKeyConfigurationViewController.swift new file mode 100644 index 0000000..db16f78 --- /dev/null +++ b/RealityMixerMac/ChromaKey/ViewControllers/ChromaKeyConfigurationViewController.swift @@ -0,0 +1,206 @@ +// +// ChromaKeyConfigurationViewController.swift +// RealityMixerMac +// +// Created by Fabio Dela Antonio on 07/07/2022. +// + +import Cocoa +import AVFoundation +import SceneKit +import RealityMixerKit + +class ChromaKeyConfigurationViewController: NSViewController { + + private let chromaConfigurationStorage: ChromaKeyConfigurationStorage + private let maskStorage: ChromaKeyMaskStorage + + private let device: AVCaptureDevice + private let onNavigateBack: () -> Void + + @IBOutlet private weak var sceneView: SCNView! + + @IBOutlet private weak var sensitivitySlider: NSSlider! + @IBOutlet private weak var sensitivityLabel: NSTextField! + @IBOutlet private weak var smoothnessSlider: NSSlider! + @IBOutlet private weak var smoothnessLabel: NSTextField! + @IBOutlet private weak var colorWell: NSColorWell! + @IBOutlet private weak var editMaskButton: NSButton! + + private static let defaultChromaColor = NSColor(red: 0, green: 1, blue: 0, alpha: 1) + + private var chromaColor: NSColor + private var maskImage: NSImage? + + private var textureCache: CVMetalTextureCache? + private var backgroundPlaneNode: SCNNode? + private var planeNode: SCNNode? + + private var first = true + + private var isPresentingColorPicker: Bool { + colorWell.isActive + } + + init(device: AVCaptureDevice, onNavigateBack: @escaping () -> Void) { + self.device = device + self.onNavigateBack = onNavigateBack + self.chromaColor = Self.defaultChromaColor + self.chromaConfigurationStorage = ChromaKeyConfigurationStorage(device.uniqueID) + self.maskStorage = ChromaKeyMaskStorage(device.uniqueID) + super.init(nibName: String(describing: type(of: self)), bundle: Bundle(for: type(of: self))) + } + + required init?(coder: NSCoder) { + fatalError("init(coder:) has not been implemented") + } + + override func viewDidLoad() { + super.viewDidLoad() + configureScene() + configureSliders() + resetValues() + + if let currentConfiguration = chromaConfigurationStorage.configuration { + sensitivitySlider.floatValue = currentConfiguration.sensitivity + smoothnessSlider.floatValue = currentConfiguration.smoothness + chromaColor = currentConfiguration.color.nsColor + colorWell.color = chromaColor + updateValueLabels() + } + + maskImage = maskStorage.load() + updateMaskButton() + } + + private func configureScene() { + let scene = SCNScene() + sceneView.scene = scene + + SceneKitHelpers.create(textureCache: &textureCache, for: sceneView) + } + + private func configureSliders() { + sensitivitySlider.minValue = 0.0 + sensitivitySlider.maxValue = 0.6 + + smoothnessSlider.minValue = 0 + smoothnessSlider.maxValue = 0.1 + } + + private func resetValues() { + sensitivitySlider.doubleValue = 0.15 + smoothnessSlider.doubleValue = 0 + chromaColor = Self.defaultChromaColor + colorWell.color = chromaColor + updateValueLabels() + } + + func updateValueLabels() { + sensitivityLabel.stringValue = String(format: "%.2lf", sensitivitySlider.doubleValue) + smoothnessLabel.stringValue = String(format: "%.2lf", smoothnessSlider.doubleValue) + } + + func updateMaskButton() { + if maskImage != nil { + editMaskButton.title = "Remove Mask" + } else { + editMaskButton.title = "Add Mask" + } + } + + func didUpdateValues() { + var red: CGFloat = 0 + var green: CGFloat = 0 + var blue: CGFloat = 0 + chromaColor.getRed(&red, green: &green, blue: &blue, alpha: nil) + + if let maskImage = maskImage, !isPresentingColorPicker { + planeNode?.geometry?.firstMaterial?.ambient.contents = maskImage + } else { + planeNode?.geometry?.firstMaterial?.ambient.contents = NSColor.white + } + + planeNode?.geometry?.firstMaterial?.setValue( + SCNVector3(red, green, blue), + forKey: "maskColor" + ) + + if isPresentingColorPicker { + planeNode?.geometry?.firstMaterial?.setValue(0.0, forKey: "sensitivity") + planeNode?.geometry?.firstMaterial?.setValue(0.0, forKey: "smoothness") + } else { + planeNode?.geometry?.firstMaterial?.setValue( + sensitivitySlider.doubleValue, + forKey: "sensitivity" + ) + + planeNode?.geometry?.firstMaterial?.setValue( + smoothnessSlider.doubleValue, + forKey: "smoothness" + ) + } + + updateValueLabels() + } + + func currentConfiguration() -> ChromaKeyConfiguration { + .init( + color: .init(nsColor: chromaColor), + sensitivity: sensitivitySlider.floatValue, + smoothness: smoothnessSlider.floatValue + ) + } + + // MARK: - Actions + + @IBAction func resetValuesAction(_ sender: Any) { + resetValues() + didUpdateValues() + } + + @IBAction private func editMaskAction(_ sender: Any) { + + } + + @IBAction private func saveAction(_ sender: Any) { + try? chromaConfigurationStorage.save(configuration: currentConfiguration()) + try? maskStorage.update(mask: maskImage) + onNavigateBack() + } + + @IBAction private func goBackAction(_ sender: Any) { + onNavigateBack() + } + + @IBAction func valueDidChange(_ sender: Any) { + didUpdateValues() + } + + @IBAction func colorDidChange(_ sender: Any) { + chromaColor = colorWell.color + didUpdateValues() + } +} + +extension ChromaKeyConfigurationViewController: CameraSessionDelegate { + + func cameraSession(_ cameraSession: CameraSession, didReceiveFrame: CMSampleBuffer) { + + } +} + +import SwiftUI + +struct ChromaKeyConfigurationScreen: NSViewControllerRepresentable { + typealias NSViewControllerType = ChromaKeyConfigurationViewController + let device: AVCaptureDevice + let onNavigateBack: () -> Void + + func makeNSViewController(context: Context) -> ChromaKeyConfigurationViewController { + ChromaKeyConfigurationViewController(device: device, onNavigateBack: onNavigateBack) + } + + func updateNSViewController(_ nsViewController: ChromaKeyConfigurationViewController, context: Context) { + } +} diff --git a/RealityMixerMac/ChromaKey/ViewControllers/ChromaKeyConfigurationViewController.xib b/RealityMixerMac/ChromaKey/ViewControllers/ChromaKeyConfigurationViewController.xib new file mode 100644 index 0000000..27e4c87 --- /dev/null +++ b/RealityMixerMac/ChromaKey/ViewControllers/ChromaKeyConfigurationViewController.xib @@ -0,0 +1,248 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/RealityMixerMac/ChromaKey/ViewModels/ChromaKeyConfigurationViewModel.swift b/RealityMixerMac/ChromaKey/ViewModels/ChromaKeyConfigurationViewModel.swift new file mode 100644 index 0000000..fae3464 --- /dev/null +++ b/RealityMixerMac/ChromaKey/ViewModels/ChromaKeyConfigurationViewModel.swift @@ -0,0 +1,254 @@ +//// +//// ChromaKeyConfigurationViewModel.swift +//// RealityMixerMac +//// +//// Created by Fabio Dela Antonio on 06/07/2022. +//// +// +//import SwiftUI +//import SceneKit +//import AVFoundation +//import RealityMixerKit +// +//struct ChromaKeyReadyState { +// let scene: SCNScene +// let pointOfView: SCNNode +// let rendererDelegate: WeakSCNSceneRendererDelegate +// let sensitivity: Float +// let smoothness: Float +// let color: NSColor +// let hasMask: Bool +//} +// +//final class WeakSCNSceneRendererDelegate { +// weak var delegate: SCNSceneRendererDelegate? +// +// init(delegate: SCNSceneRendererDelegate) { +// self.delegate = delegate +// } +//} +// +//enum ChromaKeyState { +// case loading +// case error +// case notAuthorized +// case ready(ChromaKeyReadyState) +//} +// +//final class ChromaKeyConfigurationViewModel: NSObject, ChromaKeyConfigurationViewModelProtocol { +// @Published var state: ChromaKeyState = .loading +// +// private let device: AVCaptureDevice? +// private var cameraSession: CameraSession? +// +// private var chromaConfigurationStorage: ChromaKeyConfigurationStorage? +// private var maskStorage: ChromaKeyMaskStorage? +// +// private let defaultChromaKeyColor = NSColor(red: 0, green: 1, blue: 0, alpha: 1) +// private let defaultSensitivity: Float = 0.15 +// private let defaultSmoothness: Float = 0 +// +// private var imageSize: CGSize? +// +// private var isSceneInitialized = false +// private let scene: SCNScene +// private let pointOfView: SCNNode +// private var chromaColor: NSColor +// private var maskImage: NSImage? +// private var sensitivity: Float +// private var smoothness: Float +// +// private var textureCache: CVMetalTextureCache? +// private var backgroundPlaneNode: SCNNode? +// private var planeNode: SCNNode? +// +// private let onNavigateBack: () -> Void +// +// init(device: AVCaptureDevice?, onNavigateBack: @escaping () -> Void) { +// self.device = device +// self.onNavigateBack = onNavigateBack +// +// self.chromaColor = defaultChromaKeyColor +// self.sensitivity = defaultSensitivity +// self.smoothness = defaultSmoothness +// +// self.scene = SCNScene() +// +// let camera = SCNCamera() +// camera.usesOrthographicProjection = true +// camera.orthographicScale = 1 +// +// let cameraNode = SCNNode() +// cameraNode.camera = camera +// cameraNode.position = .init(0, 0, 0) +// self.pointOfView = cameraNode +// self.scene.rootNode.addChildNode(cameraNode) +// } +// +// func onAppear() { +// guard case .loading = state else { +// return +// } +// +// guard let device = device else { +// self.state = .error +// return +// } +// +// do { +// self.cameraSession = try CameraSession(device: device, delegate: self) +// cameraSession?.startRunning() +// } catch CameraSessionError.notAuthorized { +// self.state = .notAuthorized +// } catch { +// self.state = .error +// } +// } +// +// private func initializeConfiguration(device: AVCaptureDevice) { +// self.chromaConfigurationStorage = ChromaKeyConfigurationStorage(device.uniqueID) +// self.maskStorage = ChromaKeyMaskStorage(device.uniqueID) +// +// if let currentConfiguration = chromaConfigurationStorage?.configuration { +// sensitivity = currentConfiguration.sensitivity +// smoothness = currentConfiguration.smoothness +// chromaColor = currentConfiguration.color.nsColor +// } +// +// self.maskImage = maskStorage?.load() +// +// self.state = .ready( +// .init( +// scene: scene, +// pointOfView: pointOfView, +// rendererDelegate: .init(delegate: self), +// sensitivity: sensitivity, +// smoothness: smoothness, +// color: chromaColor, +// hasMask: (maskImage != nil) +// ) +// ) +// } +// +// private func initializeScene(renderer: SCNSceneRenderer, imageSize: CGSize) { +// guard let metalDevice = renderer.device else { +// self.state = .error +// return +// } +// +// SceneKitHelpers.create(textureCache: &textureCache, forDevice: metalDevice) +// +// let planeSize: CGSize = { +// if imageSize.width > imageSize.height { +// return CGSize(width: 2 * (imageSize.width/imageSize.height), height: 2) +// } else { +// return CGSize(width: 2, height: 2 * (imageSize.height/imageSize.width)) +// } +// }() +// +// configureBackgroundPlane(with: imageSize, planeSize: planeSize) +// configurePlane(with: planeSize) +// didUpdateValues() +// +// isSceneInitialized = true +// } +// +// private func configureBackgroundPlane(with imageSize: CGSize, planeSize: CGSize) { +// let backgroundPlaneNode = SceneKitHelpers.makePlane(size: planeSize, distance: 1) +// +// let planeMaterial = backgroundPlaneNode.geometry?.firstMaterial +// planeMaterial?.lightingModel = .constant +// planeMaterial?.diffuse.contents = NSImage(named: "tile") +// +// let repeatX = (imageSize.width/6.4).rounded() +// let repeatY = (imageSize.height/6.4).rounded() +// planeMaterial?.diffuse.contentsTransform = SCNMatrix4MakeScale(repeatX, repeatY, 0) +// +// planeMaterial?.diffuse.wrapS = SCNWrapMode.repeat +// planeMaterial?.diffuse.wrapT = SCNWrapMode.repeat +// +// scene.rootNode.addChildNode(backgroundPlaneNode) +// self.backgroundPlaneNode = backgroundPlaneNode +// } +// +// private func configurePlane(with planeSize: CGSize) { +// let planeNode = SceneKitHelpers.makePlane(size: planeSize, distance: 0.1) +// planeNode.geometry?.firstMaterial?.lightingModel = .constant +// planeNode.geometry?.firstMaterial?.transparencyMode = .rgbZero +// planeNode.geometry?.firstMaterial?.shaderModifiers = [.surface: Shaders.surfaceChromaKeyConfiguration()] +// scene.rootNode.addChildNode(planeNode) +// self.planeNode = planeNode +// } +// +// private func updatePlaneImage(with pixelBuffer: CVPixelBuffer) { +// let luma = SceneKitHelpers.texture(from: pixelBuffer, format: .r8Unorm, planeIndex: 0, textureCache: textureCache) +// let chroma = SceneKitHelpers.texture(from: pixelBuffer, format: .rg8Unorm, planeIndex: 1, textureCache: textureCache) +// +// planeNode?.geometry?.firstMaterial?.transparent.contents = luma +// planeNode?.geometry?.firstMaterial?.diffuse.contents = chroma +// } +// +// private func didUpdateValues() { +// var red: CGFloat = 0 +// var green: CGFloat = 0 +// var blue: CGFloat = 0 +// chromaColor.getRed(&red, green: &green, blue: &blue, alpha: nil) +// +// if let maskImage = maskImage/*, !isPresentingColorPicker */ { +// planeNode?.geometry?.firstMaterial?.ambient.contents = maskImage +// } else { +// planeNode?.geometry?.firstMaterial?.ambient.contents = NSColor.white +// } +// +// planeNode?.geometry?.firstMaterial?.setValue( +// SCNVector3(red, green, blue), +// forKey: "maskColor" +// ) +// +//// if isPresentingColorPicker { +//// planeNode?.geometry?.firstMaterial?.setValue(0.0, forKey: "sensitivity") +//// planeNode?.geometry?.firstMaterial?.setValue(0.0, forKey: "smoothness") +//// } else { +// planeNode?.geometry?.firstMaterial?.setValue( +// sensitivity, +// forKey: "sensitivity" +// ) +// +// planeNode?.geometry?.firstMaterial?.setValue( +// smoothness, +// forKey: "smoothness" +// ) +//// } +// } +// +// func navigateBack() { +// onNavigateBack() +// } +//} +// +//extension ChromaKeyConfigurationViewModel: SCNSceneRendererDelegate { +// +// func renderer(_ renderer: SCNSceneRenderer, updateAtTime time: TimeInterval) { +// DispatchQueue.main.async { +// if !self.isSceneInitialized, let imageSize = self.imageSize { +// self.initializeScene(renderer: renderer, imageSize: imageSize) +// } +// } +// } +//} +// +//extension ChromaKeyConfigurationViewModel: CameraSessionDelegate { +// +// func cameraSession(_ cameraSession: CameraSession, didReceiveFrame buffer: CMSampleBuffer) { +// guard let imageBuffer = CMSampleBufferGetImageBuffer(buffer) else { return } +// +// // Assuming that this size won't change during this session +// self.imageSize = SceneKitHelpers.size(from: imageBuffer) +// +// if isSceneInitialized { +// updatePlaneImage(with: imageBuffer) +// } else { +// initializeConfiguration(device: cameraSession.device) +// } +// } +//} diff --git a/RealityMixerMac/Main/Screens/CameraSelectionScreen.swift b/RealityMixerMac/Main/Screens/CameraSelectionScreen.swift new file mode 100644 index 0000000..af44ac2 --- /dev/null +++ b/RealityMixerMac/Main/Screens/CameraSelectionScreen.swift @@ -0,0 +1,107 @@ +// +// CameraSelectionScreen.swift +// RealityMixerMac +// +// Created by Fabio Dela Antonio on 05/07/2022. +// + +import SwiftUI + +protocol CameraSelectionViewModelProtocol: ObservableObject { + var state: CameraSelectionState { get } + func onAppear() + func select(_ camera: CameraOption) + func openPrivacySettings() +} + +struct CameraSelectionScreen: View { + @StateObject var viewModel: ViewModel + + @ViewBuilder var content: some View { + switch viewModel.state { + case .notAuthorized: + VStack(spacing: 20) { + Text("Reality Mixer is not authorized to access your camera.\nPlease update your privacy preferences.") + .multilineTextAlignment(.leading) + Button(action: viewModel.openPrivacySettings) { + Text("Privacy Settings") + } + } + case .options(let cameras): + CameraPicker(cameras: cameras, onCameraSelection: viewModel.select) + case .loading: + ProgressView() + .progressViewStyle(CircularProgressViewStyle()) + } + } + + var body: some View { + content + .padding() + .frame(maxWidth: .infinity, maxHeight: .infinity) + .onAppear(perform: viewModel.onAppear) + } +} + +struct CameraPicker: View { + let cameras: [CameraOption] + let onCameraSelection: (CameraOption) -> Void + @State var selectedCamera: CameraOption? + + var body: some View { + VStack(spacing: 20) { + Text("Please choose the camera you want to use:") + Menu { + ForEach(cameras) { camera in + Button(action: { selectedCamera = camera }) { + Text("\(camera.localizedName)") + } + } + } label: { + if let selectedCamera = selectedCamera { + Text(selectedCamera.localizedName) + } else if cameras.isEmpty { + Text("No cameras found...") + } else { + Text("Cameras") + } + } + Button(action: { + guard let selection = selectedCamera else { return } + onCameraSelection(selection) + }) { + Text("OK") + } + .disabled(selectedCamera == nil) + } + } +} + +struct CameraSelectionScreen_Previews: PreviewProvider { + final class FakeViewModel: CameraSelectionViewModelProtocol { + let state: CameraSelectionState + func onAppear() { } + func select(_ camera: CameraOption) { } + func openPrivacySettings() { } + + init(_ state: CameraSelectionState) { + self.state = state + } + } + + static var previews: some View { + CameraSelectionScreen(viewModel: + FakeViewModel(.loading) + ) + + CameraSelectionScreen(viewModel: + FakeViewModel(.notAuthorized) + ) + + CameraSelectionScreen(viewModel: + FakeViewModel( + .options([CameraOption(id: "1", localizedName: "Camera 1")]) + ) + ) + } +} diff --git a/RealityMixerMac/Main/Screens/CameraTestScreen.swift b/RealityMixerMac/Main/Screens/CameraTestScreen.swift new file mode 100644 index 0000000..de00678 --- /dev/null +++ b/RealityMixerMac/Main/Screens/CameraTestScreen.swift @@ -0,0 +1,58 @@ +// +// CameraTestScreen.swift +// RealityMixerMac +// +// Created by Fabio Dela Antonio on 05/07/2022. +// + +import SwiftUI + +protocol CameraTestViewModelProtocol: ObservableObject { + var state: CameraTestState { get } + func onAppear() + func navigateBack() +} + +struct CameraTestScreen: View { + @StateObject var viewModel: ViewModel + + @ViewBuilder var content: some View { + switch viewModel.state { + case .notAuthorized: + Text("Reality Mixer is no longer authorized to access your camera.") + case .error: + Text("Reality Mixer was not able to access your camera.") + case .ready(let image): + if let image = image { + GeometryReader { geometry in + Image(nsImage: image) + .resizable() + .scaledToFit() + .frame( + width: geometry.size.width, + height: geometry.size.height, + alignment: .center + ) + .clipped() + } + } else { + Color.black + } + case .loading: + ProgressView() + .progressViewStyle(CircularProgressViewStyle()) + } + } + + var body: some View { + VStack(spacing: 20) { + content + Button(action: viewModel.navigateBack) { + Text("Go back") + } + } + .padding() + .frame(maxWidth: .infinity, maxHeight: .infinity) + .onAppear(perform: viewModel.onAppear) + } +} diff --git a/RealityMixerMac/Main/Screens/MainWindow.swift b/RealityMixerMac/Main/Screens/MainWindow.swift new file mode 100644 index 0000000..6e3133e --- /dev/null +++ b/RealityMixerMac/Main/Screens/MainWindow.swift @@ -0,0 +1,102 @@ +// +// MainWindow.swift +// RealityMixerMac +// +// Created by Fabio Dela Antonio on 05/07/2022. +// + +import SwiftUI +import AVFoundation + +struct MainWindow: View { + + @State var cameraSelection: AVCaptureDevice? + + @ViewBuilder var content: some View { + if case .none = cameraSelection { + CameraSelectionScreen( + viewModel: CameraSelectionViewModel( + cameraSelection: $cameraSelection + ) + ) + } else { + MainNavigation(cameraSelection: $cameraSelection) + } + } + + var body: some View { + content + .frame(minWidth: 600, minHeight: 400) + } +} + +enum ChildScreen { + case menu + case calibration + case mixedReality + case about + case testCamera + case chromaKey +} + +struct MainNavigation: View { + @Binding var cameraSelection: AVCaptureDevice? + @State var childScreen: ChildScreen = .menu + + @ViewBuilder var mainMenu: some View { + VStack(spacing: 20) { + Text("Camera: \(cameraSelection?.localizedName ?? "")") + + Button(action: { }) { + Text("Start Calibration") + } + + Button(action: { }) { + Text("Start Mixed Reality") + } + + Button(action: { }) { + Text("About") + } + + Button(action: { childScreen = .testCamera }) { + Text("Test Camera") + } + + Button(action: { childScreen = .chromaKey }) { + Text("Configure Chroma Key") + } + + Button(action: { cameraSelection = nil }) { + Text("Change Camera") + } + } + } + + var body: some View { + switch childScreen { + case .menu: + mainMenu + .padding() + .frame(maxWidth: .infinity, maxHeight: .infinity) + case .testCamera: + CameraTestScreen( + viewModel: CameraTestViewModel( + device: cameraSelection, + onNavigateBack: { childScreen = .menu } + ) + ) + case .chromaKey: + if let cameraSelection = cameraSelection { + ChromaKeyConfigurationScreen( + device: cameraSelection, + onNavigateBack: { childScreen = .menu } + ) + } else { + EmptyView() + } + default: + EmptyView() + } + } +} diff --git a/RealityMixerMac/Main/ViewModels/CameraSelectionViewModel.swift b/RealityMixerMac/Main/ViewModels/CameraSelectionViewModel.swift new file mode 100644 index 0000000..a3e7084 --- /dev/null +++ b/RealityMixerMac/Main/ViewModels/CameraSelectionViewModel.swift @@ -0,0 +1,86 @@ +// +// CameraSelectionViewModel.swift +// RealityMixerMac +// +// Created by Fabio Dela Antonio on 05/07/2022. +// + +import SwiftUI +import AVFoundation + +struct CameraOption: Identifiable { + let id: String + let localizedName: String + + init(id: String, localizedName: String) { + self.id = id + self.localizedName = localizedName + } + + init(captureDevice: AVCaptureDevice) { + self.id = captureDevice.uniqueID + self.localizedName = captureDevice.localizedName + } +} + +enum CameraSelectionState { + case loading + case notAuthorized + case options([CameraOption]) +} + +final class CameraSelectionViewModel: CameraSelectionViewModelProtocol { + @Published var state: CameraSelectionState = .loading + @Binding private var cameraSelection: AVCaptureDevice? + + init(cameraSelection: Binding) { + self._cameraSelection = cameraSelection + } + + func onAppear() { + switch AVCaptureDevice.authorizationStatus(for: .video) { + case .authorized: + discoverCameras() + + case .notDetermined: + AVCaptureDevice.requestAccess(for: .video) { [weak self] granted in + DispatchQueue.main.async { [weak self] in + if granted { + self?.discoverCameras() + } else { + self?.state = .notAuthorized + } + } + } + + case .denied, .restricted: + self.state = .notAuthorized + @unknown default: + self.state = .notAuthorized + } + } + + func select(_ camera: CameraOption) { + guard case .options(let options) = state, + Set(options.map({ $0.id })).contains(camera.id), + let device = AVCaptureDevice(uniqueID: camera.id) + else { return } + + cameraSelection = device + } + + func openPrivacySettings() { + guard let url = URL(string: "x-apple.systempreferences:com.apple.preference.security?Privacy_Camera") else { return } + NSWorkspace.shared.open(url) + } + + private func discoverCameras() { + let discovery = AVCaptureDevice.DiscoverySession( + deviceTypes: [.builtInWideAngleCamera, .externalUnknown], + mediaType: .video, + position: .unspecified + ) + + self.state = .options(discovery.devices.map(CameraOption.init(captureDevice:))) + } +} diff --git a/RealityMixerMac/Main/ViewModels/CameraTestViewModel.swift b/RealityMixerMac/Main/ViewModels/CameraTestViewModel.swift new file mode 100644 index 0000000..c749021 --- /dev/null +++ b/RealityMixerMac/Main/ViewModels/CameraTestViewModel.swift @@ -0,0 +1,71 @@ +// +// CameraTestViewModel.swift +// RealityMixerMac +// +// Created by Fabio Dela Antonio on 06/07/2022. +// + +import SwiftUI +import AVFoundation + +enum CameraTestState { + case loading + case error + case notAuthorized + case ready(NSImage?) +} + +final class CameraTestViewModel: CameraTestViewModelProtocol { + @Published var state: CameraTestState = .loading + + private let device: AVCaptureDevice? + private let onNavigateBack: () -> Void + private var cameraSession: CameraSession? + + init(device: AVCaptureDevice?, onNavigateBack: @escaping () -> Void) { + self.device = device + self.onNavigateBack = onNavigateBack + } + + func onAppear() { + guard case .loading = state else { + return + } + + guard let device = device else { + self.state = .error + return + } + + do { + self.cameraSession = try CameraSession(device: device, delegate: self) + self.state = .ready(nil) + cameraSession?.startRunning() + } catch CameraSessionError.notAuthorized { + self.state = .notAuthorized + } catch { + self.state = .error + } + } + + func navigateBack() { + onNavigateBack() + } +} + +extension CameraTestViewModel: CameraSessionDelegate { + + func cameraSession(_ cameraSession: CameraSession, didReceiveFrame buffer: CMSampleBuffer) { + guard case .ready = state, + let imageBuffer = CMSampleBufferGetImageBuffer(buffer) + else { return } + + let ciImage = CIImage(cvPixelBuffer: imageBuffer) + + let rep = NSCIImageRep(ciImage: ciImage) + let nsImage = NSImage(size: rep.size) + nsImage.addRepresentation(rep) + + self.state = .ready(nsImage) + } +} diff --git a/RealityMixerMac/RealityMixerMac.entitlements b/RealityMixerMac/RealityMixerMac.entitlements new file mode 100644 index 0000000..4d6884f --- /dev/null +++ b/RealityMixerMac/RealityMixerMac.entitlements @@ -0,0 +1,16 @@ + + + + + com.apple.security.app-sandbox + + com.apple.security.cs.disable-library-validation + + com.apple.security.device.camera + + com.apple.security.files.user-selected.read-only + + com.apple.security.network.client + + + diff --git a/RealityMixerMac/RealityMixerMac.xcconfig b/RealityMixerMac/RealityMixerMac.xcconfig new file mode 100644 index 0000000..e90397f --- /dev/null +++ b/RealityMixerMac/RealityMixerMac.xcconfig @@ -0,0 +1,8 @@ +CODE_SIGN_IDENTITY= iPhone Developer +DEVELOPMENT_TEAM = ZTJ55TM37B +CODE_SIGN_STYLE = Automatic +ORGANIZATION_IDENTIFIER = com.bluenose +DEVELOPER_ENTITLEMENTS = +PROVISIONING_PROFILE_SPECIFIER = + +#include? "DeveloperSettings.xcconfig" diff --git a/RealityMixerMac/RealityMixerMacApp.swift b/RealityMixerMac/RealityMixerMacApp.swift new file mode 100644 index 0000000..501dca6 --- /dev/null +++ b/RealityMixerMac/RealityMixerMacApp.swift @@ -0,0 +1,18 @@ +// +// RealityMixerMacApp.swift +// RealityMixerMac +// +// Created by Fabio Dela Antonio on 05/07/2022. +// + +import SwiftUI + +@main +struct RealityMixerMacApp: App { + + var body: some Scene { + WindowGroup("Reality Mixer") { + MainWindow() + } + } +} diff --git a/RealityMixerMac/Resources/Assets.xcassets/AccentColor.colorset/Contents.json b/RealityMixerMac/Resources/Assets.xcassets/AccentColor.colorset/Contents.json new file mode 100644 index 0000000..eb87897 --- /dev/null +++ b/RealityMixerMac/Resources/Assets.xcassets/AccentColor.colorset/Contents.json @@ -0,0 +1,11 @@ +{ + "colors" : [ + { + "idiom" : "universal" + } + ], + "info" : { + "author" : "xcode", + "version" : 1 + } +} diff --git a/RealityMixerMac/Resources/Assets.xcassets/AppIcon.appiconset/1024.png b/RealityMixerMac/Resources/Assets.xcassets/AppIcon.appiconset/1024.png new file mode 100644 index 0000000..26ef5c8 Binary files /dev/null and b/RealityMixerMac/Resources/Assets.xcassets/AppIcon.appiconset/1024.png differ diff --git a/RealityMixerMac/Resources/Assets.xcassets/AppIcon.appiconset/128.png b/RealityMixerMac/Resources/Assets.xcassets/AppIcon.appiconset/128.png new file mode 100644 index 0000000..fa8b3ff Binary files /dev/null and b/RealityMixerMac/Resources/Assets.xcassets/AppIcon.appiconset/128.png differ diff --git a/RealityMixerMac/Resources/Assets.xcassets/AppIcon.appiconset/16.png b/RealityMixerMac/Resources/Assets.xcassets/AppIcon.appiconset/16.png new file mode 100644 index 0000000..a928c95 Binary files /dev/null and b/RealityMixerMac/Resources/Assets.xcassets/AppIcon.appiconset/16.png differ diff --git a/RealityMixerMac/Resources/Assets.xcassets/AppIcon.appiconset/256 1.png b/RealityMixerMac/Resources/Assets.xcassets/AppIcon.appiconset/256 1.png new file mode 100644 index 0000000..77af34e Binary files /dev/null and b/RealityMixerMac/Resources/Assets.xcassets/AppIcon.appiconset/256 1.png differ diff --git a/RealityMixerMac/Resources/Assets.xcassets/AppIcon.appiconset/256.png b/RealityMixerMac/Resources/Assets.xcassets/AppIcon.appiconset/256.png new file mode 100644 index 0000000..77af34e Binary files /dev/null and b/RealityMixerMac/Resources/Assets.xcassets/AppIcon.appiconset/256.png differ diff --git a/RealityMixerMac/Resources/Assets.xcassets/AppIcon.appiconset/32 1.png b/RealityMixerMac/Resources/Assets.xcassets/AppIcon.appiconset/32 1.png new file mode 100644 index 0000000..5addb46 Binary files /dev/null and b/RealityMixerMac/Resources/Assets.xcassets/AppIcon.appiconset/32 1.png differ diff --git a/RealityMixerMac/Resources/Assets.xcassets/AppIcon.appiconset/32.png b/RealityMixerMac/Resources/Assets.xcassets/AppIcon.appiconset/32.png new file mode 100644 index 0000000..5addb46 Binary files /dev/null and b/RealityMixerMac/Resources/Assets.xcassets/AppIcon.appiconset/32.png differ diff --git a/RealityMixerMac/Resources/Assets.xcassets/AppIcon.appiconset/512 1.png b/RealityMixerMac/Resources/Assets.xcassets/AppIcon.appiconset/512 1.png new file mode 100644 index 0000000..5b69197 Binary files /dev/null and b/RealityMixerMac/Resources/Assets.xcassets/AppIcon.appiconset/512 1.png differ diff --git a/RealityMixerMac/Resources/Assets.xcassets/AppIcon.appiconset/512.png b/RealityMixerMac/Resources/Assets.xcassets/AppIcon.appiconset/512.png new file mode 100644 index 0000000..5b69197 Binary files /dev/null and b/RealityMixerMac/Resources/Assets.xcassets/AppIcon.appiconset/512.png differ diff --git a/RealityMixerMac/Resources/Assets.xcassets/AppIcon.appiconset/64.png b/RealityMixerMac/Resources/Assets.xcassets/AppIcon.appiconset/64.png new file mode 100644 index 0000000..2f06809 Binary files /dev/null and b/RealityMixerMac/Resources/Assets.xcassets/AppIcon.appiconset/64.png differ diff --git a/RealityMixerMac/Resources/Assets.xcassets/AppIcon.appiconset/Contents.json b/RealityMixerMac/Resources/Assets.xcassets/AppIcon.appiconset/Contents.json new file mode 100644 index 0000000..f6b252f --- /dev/null +++ b/RealityMixerMac/Resources/Assets.xcassets/AppIcon.appiconset/Contents.json @@ -0,0 +1,68 @@ +{ + "images" : [ + { + "filename" : "16.png", + "idiom" : "mac", + "scale" : "1x", + "size" : "16x16" + }, + { + "filename" : "32.png", + "idiom" : "mac", + "scale" : "2x", + "size" : "16x16" + }, + { + "filename" : "32 1.png", + "idiom" : "mac", + "scale" : "1x", + "size" : "32x32" + }, + { + "filename" : "64.png", + "idiom" : "mac", + "scale" : "2x", + "size" : "32x32" + }, + { + "filename" : "128.png", + "idiom" : "mac", + "scale" : "1x", + "size" : "128x128" + }, + { + "filename" : "256.png", + "idiom" : "mac", + "scale" : "2x", + "size" : "128x128" + }, + { + "filename" : "256 1.png", + "idiom" : "mac", + "scale" : "1x", + "size" : "256x256" + }, + { + "filename" : "512.png", + "idiom" : "mac", + "scale" : "2x", + "size" : "256x256" + }, + { + "filename" : "512 1.png", + "idiom" : "mac", + "scale" : "1x", + "size" : "512x512" + }, + { + "filename" : "1024.png", + "idiom" : "mac", + "scale" : "2x", + "size" : "512x512" + } + ], + "info" : { + "author" : "xcode", + "version" : 1 + } +} diff --git a/RealityMixerMac/Resources/Assets.xcassets/Contents.json b/RealityMixerMac/Resources/Assets.xcassets/Contents.json new file mode 100644 index 0000000..73c0059 --- /dev/null +++ b/RealityMixerMac/Resources/Assets.xcassets/Contents.json @@ -0,0 +1,6 @@ +{ + "info" : { + "author" : "xcode", + "version" : 1 + } +} diff --git a/RealityMixerMac/Resources/Assets.xcassets/tile.imageset/Contents.json b/RealityMixerMac/Resources/Assets.xcassets/tile.imageset/Contents.json new file mode 100644 index 0000000..7627f3e --- /dev/null +++ b/RealityMixerMac/Resources/Assets.xcassets/tile.imageset/Contents.json @@ -0,0 +1,12 @@ +{ + "images" : [ + { + "filename" : "tile.png", + "idiom" : "universal" + } + ], + "info" : { + "author" : "xcode", + "version" : 1 + } +} diff --git a/RealityMixerMac/Resources/Assets.xcassets/tile.imageset/tile.png b/RealityMixerMac/Resources/Assets.xcassets/tile.imageset/tile.png new file mode 100644 index 0000000..bdf66e5 Binary files /dev/null and b/RealityMixerMac/Resources/Assets.xcassets/tile.imageset/tile.png differ