diff --git a/RealityMixer.xcodeproj/project.pbxproj b/RealityMixer.xcodeproj/project.pbxproj index 2617b7c..7ca3985 100644 --- a/RealityMixer.xcodeproj/project.pbxproj +++ b/RealityMixer.xcodeproj/project.pbxproj @@ -752,16 +752,17 @@ ASSETCATALOG_COMPILER_GLOBAL_ACCENT_COLOR_NAME = AccentColor; CLANG_ENABLE_MODULES = YES; CODE_SIGN_STYLE = Automatic; - CURRENT_PROJECT_VERSION = 18; + CURRENT_PROJECT_VERSION = 19; FRAMEWORK_SEARCH_PATHS = "$(inherited)"; HEADER_SEARCH_PATHS = ""; INFOPLIST_FILE = "$(SRCROOT)/RealityMixer/Info.plist"; + IPHONEOS_DEPLOYMENT_TARGET = 15.0; LD_RUNPATH_SEARCH_PATHS = ( "$(inherited)", "@executable_path/Frameworks", ); LIBRARY_SEARCH_PATHS = "$(inherited)"; - MARKETING_VERSION = 0.6.0; + MARKETING_VERSION = 0.7.0; PRODUCT_BUNDLE_IDENTIFIER = "$(ORGANIZATION_IDENTIFIER).reality.mixer"; PRODUCT_NAME = "$(TARGET_NAME)"; SWIFT_OBJC_BRIDGING_HEADER = "RealityMixer/RealityMixer-Bridging-Header.h"; @@ -779,16 +780,17 @@ ASSETCATALOG_COMPILER_GLOBAL_ACCENT_COLOR_NAME = AccentColor; CLANG_ENABLE_MODULES = YES; CODE_SIGN_STYLE = Automatic; - CURRENT_PROJECT_VERSION = 18; + CURRENT_PROJECT_VERSION = 19; FRAMEWORK_SEARCH_PATHS = "$(inherited)"; HEADER_SEARCH_PATHS = ""; INFOPLIST_FILE = "$(SRCROOT)/RealityMixer/Info.plist"; + IPHONEOS_DEPLOYMENT_TARGET = 15.0; LD_RUNPATH_SEARCH_PATHS = ( "$(inherited)", "@executable_path/Frameworks", ); LIBRARY_SEARCH_PATHS = "$(inherited)"; - MARKETING_VERSION = 0.6.0; + MARKETING_VERSION = 0.7.0; PRODUCT_BUNDLE_IDENTIFIER = "$(ORGANIZATION_IDENTIFIER).reality.mixer"; PRODUCT_NAME = "$(TARGET_NAME)"; SWIFT_OBJC_BRIDGING_HEADER = "RealityMixer/RealityMixer-Bridging-Header.h"; diff --git a/RealityMixer/Capture/Misc/ARConfigurationFactory.swift b/RealityMixer/Capture/Misc/ARConfigurationFactory.swift index 7ee40d0..4bcd253 100644 --- a/RealityMixer/Capture/Misc/ARConfigurationFactory.swift +++ b/RealityMixer/Capture/Misc/ARConfigurationFactory.swift @@ -38,11 +38,11 @@ final class ARConfigurationFactory { private func buildPersonSegmentationConfiguration() -> ARConfiguration { let configuration = buildWorldTrackingConfiguration() - if ARWorldTrackingConfiguration.supportsFrameSemantics(.personSegmentationWithDepth) { - configuration.frameSemantics.insert(.personSegmentationWithDepth) - } else if ARWorldTrackingConfiguration.supportsFrameSemantics(.personSegmentation) { - configuration.frameSemantics.insert(.personSegmentation) - } +// if ARWorldTrackingConfiguration.supportsFrameSemantics(.personSegmentationWithDepth) { +// configuration.frameSemantics.insert(.personSegmentationWithDepth) +// } else if ARWorldTrackingConfiguration.supportsFrameSemantics(.personSegmentation) { +// configuration.frameSemantics.insert(.personSegmentation) +// } return configuration } diff --git a/RealityMixer/Capture/Misc/ARKitHelpers.swift b/RealityMixer/Capture/Misc/ARKitHelpers.swift index 2ce32ff..b4583fa 100644 --- a/RealityMixer/Capture/Misc/ARKitHelpers.swift +++ b/RealityMixer/Capture/Misc/ARKitHelpers.swift @@ -63,7 +63,7 @@ struct ARKitHelpers { textureCache: CVMetalTextureCache? ) -> MTLTexture? { guard let textureCache = textureCache, - planeIndex >= 0, planeIndex < CVPixelBufferGetPlaneCount(pixelBuffer) //, + planeIndex >= 0/*, planeIndex < CVPixelBufferGetPlaneCount(pixelBuffer)*/ //, // CVPixelBufferGetPixelFormatType(pixelBuffer) == kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange else { return nil diff --git a/RealityMixer/Capture/Shaders/Shaders.swift b/RealityMixer/Capture/Shaders/Shaders.swift index dc0d414..1009d67 100644 --- a/RealityMixer/Capture/Shaders/Shaders.swift +++ b/RealityMixer/Capture/Shaders/Shaders.swift @@ -141,6 +141,29 @@ struct Shaders { """ } + static func surfaceSegmentation() -> String { + """ + \(yCrCbToRGB) + + #pragma body + + float luma = texture2D(u_transparentTexture, _surface.diffuseTexcoord).r; + vec2 chroma = texture2D(u_diffuseTexture, _surface.diffuseTexcoord).rg; + + vec4 textureColor = yCbCrToRGB(luma, chroma); + _surface.diffuse = textureColor; + + float maskTextureValue = texture2D(u_ambientTexture, _surface.diffuseTexcoord).r; + _surface.ambient = vec4(1.0, 1.0, 1.0, 1.0); + + if (maskTextureValue > 0.5) { + _surface.transparent = vec4(0.0, 0.0, 0.0, 1.0); + } else { + _surface.transparent = vec4(1.0, 1.0, 1.0, 1.0); + } + """ + } + static func surfaceChromaKeyConfiguration() -> String { """ \(yCrCbToRGB) diff --git a/RealityMixer/Capture/ViewControllers/MixedRealityViewController.swift b/RealityMixer/Capture/ViewControllers/MixedRealityViewController.swift index 8b4a88d..45c088f 100644 --- a/RealityMixer/Capture/ViewControllers/MixedRealityViewController.swift +++ b/RealityMixer/Capture/ViewControllers/MixedRealityViewController.swift @@ -8,6 +8,7 @@ import UIKit import ARKit import AVFoundation +import Vision import SwiftSocket final class MixedRealityViewController: UIViewController { @@ -162,9 +163,16 @@ final class MixedRealityViewController: UIViewController { } private func configureMiddle(with frame: ARFrame) { - guard case .greenScreen = configuration.captureMode, - let chromaConfiguration = chromaConfiguration - else { return } + + if case .greenScreen = configuration.captureMode, + let chromaConfiguration = chromaConfiguration { + configureGreenScreenMiddle(with: frame, chromaConfiguration: chromaConfiguration) + } else if case .personSegmentation = configuration.captureMode { + configurePersonSegmentationMiddle(with: frame) + } + } + + private func configureGreenScreenMiddle(with frame: ARFrame, chromaConfiguration: ChromaKeyConfiguration) { let middlePlaneNode = ARKitHelpers.makePlaneNodeForDistance(0.02, frame: frame) middlePlaneNode.geometry?.firstMaterial?.transparencyMode = .rgbZero @@ -202,6 +210,21 @@ final class MixedRealityViewController: UIViewController { self.middlePlaneNode = middlePlaneNode } + private func configurePersonSegmentationMiddle(with frame: ARFrame) { + let middlePlaneNode = ARKitHelpers.makePlaneNodeForDistance(0.02, frame: frame) + + middlePlaneNode.geometry?.firstMaterial?.transparencyMode = .rgbZero + + middlePlaneNode.geometry?.firstMaterial?.shaderModifiers = [ + .surface: Shaders.surfaceSegmentation() + ] + + middlePlaneNode.geometry?.firstMaterial?.ambient.contents = UIColor.white + + sceneView.pointOfView?.addChildNode(middlePlaneNode) + self.middlePlaneNode = middlePlaneNode + } + private func configureForeground(with frame: ARFrame) { guard case .visible(let useMagentaAsTransparency) = configuration.foregroundLayerOptions.visibility else { return } let foregroundPlaneNode = ARKitHelpers.makePlaneNodeForDistance(0.01, frame: frame) @@ -243,7 +266,17 @@ final class MixedRealityViewController: UIViewController { } private func updateMiddle(with pixelBuffer: CVPixelBuffer) { - guard case .greenScreen = configuration.captureMode else { return } + switch configuration.captureMode { + case .greenScreen: + updateChromaKeyMiddle(with: pixelBuffer) + case .personSegmentation: + updatePersonSegmentationMiddle(with: pixelBuffer) + default: + break + } + } + + private func updateChromaKeyMiddle(with pixelBuffer: CVPixelBuffer) { let luma = ARKitHelpers.texture(from: pixelBuffer, format: .r8Unorm, planeIndex: 0, textureCache: textureCache) let chroma = ARKitHelpers.texture(from: pixelBuffer, format: .rg8Unorm, planeIndex: 1, textureCache: textureCache) @@ -251,6 +284,33 @@ final class MixedRealityViewController: UIViewController { middlePlaneNode?.geometry?.firstMaterial?.diffuse.contents = chroma } + private func updatePersonSegmentationMiddle(with pixelBuffer: CVPixelBuffer) { + let request = VNGeneratePersonSegmentationRequest() + request.qualityLevel = .fast + request.outputPixelFormat = kCVPixelFormatType_OneComponent8 + + let handler = VNImageRequestHandler(cvPixelBuffer: pixelBuffer, options: [:]) + + do { + try handler.perform([request]) + + guard let result = request.results?.first?.pixelBuffer else { + return + } + + let mask = ARKitHelpers.texture(from: result, format: .r8Unorm, planeIndex: 0, textureCache: textureCache) + middlePlaneNode?.geometry?.firstMaterial?.ambient.contents = mask + + let luma = ARKitHelpers.texture(from: pixelBuffer, format: .r8Unorm, planeIndex: 0, textureCache: textureCache) + let chroma = ARKitHelpers.texture(from: pixelBuffer, format: .rg8Unorm, planeIndex: 1, textureCache: textureCache) + + middlePlaneNode?.geometry?.firstMaterial?.transparent.contents = luma + middlePlaneNode?.geometry?.firstMaterial?.diffuse.contents = chroma + } catch { + print("Error: \(error)") + } + } + override func viewDidAppear(_ animated: Bool) { super.viewDidAppear(animated) prepareARConfiguration()