Skip to content

[WIP] macOS Version #111

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
fabio914 wants to merge 6 commits into
base: main
Choose a base branch
from
Draft

[WIP] macOS Version #111

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
347cbe4
Preparing project
fabio914 Jul 5, 2022
b3987bf
Camera picker
fabio914 Jul 5, 2022
10d21ab
Camera test
fabio914 Jul 6, 2022
68869ab
WIP Chroma Key
fabio914 Jul 6, 2022
ea4eece
Failed attempt
fabio914 Jul 6, 2022
a63da58
WIP
fabio914 Jul 7, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
513 changes: 494 additions & 19 deletions RealityMixer.xcodeproj/project.pbxproj
View file
Open in desktop

Large diffs are not rendered by default.

99 changes: 0 additions & 99 deletions RealityMixer/Capture/Configuration/ChromaKeyConfiguration.swift
View file
Open in desktop

This file was deleted.

1 change: 1 addition & 0 deletions RealityMixer/Capture/Configuration/MixedRealityConfiguration.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
//

import Foundation
import RealityMixerKit

struct MixedRealityConfiguration: Codable, Equatable {

Expand Down
1 change: 1 addition & 0 deletions RealityMixer/Capture/Configuration/NetworkConfiguration.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
//

import Foundation
import RealityMixerKit

struct NetworkConfiguration: Codable {
let address: String
Expand Down
75 changes: 2 additions & 73 deletions RealityMixer/Capture/Misc/ARKitHelpers.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
//

import ARKit
import RealityMixerKit

struct ARKitHelpers {

Expand All @@ -19,79 +20,7 @@ struct ARKitHelpers {
return CGSize(width: CGFloat(width), height: CGFloat(height))
}

static func makePlane(size: CGSize, distance: Float) -> SCNNode {
let plane = SCNPlane(width: size.width, height: size.height)
plane.cornerRadius = 0
plane.firstMaterial?.lightingModel = .constant
plane.firstMaterial?.diffuse.contents = UIColor(red: 0, green: 0, blue: 0, alpha: 1)

let planeNode = SCNNode(geometry: plane)
planeNode.position = .init(0, 0, -distance)
return planeNode
}

static func makePlaneNodeForDistance(_ distance: Float, frame: ARFrame) -> SCNNode {
makePlane(size: planeSizeForDistance(distance, frame: frame), distance: distance)
}

@discardableResult
static func create(textureCache: inout CVMetalTextureCache?, for sceneView: SCNView) -> Bool {
guard let metalDevice = sceneView.device else {
return false
}

return create(textureCache: &textureCache, forDevice: metalDevice)
}

@discardableResult
static func create(textureCache: inout CVMetalTextureCache?, forDevice metalDevice: MTLDevice) -> Bool {
let result = CVMetalTextureCacheCreate(
kCFAllocatorDefault,
nil,
metalDevice,
nil,
&textureCache
)

return result == kCVReturnSuccess
}

static func texture(
from pixelBuffer: CVPixelBuffer,
format: MTLPixelFormat,
planeIndex: Int,
textureCache: CVMetalTextureCache?
) -> MTLTexture? {
guard let textureCache = textureCache,
planeIndex >= 0, planeIndex < CVPixelBufferGetPlaneCount(pixelBuffer) //,
// CVPixelBufferGetPixelFormatType(pixelBuffer) == kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange
else {
return nil
}

var texture: MTLTexture?

let width = CVPixelBufferGetWidthOfPlane(pixelBuffer, planeIndex)
let height = CVPixelBufferGetHeightOfPlane(pixelBuffer, planeIndex)

var textureRef : CVMetalTexture?

let result = CVMetalTextureCacheCreateTextureFromImage(
nil,
textureCache,
pixelBuffer,
nil,
format,
width,
height,
planeIndex,
&textureRef
)

if result == kCVReturnSuccess, let textureRef = textureRef {
texture = CVMetalTextureGetTexture(textureRef)
}

return texture
SceneKitHelpers.makePlane(size: planeSizeForDistance(distance, frame: frame), distance: distance)
}
}
9 changes: 5 additions & 4 deletions RealityMixer/Capture/ViewControllers/ChromaKeyConfigurationViewController.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@

import UIKit
import ARKit
import RealityMixerKit

final class ChromaKeyConfigurationViewController: UIViewController {

Expand Down Expand Up @@ -85,7 +86,7 @@ final class ChromaKeyConfigurationViewController: UIViewController {
sceneView.scene = scene
sceneView.session.delegate = self

ARKitHelpers.create(textureCache: &textureCache, for: sceneView)
SceneKitHelpers.create(textureCache: &textureCache, for: sceneView)
}

private func configureSliders() {
Expand All @@ -112,7 +113,7 @@ final class ChromaKeyConfigurationViewController: UIViewController {
private func configureBackgroundPlane(with frame: ARFrame) {
let distance: Float = 100
let backgroundPlaneSize = ARKitHelpers.planeSizeForDistance(distance, frame: frame)
let backgroundPlaneNode = ARKitHelpers.makePlane(size: backgroundPlaneSize, distance: distance)
let backgroundPlaneNode = SceneKitHelpers.makePlane(size: backgroundPlaneSize, distance: distance)

let planeMaterial = backgroundPlaneNode.geometry?.firstMaterial
planeMaterial?.lightingModel = .constant
Expand All @@ -139,8 +140,8 @@ final class ChromaKeyConfigurationViewController: UIViewController {
}

func updatePlaneImage(with pixelBuffer: CVPixelBuffer) {
let luma = ARKitHelpers.texture(from: pixelBuffer, format: .r8Unorm, planeIndex: 0, textureCache: textureCache)
let chroma = ARKitHelpers.texture(from: pixelBuffer, format: .rg8Unorm, planeIndex: 1, textureCache: textureCache)
let luma = SceneKitHelpers.texture(from: pixelBuffer, format: .r8Unorm, planeIndex: 0, textureCache: textureCache)
let chroma = SceneKitHelpers.texture(from: pixelBuffer, format: .rg8Unorm, planeIndex: 1, textureCache: textureCache)

planeNode?.geometry?.firstMaterial?.transparent.contents = luma
planeNode?.geometry?.firstMaterial?.diffuse.contents = chroma
Expand Down
1 change: 1 addition & 0 deletions RealityMixer/Capture/ViewControllers/MixedRealityConnectionViewController.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@

import UIKit
import SwiftSocket
import RealityMixerKit

final class MixedRealityConnectionViewController: UIViewController {

Expand Down
13 changes: 7 additions & 6 deletions RealityMixer/Capture/ViewControllers/MixedRealityViewController.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ import UIKit
import ARKit
import AVFoundation
import SwiftSocket
import RealityMixerKit

final class MixedRealityViewController: UIViewController {
private let client: TCPClient
Expand Down Expand Up @@ -115,10 +116,10 @@ final class MixedRealityViewController: UIViewController {
sceneView.session.delegate = self

if case .visible = configuration.backgroundLayerOptions.visibility {
sceneView.pointOfView?.addChildNode(ARKitHelpers.makePlane(size: .init(width: 9999, height: 9999), distance: 120))
sceneView.pointOfView?.addChildNode(SceneKitHelpers.makePlane(size: .init(width: 9999, height: 9999), distance: 120))
}

ARKitHelpers.create(textureCache: &textureCache, for: sceneView)
SceneKitHelpers.create(textureCache: &textureCache, for: sceneView)
}

private func configureTap() {
Expand Down Expand Up @@ -233,8 +234,8 @@ final class MixedRealityViewController: UIViewController {
}

private func updateForegroundBackground(with pixelBuffer: CVPixelBuffer) {
let luma = ARKitHelpers.texture(from: pixelBuffer, format: .r8Unorm, planeIndex: 0, textureCache: textureCache)
let chroma = ARKitHelpers.texture(from: pixelBuffer, format: .rg8Unorm, planeIndex: 1, textureCache: textureCache)
let luma = SceneKitHelpers.texture(from: pixelBuffer, format: .r8Unorm, planeIndex: 0, textureCache: textureCache)
let chroma = SceneKitHelpers.texture(from: pixelBuffer, format: .rg8Unorm, planeIndex: 1, textureCache: textureCache)

backgroundNode?.geometry?.firstMaterial?.transparent.contents = luma
backgroundNode?.geometry?.firstMaterial?.diffuse.contents = chroma
Expand All @@ -245,8 +246,8 @@ final class MixedRealityViewController: UIViewController {

private func updateMiddle(with pixelBuffer: CVPixelBuffer) {
guard case .greenScreen = configuration.captureMode else { return }
let luma = ARKitHelpers.texture(from: pixelBuffer, format: .r8Unorm, planeIndex: 0, textureCache: textureCache)
let chroma = ARKitHelpers.texture(from: pixelBuffer, format: .rg8Unorm, planeIndex: 1, textureCache: textureCache)
let luma = SceneKitHelpers.texture(from: pixelBuffer, format: .r8Unorm, planeIndex: 0, textureCache: textureCache)
let chroma = SceneKitHelpers.texture(from: pixelBuffer, format: .rg8Unorm, planeIndex: 1, textureCache: textureCache)

middlePlaneNode?.geometry?.firstMaterial?.transparent.contents = luma
middlePlaneNode?.geometry?.firstMaterial?.diffuse.contents = chroma
Expand Down
Loading