display(visionOS): integrate VisionKeyboardKit

2024-02-25 00:01:48 -08:00 · 2024-02-25 00:01:48 -08:00 · aa071bd0a5
parent 9835e2bc48
commit aa071bd0a5
6 changed files with 109 additions and 3 deletions
--- a/Platform/iOS/VMDisplayHostedView.swift
+++ b/Platform/iOS/VMDisplayHostedView.swift
@ -168,7 +168,12 @@ struct VMDisplayHostedView: UIViewControllerRepresentable {
        if let vc = uiViewController as? VMDisplayMetalViewController {
            vc.vmInput = session.primaryInput
        }
-        if state.isKeyboardShown != state.isKeyboardRequested {
+        #if os(visionOS)
+        let useSystemOsk = !(uiViewController is VMDisplayMetalViewController)
+        #else
+        let useSystemOsk = true
+        #endif
+        if useSystemOsk && state.isKeyboardShown != state.isKeyboardRequested {
            DispatchQueue.main.async {
                if state.isKeyboardRequested {
                    uiViewController.showKeyboard()
--- a/Platform/iOS/VMWindowView.swift
+++ b/Platform/iOS/VMWindowView.swift
@ -16,6 +16,9 @@

 import SwiftUI
 import SwiftUIVisualEffects
+#if os(visionOS)
+import VisionKeyboardKit
+#endif

 struct VMWindowView: View {
    let id: VMSessionState.WindowID
@ -24,7 +27,10 @@ struct VMWindowView: View {
    @State private var state: VMWindowState
    @EnvironmentObject private var session: VMSessionState
    @Environment(\.scenePhase) private var scenePhase
-    
+    #if os(visionOS)
+    @Environment(\.dismissWindow) private var dismissWindow
+    #endif
+
    private let keyboardDidShowNotification = NotificationCenter.default.publisher(for: UIResponder.keyboardDidShowNotification)
    private let keyboardDidHideNotification = NotificationCenter.default.publisher(for: UIResponder.keyboardDidHideNotification)
    private let didReceiveMemoryWarningNotification = NotificationCenter.default.publisher(for: UIApplication.didReceiveMemoryWarningNotification)
@ -228,6 +234,9 @@ struct VMWindowView: View {
            if !isInteractive {
                session.externalWindowBinding = nil
            }
+            #if os(visionOS)
+            dismissWindow(keyboardFor: state.id)
+            #endif
        }
    }
    
--- a/Platform/visionOS/UTMApp.swift
+++ b/Platform/visionOS/UTMApp.swift
@ -15,6 +15,7 @@
 //

 import SwiftUI
+import VisionKeyboardKit

@MainActor
 struct UTMApp: App {
@ -73,5 +74,6 @@ struct UTMApp: App {
        }
        .windowStyle(.plain)
        .windowResizability(.contentMinSize)
+        KeyboardWindowGroup()
    }
 }
--- a/Platform/visionOS/VMToolbarOrnamentModifier.swift
+++ b/Platform/visionOS/VMToolbarOrnamentModifier.swift
@ -15,11 +15,19 @@
 //

 import SwiftUI
+import VisionKeyboardKit
+#if !WITH_USB
+import CocoaSpiceNoUsb
+#else
+import CocoaSpice
+#endif

 struct VMToolbarOrnamentModifier: ViewModifier {
    @Binding var state: VMWindowState
    @EnvironmentObject private var session: VMSessionState
    @AppStorage("ToolbarIsCollapsed") private var isCollapsed: Bool = false
+    @Environment(\.openWindow) private var openWindow
+    @Environment(\.dismissWindow) private var dismissWindow

    func body(content: Content) -> some View {
        content.ornament(visibility: isCollapsed ? .hidden : .visible, attachmentAnchor: .scene(.top)) {
@ -71,11 +79,39 @@ struct VMToolbarOrnamentModifier: ViewModifier {
                VMToolbarDisplayMenuView(state: $state)
                    .disabled(state.isBusy)
                Button {
-                    state.isKeyboardRequested = true
+                    if case .display(_, _) = state.device {
+                        state.isKeyboardRequested = !state.isKeyboardShown
+                    } else {
+                        state.isKeyboardRequested = true
+                    }
                } label: {
                    Label("Keyboard", systemImage: "keyboard")
                }
                .disabled(state.isBusy)
+                .onChange(of: state.isKeyboardRequested) { _, newValue in
+                    guard case .display(_, _) = state.device else {
+                        return
+                    }
+                    if newValue {
+                        openWindow(keyboardFor: state.id)
+                    } else {
+                        dismissWindow(keyboardFor: state.id)
+                    }
+                }
+                .onReceive(KeyboardEvent.publisher(for: state.id)) { event in
+                    switch event {
+                    case .keyboardDidAppear:
+                        state.isKeyboardShown = true
+                        state.isKeyboardRequested = true
+                    case .keyboardDidDisappear:
+                        state.isKeyboardShown = false
+                        state.isKeyboardRequested = false
+                    case .keyUp(let keyCode, let modifier):
+                        handleKeyEvent(keyCode, modifier: modifier, isKeyDown: false)
+                    case .keyDown(let keyCode, let modifier):
+                        handleKeyEvent(keyCode, modifier: modifier, isKeyDown: true)
+                    }
+                }
                Divider()
                Button {
                    isCollapsed = true
@ -94,6 +130,18 @@ struct VMToolbarOrnamentModifier: ViewModifier {
                .modifier(ToolbarOrnamentViewModifier())
        }
    }
+
+    private func handleKeyEvent(_ keyCode: KeyboardKeyCode, modifier: KeyboardModifier, isKeyDown: Bool) {
+        guard let primaryInput = session.primaryInput else {
+            logger.debug("ignoring key event because input channel is not ready")
+            return
+        }
+        var scanCode = keyCode.ps2Set1ScanMake(modifier).reduce(Int32(0), { ($0 << 8) | Int32($1) })
+        if ((scanCode & 0xFF00) == 0xE000) {
+            scanCode = 0x100 | (scanCode & 0xFF);
+        }
+        primaryInput.send(isKeyDown ? .press : .release, code: scanCode)
+    }
 }

 // the following was suggested by Apple via Feedback to look close to .toolbar() with .bottomOrnament
--- a/UTM.xcodeproj/project.pbxproj
+++ b/UTM.xcodeproj/project.pbxproj
@ -646,6 +646,9 @@
 		CE8813D324CD230300532628 /* ActivityView.swift in Sources */ = {isa = PBXBuildFile; fileRef = CE8813D224CD230300532628 /* ActivityView.swift */; };
 		CE8813D524CD265700532628 /* VMShareFileModifier.swift in Sources */ = {isa = PBXBuildFile; fileRef = CE8813D424CD265700532628 /* VMShareFileModifier.swift */; };
 		CE8813D624CD265700532628 /* VMShareFileModifier.swift in Sources */ = {isa = PBXBuildFile; fileRef = CE8813D424CD265700532628 /* VMShareFileModifier.swift */; };
+		CE89CB0E2B8B1B5A006B2CC2 /* VisionKeyboardKit in Frameworks */ = {isa = PBXBuildFile; platformFilters = (xros, ); productRef = CE89CB0D2B8B1B5A006B2CC2 /* VisionKeyboardKit */; };
+		CE89CB102B8B1B6A006B2CC2 /* VisionKeyboardKit in Frameworks */ = {isa = PBXBuildFile; platformFilters = (xros, ); productRef = CE89CB0F2B8B1B6A006B2CC2 /* VisionKeyboardKit */; };
+		CE89CB122B8B1B7A006B2CC2 /* VisionKeyboardKit in Frameworks */ = {isa = PBXBuildFile; platformFilters = (xros, ); productRef = CE89CB112B8B1B7A006B2CC2 /* VisionKeyboardKit */; };
 		CE928C2A26ABE6690099F293 /* UTMAppleVirtualMachine.swift in Sources */ = {isa = PBXBuildFile; fileRef = CE928C2926ABE6690099F293 /* UTMAppleVirtualMachine.swift */; };
 		CE928C3126ACCDEA0099F293 /* VMAppleRemovableDrivesView.swift in Sources */ = {isa = PBXBuildFile; fileRef = CE928C3026ACCDEA0099F293 /* VMAppleRemovableDrivesView.swift */; };
 		CE93758924B930270074066F /* BusyOverlay.swift in Sources */ = {isa = PBXBuildFile; fileRef = CE7D972B24B2B17D0080CB69 /* BusyOverlay.swift */; };
@ -2121,6 +2124,7 @@
 				84818C0C2898A07A009EDB67 /* AVFAudio.framework in Frameworks */,
 				CE2D934924AD46670059923A /* gstreamer-1.0.0.framework in Frameworks */,
 				CE2D934B24AD46670059923A /* json-glib-1.0.0.framework in Frameworks */,
+				CE89CB0E2B8B1B5A006B2CC2 /* VisionKeyboardKit in Frameworks */,
 				CE2D934C24AD46670059923A /* ffi.7.framework in Frameworks */,
 				CE2D934D24AD46670059923A /* gstnet-1.0.0.framework in Frameworks */,
 				CE2D934E24AD46670059923A /* gstbase-1.0.0.framework in Frameworks */,
@ -2251,6 +2255,7 @@
 				CEA45F25263519B5002FA97D /* libgstaudiotestsrc.a in Frameworks */,
 				CEA45F26263519B5002FA97D /* libgstvideoconvert.a in Frameworks */,
 				CEA45F27263519B5002FA97D /* libgstaudioconvert.a in Frameworks */,
+				CE89CB102B8B1B6A006B2CC2 /* VisionKeyboardKit in Frameworks */,
 				8401865C2887AFDC0050AC51 /* SwiftTerm in Frameworks */,
 				CEA45F28263519B5002FA97D /* libgstvideoscale.a in Frameworks */,
 				CEA45F29263519B5002FA97D /* IQKeyboardManagerSwift in Frameworks */,
@ -2386,6 +2391,7 @@
 				CEF7F6692AEEDCC400E34952 /* opus.0.framework in Frameworks */,
 				CEF7F66A2AEEDCC400E34952 /* glib-2.0.0.framework in Frameworks */,
 				CEF7F66B2AEEDCC400E34952 /* png16.16.framework in Frameworks */,
+				CE89CB122B8B1B7A006B2CC2 /* VisionKeyboardKit in Frameworks */,
 				CEF7F66C2AEEDCC400E34952 /* gstfft-1.0.0.framework in Frameworks */,
 				CEF7F66D2AEEDCC400E34952 /* crypto.1.1.framework in Frameworks */,
 				CEF7F66E2AEEDCC400E34952 /* gstpbutils-1.0.0.framework in Frameworks */,
@ -3040,6 +3046,7 @@
 				84CE3DAB2904C14100FF068B /* InAppSettingsKit */,
 				84A0A8892A47D5D10038F329 /* QEMUKit */,
 				CE9B15372B11A4A7003A32DD /* SwiftConnect */,
+				CE89CB0D2B8B1B5A006B2CC2 /* VisionKeyboardKit */,
 			);
 			productName = UTM;
 			productReference = CE2D93BE24AD46670059923A /* UTM.app */;
@ -3121,6 +3128,7 @@
 				846D878529050B6B0095F10B /* InAppSettingsKit */,
 				84A0A88B2A47D5D70038F329 /* QEMUKit */,
 				CE9B15392B11A4AE003A32DD /* SwiftConnect */,
+				CE89CB0F2B8B1B6A006B2CC2 /* VisionKeyboardKit */,
 			);
 			productName = UTM;
 			productReference = CEA45FB9263519B5002FA97D /* UTM SE.app */;
@ -3169,6 +3177,7 @@
 				CEF7F5922AEEDCC400E34952 /* InAppSettingsKit */,
 				CEF7F6D52AEEEF7D00E34952 /* CocoaSpiceNoUsb */,
 				CE9B153B2B11A4B4003A32DD /* SwiftConnect */,
+				CE89CB112B8B1B7A006B2CC2 /* VisionKeyboardKit */,
 			);
 			productName = UTM;
 			productReference = CEF7F6D32AEEDCC400E34952 /* UTM Remote.app */;
@ -3240,6 +3249,7 @@
 				84A0A8862A47D5C50038F329 /* XCRemoteSwiftPackageReference "QEMUKit" */,
 				CE9B15342B11A491003A32DD /* XCRemoteSwiftPackageReference "SwiftConnect" */,
 				CEDD11BF2B7C74D7004DDAC6 /* XCRemoteSwiftPackageReference "SwiftPortmap" */,
+				CE89CB0C2B8B1B49006B2CC2 /* XCRemoteSwiftPackageReference "VisionKeyboardKit" */,
 			);
 			productRefGroup = CE550BCA225947990063E575 /* Products */;
 			projectDirPath = "";
@ -5090,6 +5100,14 @@
 				minimumVersion = 1.5.3;
 			};
 		};
+		CE89CB0C2B8B1B49006B2CC2 /* XCRemoteSwiftPackageReference "VisionKeyboardKit" */ = {
+			isa = XCRemoteSwiftPackageReference;
+			repositoryURL = "https://github.com/utmapp/VisionKeyboardKit.git";
+			requirement = {
+				branch = main;
+				kind = branch;
+			};
+		};
 		CE93759724BB821F0074066F /* XCRemoteSwiftPackageReference "IQKeyboardManager" */ = {
 			isa = XCRemoteSwiftPackageReference;
 			repositoryURL = "https://github.com/hackiftekhar/IQKeyboardManager.git";
@ -5295,6 +5313,21 @@
 			package = CE020BA524AEDEF000B44AB6 /* XCRemoteSwiftPackageReference "swift-log" */;
 			productName = Logging;
 		};
+		CE89CB0D2B8B1B5A006B2CC2 /* VisionKeyboardKit */ = {
+			isa = XCSwiftPackageProductDependency;
+			package = CE89CB0C2B8B1B49006B2CC2 /* XCRemoteSwiftPackageReference "VisionKeyboardKit" */;
+			productName = VisionKeyboardKit;
+		};
+		CE89CB0F2B8B1B6A006B2CC2 /* VisionKeyboardKit */ = {
+			isa = XCSwiftPackageProductDependency;
+			package = CE89CB0C2B8B1B49006B2CC2 /* XCRemoteSwiftPackageReference "VisionKeyboardKit" */;
+			productName = VisionKeyboardKit;
+		};
+		CE89CB112B8B1B7A006B2CC2 /* VisionKeyboardKit */ = {
+			isa = XCSwiftPackageProductDependency;
+			package = CE89CB0C2B8B1B49006B2CC2 /* XCRemoteSwiftPackageReference "VisionKeyboardKit" */;
+			productName = VisionKeyboardKit;
+		};
 		CE93759824BB821F0074066F /* IQKeyboardManagerSwift */ = {
 			isa = XCSwiftPackageProductDependency;
 			package = CE93759724BB821F0074066F /* XCRemoteSwiftPackageReference "IQKeyboardManager" */;
--- a/UTM.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved
+++ b/UTM.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved
@ -108,6 +108,15 @@
        "version" : "1.0.3"
      }
    },
+    {
+      "identity" : "visionkeyboardkit",
+      "kind" : "remoteSourceControl",
+      "location" : "https://github.com/utmapp/VisionKeyboardKit.git",
+      "state" : {
+        "branch" : "main",
+        "revision" : "0804e4d64267acc8d08fb23160f5b6ac6134414f"
+      }
+    },
    {
      "identity" : "zipfoundation",
      "kind" : "remoteSourceControl",