From 74a49c1aa8f148b6b14e35574e3412906402c309 Mon Sep 17 00:00:00 2001 From: Max Desiatov Date: Mon, 17 Jan 2022 09:04:44 +0000 Subject: [PATCH] Use `async`/`await` and actors instead of Combine (#283) This makes our codebase smaller by ~80 lines and arguably more readable. Also removes OpenCombine dependency. * Use `async`/`await` and actors instead of Combine * Remove OpenCombine dependency * Fix progress animation not updated * Stop building with Swift 5.4, clean up terminal output * Update requirements in `README.md` * Add `description` to `InvalidResponseCode` * Fix Linux build error * Rename `main.swift` to `Main.swift` * Work around IR/TDB warnings * Pass IR/TDB arguments correctly to `swift build` * Support `async` process runner in `carton-release` * Use Xcode 13.2.1 on macOS * Pass TDB/IR flags to `swift test` as well * Make `Install` command async * Add doc comments, handle subsequent rebuilds --- .github/workflows/swift.yml | 11 +- Package.resolved | 9 - Package.swift | 14 +- README.md | 9 +- Sources/Carton/{main.swift => Main.swift} | 6 +- Sources/CartonCLI/Commands/Bundle.swift | 12 +- Sources/CartonCLI/Commands/Dev.swift | 25 ++- Sources/CartonCLI/Commands/Init.swift | 6 +- Sources/CartonCLI/Commands/Package.swift | 8 +- Sources/CartonCLI/Commands/SDK/Install.swift | 6 +- Sources/CartonCLI/Commands/Test.swift | 26 ++- Sources/CartonHelpers/Async.swift | 59 +++++++ Sources/CartonHelpers/AsyncFileDownload.swift | 65 ++++++++ Sources/CartonHelpers/Process.swift | 141 +++++++++++++++- Sources/CartonHelpers/ProcessRunner.swift | 157 ------------------ Sources/CartonKit/Combine/Watcher.swift | 38 ----- Sources/CartonKit/Model/Template.swift | 16 +- Sources/CartonKit/Server/Application.swift | 8 +- Sources/CartonKit/Server/Server.swift | 151 +++++++++++------ Sources/SwiftToolchain/Builder.swift | 99 ++++------- .../SwiftToolchain/ProgressAnimation.swift | 47 ------ Sources/SwiftToolchain/Toolchain.swift | 39 ++--- .../ToolchainInstallation.swift | 100 ++++------- .../SwiftToolchain/ToolchainManagement.swift | 4 +- Sources/carton-release/HashArchive.swift | 15 +- .../carton-release/{main.swift => Main.swift} | 6 +- Tests/Fixtures/Milk/Package.resolved | 12 +- 27 files changed, 525 insertions(+), 564 deletions(-) rename Sources/Carton/{main.swift => Main.swift} (88%) create mode 100644 Sources/CartonHelpers/Async.swift create mode 100644 Sources/CartonHelpers/AsyncFileDownload.swift delete mode 100644 Sources/CartonHelpers/ProcessRunner.swift delete mode 100644 Sources/CartonKit/Combine/Watcher.swift delete mode 100644 Sources/SwiftToolchain/ProgressAnimation.swift rename Sources/carton-release/{main.swift => Main.swift} (90%) diff --git a/.github/workflows/swift.yml b/.github/workflows/swift.yml index 38b7802..89603ba 100644 --- a/.github/workflows/swift.yml +++ b/.github/workflows/swift.yml @@ -11,12 +11,9 @@ jobs: strategy: matrix: include: - - os: macos-11 - swift_version: 5.4 - xcode: /Applications/Xcode_12.5.1.app/Contents/Developer - os: macos-11 swift_version: 5.5 - xcode: /Applications/Xcode_13.1.app/Contents/Developer + xcode: /Applications/Xcode_13.2.1.app/Contents/Developer - os: ubuntu-18.04 swift_version: 5.5 - os: ubuntu-20.04 @@ -45,7 +42,9 @@ jobs: curl https://get.wasmer.io -sSfL | sh - name: Build the project - run: swift build + run: | + swift -v + swift build -Xswiftc -Xfrontend -Xswiftc -validate-tbd-against-ir=none - name: Build and install JavaScript and sanitizer resources run: | @@ -61,7 +60,7 @@ jobs: if [ -e /home/runner/.wasmer/wasmer.sh ]; then source /home/runner/.wasmer/wasmer.sh fi - swift test + swift test -Xswiftc -Xfrontend -Xswiftc -validate-tbd-against-ir=none env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/Package.resolved b/Package.resolved index b01293e..fb3607e 100644 --- a/Package.resolved +++ b/Package.resolved @@ -37,15 +37,6 @@ "version": "4.5.1" } }, - { - "package": "OpenCombine", - "repositoryURL": "https://github.com/OpenCombine/OpenCombine.git", - "state": { - "branch": null, - "revision": "28993ae57de5a4ea7e164787636cafad442d568c", - "version": "0.12.0" - } - }, { "package": "routing-kit", "repositoryURL": "https://github.com/vapor/routing-kit.git", diff --git a/Package.swift b/Package.swift index 59a4830..376adec 100644 --- a/Package.swift +++ b/Package.swift @@ -1,14 +1,8 @@ -// swift-tools-version:5.4 +// swift-tools-version:5.5 // The swift-tools-version declares the minimum version of Swift required to build this package. import PackageDescription -let openCombineProduct = Target.Dependency.product( - name: "OpenCombine", - package: "OpenCombine", - condition: .when(platforms: [.linux]) -) - let package = Package( name: "carton", platforms: [.macOS(.v10_15)], @@ -39,7 +33,6 @@ let package = Package( url: "https://github.com/apple/swift-tools-support-core.git", .branch("release/5.5") ), - .package(url: "https://github.com/OpenCombine/OpenCombine.git", from: "0.12.0"), .package(url: "https://github.com/vapor/vapor.git", from: "4.53.0"), .package(url: "https://github.com/apple/swift-crypto.git", from: "1.1.0"), .package(url: "https://github.com/JohnSundell/Splash.git", from: "0.16.0"), @@ -65,12 +58,10 @@ let package = Package( .target( name: "CartonKit", dependencies: [ - .product(name: "ArgumentParser", package: "swift-argument-parser"), .product(name: "AsyncHTTPClient", package: "async-http-client"), .product(name: "Crypto", package: "swift-crypto"), .product(name: "Vapor", package: "vapor"), "CartonHelpers", - openCombineProduct, "SwiftToolchain", ] ), @@ -81,7 +72,6 @@ let package = Package( .product(name: "NIOFoundationCompat", package: "swift-nio"), .product(name: "SwiftPMDataModel-auto", package: "SwiftPM"), "CartonHelpers", - openCombineProduct, "WasmTransformer", ] ), @@ -89,7 +79,7 @@ let package = Package( name: "CartonHelpers", dependencies: [ .product(name: "AsyncHTTPClient", package: "async-http-client"), - openCombineProduct, + .product(name: "ArgumentParser", package: "swift-argument-parser"), "Splash", "WasmTransformer", ] diff --git a/README.md b/README.md index a6ec447..0eefd47 100644 --- a/README.md +++ b/README.md @@ -24,8 +24,8 @@ development workflow such as toolchain and SDK installations. ### Requirements -- macOS 11 and Xcode 12.5.1 or later. macOS 10.15 may work, but is untested. -- [Swift 5.4 or later](https://swift.org/download/) and Ubuntu 18.04 or 20.04 for Linux users. +- macOS 11 and Xcode 13.2.1 or later. macOS 10.15 may work, but is untested. +- [Swift 5.5 or later](https://swift.org/download/) and Ubuntu 18.04 or 20.04 for Linux users. ### Installation @@ -112,9 +112,8 @@ is that your `Package.swift` contains at least a single executable product, whic for WebAssembly and served when you start `carton dev` in the directory where `Package.swift` is located. `carton` is built with [Vapor](https://vapor.codes/), [SwiftNIO](https://github.com/apple/swift-nio), -[swift-tools-support-core](https://github.com/apple/swift-tools-support-core), and -[OpenCombine](https://github.com/OpenCombine/OpenCombine), and supports both macOS and Linux. (Many -thanks to everyone supporting and maintaining those projects!) +[swift-tools-support-core](https://github.com/apple/swift-tools-support-core), and supports both +macOS and Linux. (Many thanks to everyone supporting and maintaining those projects!) ### Running `carton dev` with the `release` configuration diff --git a/Sources/Carton/main.swift b/Sources/Carton/Main.swift similarity index 88% rename from Sources/Carton/main.swift rename to Sources/Carton/Main.swift index ca19325..d13521a 100644 --- a/Sources/Carton/main.swift +++ b/Sources/Carton/Main.swift @@ -13,5 +13,9 @@ // limitations under the License. import CartonCLI +import CartonHelpers -Carton.main() +@main +struct Main: AsyncMain { + typealias Command = Carton +} diff --git a/Sources/CartonCLI/Commands/Bundle.swift b/Sources/CartonCLI/Commands/Bundle.swift index 1cd4ee4..f84ef0d 100644 --- a/Sources/CartonCLI/Commands/Bundle.swift +++ b/Sources/CartonCLI/Commands/Bundle.swift @@ -26,7 +26,7 @@ private let dependency = Entrypoint( sha256: bundleEntrypointSHA256 ) -struct Bundle: ParsableCommand { +struct Bundle: AsyncParsableCommand { @Option(help: "Specify name of an executable product to produce the bundle for.") var product: String? @@ -50,15 +50,15 @@ struct Bundle: ParsableCommand { ) } - func run() throws { + func run() async throws { let terminal = InteractiveWriter.stdout try dependency.check(on: localFileSystem, terminal) - let toolchain = try Toolchain(localFileSystem, terminal) + let toolchain = try await Toolchain(localFileSystem, terminal) let flavor = buildFlavor() - let build = try toolchain.buildCurrentProject( + let build = try await toolchain.buildCurrentProject( product: product, flavor: flavor ) @@ -80,10 +80,10 @@ struct Bundle: ParsableCommand { try localFileSystem.removeFileTree(bundleDirectory) try localFileSystem.createDirectory(bundleDirectory) let optimizedPath = AbsolutePath(bundleDirectory, "main.wasm") - try ProcessRunner( + try await Process.run( ["wasm-opt", "-Os", build.mainWasmPath.pathString, "-o", optimizedPath.pathString], terminal - ).waitUntilFinished() + ) try terminal.logLookup( "After stripping debug info the main binary size is ", localFileSystem.humanReadableFileSize(optimizedPath), diff --git a/Sources/CartonCLI/Commands/Dev.swift b/Sources/CartonCLI/Commands/Dev.swift index e701ff1..e442dc9 100644 --- a/Sources/CartonCLI/Commands/Dev.swift +++ b/Sources/CartonCLI/Commands/Dev.swift @@ -14,17 +14,12 @@ import ArgumentParser import CartonHelpers -import Foundation -#if canImport(Combine) -import Combine -#else -import OpenCombine -#endif import CartonKit +import Foundation import SwiftToolchain import TSCBasic -struct Dev: ParsableCommand { +struct Dev: AsyncParsableCommand { static let entrypoint = Entrypoint(fileName: "dev.js", sha256: devEntrypointSHA256) @Option(help: "Specify name of an executable product in development.") @@ -71,12 +66,12 @@ struct Dev: ParsableCommand { ) } - func run() throws { + func run() async throws { let terminal = InteractiveWriter.stdout try Self.entrypoint.check(on: localFileSystem, terminal) - let toolchain = try Toolchain(localFileSystem, terminal) + let toolchain = try await Toolchain(localFileSystem, terminal) if !verbose { terminal.clearWindow() @@ -97,7 +92,7 @@ struct Dev: ParsableCommand { } let flavor = buildFlavor() - let build = try toolchain.buildCurrentProject( + let build = try await toolchain.buildCurrentProject( product: product, flavor: flavor ) @@ -113,7 +108,7 @@ struct Dev: ParsableCommand { let sources = try paths.flatMap { try localFileSystem.traverseRecursively($0) } - try Server( + try await Server( .init( builder: Builder( arguments: build.arguments, @@ -125,16 +120,16 @@ struct Dev: ParsableCommand { ), mainWasmPath: build.mainWasmPath, verbose: verbose, - skipAutoOpen: skipAutoOpen, + shouldSkipAutoOpen: skipAutoOpen, port: port, host: host, customIndexContent: HTML.readCustomIndexPage(at: customIndexPage, on: localFileSystem), // swiftlint:disable:next force_try manifest: try! toolchain.manifest.get(), product: build.product, - entrypoint: Self.entrypoint - ), - terminal + entrypoint: Self.entrypoint, + terminal: terminal + ) ).run() } } diff --git a/Sources/CartonCLI/Commands/Init.swift b/Sources/CartonCLI/Commands/Init.swift index 0ff1b1f..cdfcda4 100644 --- a/Sources/CartonCLI/Commands/Init.swift +++ b/Sources/CartonCLI/Commands/Init.swift @@ -19,7 +19,7 @@ import Foundation import SwiftToolchain import TSCBasic -struct Init: ParsableCommand { +struct Init: AsyncParsableCommand { static let configuration = CommandConfiguration( abstract: "Create a Swift package for a new SwiftWasm project.", subcommands: [ListTemplates.self] @@ -33,7 +33,7 @@ struct Init: ParsableCommand { @Option(name: .long, help: "The name of the project") var name: String? - func run() throws { + func run() async throws { let terminal = InteractiveWriter.stdout guard let name = name ?? localFileSystem.currentWorkingDirectory?.basename else { @@ -58,7 +58,7 @@ struct Init: ParsableCommand { return } try localFileSystem.createDirectory(packagePath) - try template.template.create( + try await template.template.create( on: localFileSystem, project: .init(name: name, path: packagePath, inPlace: self.name == nil), terminal diff --git a/Sources/CartonCLI/Commands/Package.swift b/Sources/CartonCLI/Commands/Package.swift index 691a92d..7f9f06f 100644 --- a/Sources/CartonCLI/Commands/Package.swift +++ b/Sources/CartonCLI/Commands/Package.swift @@ -19,7 +19,7 @@ import SwiftToolchain import TSCBasic /// Proxy swift-package command to locally pinned toolchain version. -struct Package: ParsableCommand { +struct Package: AsyncParsableCommand { static let configuration = CommandConfiguration(abstract: """ Perform operations on Swift packages. """) @@ -27,10 +27,10 @@ struct Package: ParsableCommand { @Argument(wrappedValue: [], parsing: .remaining) var arguments: [String] - func run() throws { + func run() async throws { let terminal = InteractiveWriter.stdout - let toolchain = try Toolchain(localFileSystem, terminal) - try toolchain.runPackage(arguments) + let toolchain = try await Toolchain(localFileSystem, terminal) + try await toolchain.runPackage(arguments) } } diff --git a/Sources/CartonCLI/Commands/SDK/Install.swift b/Sources/CartonCLI/Commands/SDK/Install.swift index 64bd0c3..6f5cfb5 100644 --- a/Sources/CartonCLI/Commands/SDK/Install.swift +++ b/Sources/CartonCLI/Commands/SDK/Install.swift @@ -17,17 +17,17 @@ import CartonHelpers import SwiftToolchain import TSCBasic -struct Install: ParsableCommand { +struct Install: AsyncParsableCommand { static let configuration = CommandConfiguration( abstract: "Install new Swift toolchain/SDK." ) @Argument() var version: String? - func run() throws { + func run() async throws { let terminal = InteractiveWriter.stdout - _ = try Toolchain(for: version, localFileSystem, terminal) + _ = try await Toolchain(for: version, localFileSystem, terminal) terminal.write("\nSDK successfully installed!\n", inColor: .green) } } diff --git a/Sources/CartonCLI/Commands/Test.swift b/Sources/CartonCLI/Commands/Test.swift index 7f598e7..51ffbff 100644 --- a/Sources/CartonCLI/Commands/Test.swift +++ b/Sources/CartonCLI/Commands/Test.swift @@ -13,11 +13,6 @@ // limitations under the License. import ArgumentParser -#if canImport(Combine) -import Combine -#else -import OpenCombine -#endif import CartonHelpers import CartonKit import SwiftToolchain @@ -39,7 +34,7 @@ private enum Environment: String, CaseIterable, ExpressibleByArgument { extension SanitizeVariant: ExpressibleByArgument {} -struct Test: ParsableCommand { +struct Test: AsyncParsableCommand { static let entrypoint = Entrypoint(fileName: "test.js", sha256: testEntrypointSHA256) static let configuration = CommandConfiguration(abstract: "Run the tests in a WASI environment.") @@ -78,13 +73,13 @@ struct Test: ParsableCommand { ) } - func run() throws { + func run() async throws { let terminal = InteractiveWriter.stdout try Self.entrypoint.check(on: localFileSystem, terminal) - let toolchain = try Toolchain(localFileSystem, terminal) + let toolchain = try await Toolchain(localFileSystem, terminal) let flavor = buildFlavor() - let testBundlePath = try toolchain.buildTestBundle(flavor: flavor) + let testBundlePath = try await toolchain.buildTestBundle(flavor: flavor) if environment == .wasmer { terminal.write("\nRunning the test bundle with wasmer:\n", inColor: .yellow) @@ -95,25 +90,24 @@ struct Test: ParsableCommand { wasmerArguments.append("--") wasmerArguments.append(contentsOf: testCases) } - let runner = ProcessRunner(wasmerArguments, parser: TestsParser(), terminal) - try runner.waitUntilFinished() + try await Process.run(wasmerArguments, parser: TestsParser(), terminal) } else { - try Server( + try await Server( .init( builder: nil, mainWasmPath: testBundlePath, verbose: true, - skipAutoOpen: false, + shouldSkipAutoOpen: false, port: port, host: host, customIndexContent: nil, // swiftlint:disable:next force_try manifest: try! toolchain.manifest.get(), product: nil, - entrypoint: Self.entrypoint - ), - terminal + entrypoint: Self.entrypoint, + terminal: terminal + ) ).run() } } diff --git a/Sources/CartonHelpers/Async.swift b/Sources/CartonHelpers/Async.swift new file mode 100644 index 0000000..6f46382 --- /dev/null +++ b/Sources/CartonHelpers/Async.swift @@ -0,0 +1,59 @@ +// Copyright 2021 Carton contributors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import ArgumentParser + +public extension Sequence { + func asyncMap( + _ transform: (Element) async throws -> T + ) async rethrows -> [T] { + var values = [T]() + + for element in self { + try await values.append(transform(element)) + } + + return values + } +} + +/// A type that can be executed as part of a nested tree of commands. +public protocol AsyncParsableCommand: ParsableCommand { + mutating func run() async throws +} + +public extension AsyncParsableCommand { + mutating func run() throws { + throw CleanExit.helpRequest(self) + } +} + +public protocol AsyncMain { + associatedtype Command: ParsableCommand +} + +public extension AsyncMain { + static func main() async { + do { + var command = try Command.parseAsRoot() + if var command = command as? AsyncParsableCommand { + try await command.run() + } else { + try command.run() + } + } catch { + Command.exit(withError: error) + } + } +} diff --git a/Sources/CartonHelpers/AsyncFileDownload.swift b/Sources/CartonHelpers/AsyncFileDownload.swift new file mode 100644 index 0000000..4696e35 --- /dev/null +++ b/Sources/CartonHelpers/AsyncFileDownload.swift @@ -0,0 +1,65 @@ +// Copyright 2022 Carton contributors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import AsyncHTTPClient +import Foundation + +public struct InvalidResponseCode: Error { + let code: UInt + + var description: String { + """ + While attempting to download an archive, the server returned an invalid response code \(code) + """ + } +} + +public final class AsyncFileDownload { + public let progressStream: AsyncThrowingStream + + public init(path: String, _ url: URL, _ client: HTTPClient, onTotalBytes: @escaping (Int) -> ()) { + progressStream = .init { continuation in + do { + let request = try HTTPClient.Request.get(url: url) + + let delegate = try FileDownloadDelegate( + path: path, + reportHead: { + guard $0.status == .ok, + let totalBytes = $0.headers.first(name: "Content-Length").flatMap(Int.init) + else { + continuation + .finish(throwing: InvalidResponseCode(code: $0.status.code)) + return + } + onTotalBytes(totalBytes) + }, + reportProgress: { + continuation.yield($0) + } + ) + + Task { + _ = try await client.execute(request: request, delegate: delegate) + .futureResult + .get() + + continuation.finish() + } + } catch { + continuation.finish(throwing: error) + } + } + } +} diff --git a/Sources/CartonHelpers/Process.swift b/Sources/CartonHelpers/Process.swift index dc06ce5..41c845e 100644 --- a/Sources/CartonHelpers/Process.swift +++ b/Sources/CartonHelpers/Process.swift @@ -12,26 +12,157 @@ // See the License for the specific language governing permissions and // limitations under the License. +import Dispatch import Foundation import TSCBasic public func processDataOutput(_ arguments: [String]) throws -> [UInt8] { - let process = Process(arguments: arguments, startNewProcessGroup: false) + let process = TSCBasic.Process(arguments: arguments, startNewProcessGroup: false) try process.launch() let result = try process.waitUntilExit() guard case .terminated(code: EXIT_SUCCESS) = result.exitStatus else { - var description = "Process failed with non-zero exit status" + let stdout: String? if let output = try ByteString(result.output.get()).validDescription, !output.isEmpty { - description += " and following output:\n\(output)" + stdout = output + } else { + stdout = nil } + var stderr: String? if let output = try ByteString(result.stderrOutput.get()).validDescription { - description += " and following error output:\n\(output)" + stderr = output + } else { + stderr = nil } - throw ProcessRunnerError(description: description) + throw ProcessError(stderr: stderr, stdout: stdout) } return try result.output.get() } + +struct ProcessError: Error { + let stderr: String? + let stdout: String? +} + +extension ProcessError: CustomStringConvertible { + var description: String { + var result = "Process failed with non-zero exit status" + if let stdout = stdout { + result += " and following output:\n\(stdout)" + } + + if let stderr = stderr { + result += " and following error output:\n\(stderr)" + } + return result + } +} + +public extension TSCBasic.Process { + // swiftlint:disable:next function_body_length + static func run( + _ arguments: [String], + loadingMessage: String = "Running...", + parser: ProcessOutputParser? = nil, + _ terminal: InteractiveWriter + ) async throws { + terminal.clearLine() + terminal.write("\(loadingMessage)\n", inColor: .yellow) + + let processName = arguments[0].first == "/" ? + AbsolutePath(arguments[0]).basename : arguments[0] + + do { + try await withCheckedThrowingContinuation { (continuation: CheckedContinuation<(), Swift.Error>) in + DispatchQueue.global().async { + var stdoutBuffer = "" + + let stdout: TSCBasic.Process.OutputClosure = { + guard let string = String(data: Data($0), encoding: .utf8) else { return } + if parser != nil { + // Aggregate this for formatting later + stdoutBuffer += string + } else { + terminal.write(string) + } + } + + var stderrBuffer = [UInt8]() + + let stderr: TSCBasic.Process.OutputClosure = { + stderrBuffer.append(contentsOf: $0) + } + + let process = Process( + arguments: arguments, + outputRedirection: .stream(stdout: stdout, stderr: stderr), + verbose: true, + startNewProcessGroup: true + ) + + let result = Result { + try process.launch() + return try process.waitUntilExit() + } + + switch result.map(\.exitStatus) { + case .success(.terminated(code: EXIT_SUCCESS)): + terminal.write("\n") + if let parser = parser { + if parser.parsingConditions.contains(.success) { + parser.parse(stdoutBuffer, terminal) + } + } else { + terminal.write(stdoutBuffer) + } + terminal.write( + "\n`\(processName)` process finished successfully\n", + inColor: .green, + bold: false + ) + continuation.resume() + + case let .failure(error): + continuation.resume(throwing: error) + default: + continuation.resume( + throwing: ProcessError( + stderr: String(data: Data(stderrBuffer), encoding: .utf8) ?? "", + stdout: stdoutBuffer + ) + ) + } + } + } + } catch { + let errorString = String(describing: error) + if errorString.isEmpty { + terminal.clearLine() + terminal.write( + "\(processName) process failed.\n\n", + inColor: .red + ) + if let error = error as? ProcessError, let stdout = error.stdout { + if let parser = parser { + if parser.parsingConditions.contains(.failure) { + parser.parse(stdout, terminal) + } + } else { + terminal.write(stdout) + } + } + } else { + terminal.write( + "\nProcess failed and produced following output: \n", + inColor: .red + ) + print(error) + } + + throw error + } + } +} diff --git a/Sources/CartonHelpers/ProcessRunner.swift b/Sources/CartonHelpers/ProcessRunner.swift deleted file mode 100644 index 0d3ab2a..0000000 --- a/Sources/CartonHelpers/ProcessRunner.swift +++ /dev/null @@ -1,157 +0,0 @@ -// Copyright 2020 Carton contributors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -import Dispatch -import Foundation -#if canImport(Combine) -import Combine -#else -import OpenCombine -#endif -import TSCBasic - -public extension Subscribers.Completion { - var result: Result<(), Failure> { - switch self { - case let .failure(error): - return .failure(error) - case .finished: - return .success(()) - } - } -} - -struct ProcessRunnerError: Error, CustomStringConvertible { - let description: String -} - -public final class ProcessRunner { - public let publisher: AnyPublisher - - private var subscription: AnyCancellable? - - // swiftlint:disable:next function_body_length - public init( - _ arguments: [String], - loadingMessage: String = "Running...", - parser: ProcessOutputParser? = nil, - _ terminal: InteractiveWriter - ) { - let subject = PassthroughSubject() - var tmpOutput = "" - publisher = subject - .handleEvents( - receiveSubscription: { _ in - terminal.clearLine() - terminal.write("\(loadingMessage)\n", inColor: .yellow) - }, - receiveOutput: { - if parser != nil { - // Aggregate this for formatting later - tmpOutput += $0 - } else { - terminal.write($0) - } - }, receiveCompletion: { - switch $0 { - case .finished: - let processName = arguments[0].first == "/" ? - AbsolutePath(arguments[0]).basename : arguments[0] - terminal.write("\n") - if let parser = parser { - if parser.parsingConditions.contains(.success) { - parser.parse(tmpOutput, terminal) - } - } else { - terminal.write(tmpOutput) - } - terminal.write( - "\n`\(processName)` process finished successfully\n", - inColor: .green, - bold: false - ) - case let .failure(error): - let errorString = String(describing: error) - if errorString.isEmpty { - terminal.clearLine() - terminal.write( - "Compilation failed.\n\n", - inColor: .red - ) - if let parser = parser { - if parser.parsingConditions.contains(.failure) { - parser.parse(tmpOutput, terminal) - } - } else { - terminal.write(tmpOutput) - } - } else { - terminal.write( - "\nProcess failed and produced following output: \n", - inColor: .red - ) - print(error) - } - } - } - ) - .eraseToAnyPublisher() - - DispatchQueue.global().async { - let stdout: TSCBasic.Process.OutputClosure = { - guard let string = String(data: Data($0), encoding: .utf8) else { return } - subject.send(string) - } - - var stderrBuffer = [UInt8]() - - let stderr: TSCBasic.Process.OutputClosure = { - stderrBuffer.append(contentsOf: $0) - } - - let process = Process( - arguments: arguments, - outputRedirection: .stream(stdout: stdout, stderr: stderr), - verbose: true, - startNewProcessGroup: true - ) - - let result = Result { - try process.launch() - return try process.waitUntilExit() - } - - switch result.map(\.exitStatus) { - case .success(.terminated(code: EXIT_SUCCESS)): - subject.send(completion: .finished) - case let .failure(error): - subject.send(completion: .failure(error)) - default: - let errorDescription = String(data: Data(stderrBuffer), encoding: .utf8) ?? "" - return subject - .send(completion: .failure(ProcessRunnerError(description: errorDescription))) - } - } - } - - public func waitUntilFinished() throws { - try tsc_await { completion in - subscription = publisher - .sink( - receiveCompletion: { completion($0.result) }, - receiveValue: { _ in } - ) - } - } -} diff --git a/Sources/CartonKit/Combine/Watcher.swift b/Sources/CartonKit/Combine/Watcher.swift deleted file mode 100644 index 6f19d3d..0000000 --- a/Sources/CartonKit/Combine/Watcher.swift +++ /dev/null @@ -1,38 +0,0 @@ -// Copyright 2020 Carton contributors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#if canImport(Combine) -import Combine -#else -import OpenCombine -#endif -import TSCBasic -import TSCUtility - -final class Watcher { - private let subject = PassthroughSubject<[AbsolutePath], Never>() - private var fsWatch: FSWatch! - let publisher: AnyPublisher<[AbsolutePath], Never> - - init(_ paths: [AbsolutePath]) throws { - publisher = subject.eraseToAnyPublisher() - - guard !paths.isEmpty else { return } - - fsWatch = FSWatch(paths: paths, latency: 0.1) { [weak self] in - self?.subject.send($0) - } - try fsWatch.start() - } -} diff --git a/Sources/CartonKit/Model/Template.swift b/Sources/CartonKit/Model/Template.swift index 80fc2a9..fe8a2ca 100644 --- a/Sources/CartonKit/Model/Template.swift +++ b/Sources/CartonKit/Model/Template.swift @@ -34,7 +34,7 @@ public protocol Template { on fileSystem: FileSystem, project: Project, _ terminal: InteractiveWriter - ) throws + ) async throws } enum TemplateError: Error { @@ -78,9 +78,9 @@ extension Template { fileSystem: FileSystem, project: Project, _ terminal: InteractiveWriter - ) throws { - try Toolchain(fileSystem, terminal) - .packageInit(name: project.name, type: type, inPlace: project.inPlace) + ) async throws { + try await Toolchain(fileSystem, terminal) + .runPackageInit(name: project.name, type: type, inPlace: project.inPlace) } static func createManifest( @@ -135,9 +135,9 @@ extension Templates { on fileSystem: FileSystem, project: Project, _ terminal: InteractiveWriter - ) throws { + ) async throws { try fileSystem.changeCurrentWorkingDirectory(to: project.path) - try createPackage(type: .executable, fileSystem: fileSystem, project: project, terminal) + try await createPackage(type: .executable, fileSystem: fileSystem, project: project, terminal) try createManifest( fileSystem: fileSystem, project: project, @@ -165,9 +165,9 @@ extension Templates { on fileSystem: FileSystem, project: Project, _ terminal: InteractiveWriter - ) throws { + ) async throws { try fileSystem.changeCurrentWorkingDirectory(to: project.path) - try createPackage(type: .executable, + try await createPackage(type: .executable, fileSystem: fileSystem, project: project, terminal) diff --git a/Sources/CartonKit/Server/Application.swift b/Sources/CartonKit/Server/Application.swift index f3c2213..bf2dd98 100644 --- a/Sources/CartonKit/Server/Application.swift +++ b/Sources/CartonKit/Server/Application.swift @@ -27,8 +27,8 @@ extension Application { let manifest: Manifest let product: ProductDescription? let entrypoint: Entrypoint - let onWebSocketOpen: (WebSocket, DestinationEnvironment) -> () - let onWebSocketClose: (WebSocket) -> () + let onWebSocketOpen: (WebSocket, DestinationEnvironment) async -> () + let onWebSocketClose: (WebSocket) async -> () } func configure(_ configuration: Configuration) { @@ -53,8 +53,8 @@ extension Application { let environment = request.headers["User-Agent"].compactMap(DestinationEnvironment.init).first ?? .other - configuration.onWebSocketOpen(ws, environment) - ws.onClose.whenComplete { _ in configuration.onWebSocketClose(ws) } + Task { await configuration.onWebSocketOpen(ws, environment) } + ws.onClose.whenComplete { _ in Task { await configuration.onWebSocketClose(ws) } } } get("main.wasm") { diff --git a/Sources/CartonKit/Server/Server.swift b/Sources/CartonKit/Server/Server.swift index a76ede4..43d40d4 100644 --- a/Sources/CartonKit/Server/Server.swift +++ b/Sources/CartonKit/Server/Server.swift @@ -13,14 +13,10 @@ // limitations under the License. import CartonHelpers -#if canImport(Combine) -import Combine -#else -import OpenCombine -#endif import PackageModel import SwiftToolchain import TSCBasic +import TSCUtility import Vapor private enum Event { @@ -56,7 +52,7 @@ extension Event: Decodable { } } -/// This `Hashable` conformance is required to handle simulatenous connections with `Set` +/// This `Hashable` conformance is required to handle simultaneous connections with `Set` extension WebSocket: Hashable { public static func == (lhs: WebSocket, rhs: WebSocket) -> Bool { lhs === rhs @@ -67,71 +63,85 @@ extension WebSocket: Hashable { } } -public final class Server { +public actor Server { + /// Used for decoding `Event` values sent from the WebSocket client. private let decoder = JSONDecoder() + + /// A set of connected WebSocket clients currently connected to this server. private var connections = Set() - private var subscriptions = [AnyCancellable]() - private let watcher: Watcher? + + /// Filesystem watcher monitoring relevant source files for changes. + private var watcher: FSWatch? + + /// An instance of Vapor server application. private let app: Application + + /// Local URL of this server, `https://128.0.0.1:8080/` by default. private let localURL: String - private let skipAutoOpen: Bool + + /// Whether a browser tab should automatically open as soon as the server is ready. + private let shouldSkipAutoOpen: Bool + + /// Whether a build that could be triggered by this server is currently running. + private var isBuildCurrentlyRunning = false + + /// Whether a subsequent build is currently scheduled on top of a currently running build. + private var isSubsequentBuildScheduled = false public struct Configuration { let builder: Builder? let mainWasmPath: AbsolutePath let verbose: Bool - let skipAutoOpen: Bool + let shouldSkipAutoOpen: Bool let port: Int let host: String let customIndexContent: String? let manifest: Manifest let product: ProductDescription? let entrypoint: Entrypoint + let terminal: InteractiveWriter public init( builder: Builder?, mainWasmPath: AbsolutePath, verbose: Bool, - skipAutoOpen: Bool, + shouldSkipAutoOpen: Bool, port: Int, host: String, customIndexContent: String?, manifest: Manifest, product: ProductDescription?, - entrypoint: Entrypoint + entrypoint: Entrypoint, + terminal: InteractiveWriter ) { self.builder = builder self.mainWasmPath = mainWasmPath self.verbose = verbose - self.skipAutoOpen = skipAutoOpen + self.shouldSkipAutoOpen = shouldSkipAutoOpen self.port = port self.host = host self.customIndexContent = customIndexContent self.manifest = manifest self.product = product self.entrypoint = entrypoint + self.terminal = terminal } } public init( - _ configuration: Configuration, - _ terminal: InteractiveWriter - ) throws { - if let builder = configuration.builder { - watcher = try Watcher(builder.pathsToWatch) - } else { - watcher = nil - } - + _ configuration: Configuration + ) async throws { var env = Environment( name: configuration.verbose ? "development" : "production", arguments: ["vapor"] ) localURL = "http://\(configuration.host):\(configuration.port)/" - skipAutoOpen = configuration.skipAutoOpen + shouldSkipAutoOpen = configuration.shouldSkipAutoOpen try LoggingSystem.bootstrap(from: &env) app = Application(env) + watcher = nil + app.configure( .init( port: configuration.port, @@ -142,39 +152,76 @@ public final class Server { product: configuration.product, entrypoint: configuration.entrypoint, onWebSocketOpen: { [weak self] ws, environment in - if let handler = self?.createWSHandler( + if let handler = await self?.createWSHandler( configuration, in: environment, - terminal: terminal + terminal: configuration.terminal ) { ws.onText(handler) } - self?.connections.insert(ws) + + await self?.add(connection: ws) }, - onWebSocketClose: { [weak self] in self?.connections.remove($0) } + onWebSocketClose: { [weak self] in await self?.remove(connection: $0) } ) ) // Listen to Vapor App lifecycle events app.lifecycle.use(self) - guard let builder = configuration.builder, let watcher = watcher else { + guard let builder = configuration.builder else { return } - watcher.publisher - .flatMap(maxPublishers: .max(1)) { changes -> AnyPublisher in - if !configuration.verbose { - terminal.clearWindow() - } - terminal.write("\nThese paths have changed, rebuilding...\n", inColor: .yellow) - for change in changes.map(\.pathString) { - terminal.write("- \(change)\n", inColor: .cyan) - } + watcher = FSWatch(paths: builder.pathsToWatch, latency: 0.1) { [weak self] changes in + guard let self = self, !changes.isEmpty else { return } + Task { try await self.onChange(changes, configuration) } + } + try watcher?.start() + } - return self.run(builder, terminal) + private func onChange(_ changes: [AbsolutePath], _ configuration: Configuration) async throws { + guard !isBuildCurrentlyRunning else { + if !isSubsequentBuildScheduled { + isSubsequentBuildScheduled = true } - .sink(receiveValue: { _ in }) - .store(in: &subscriptions) + return + } + + if !configuration.verbose { + configuration.terminal.clearWindow() + } + configuration.terminal.write( + "\nThese paths have changed, rebuilding...\n", + inColor: .yellow + ) + for change in changes.map(\.pathString) { + configuration.terminal.write("- \(change)\n", inColor: .cyan) + } + + isBuildCurrentlyRunning = true + // `configuration.builder` is guaranteed to be non-nil here as its presence is checked in `init` + try await run(configuration.builder!, configuration.terminal) + + if isSubsequentBuildScheduled { + configuration.terminal.write( + "\nMore paths have changed during the build, rebuilding again...\n", + inColor: .yellow + ) + try await run(configuration.builder!, configuration.terminal) + } + + isSubsequentBuildScheduled = false + isBuildCurrentlyRunning = false + } + + private func add(pendingChanges: [AbsolutePath]) {} + + private func add(connection: WebSocket) { + connections.insert(connection) + } + + private func remove(connection: WebSocket) { + connections.remove(connection) } /// Blocking function that starts the HTTP server. @@ -189,18 +236,12 @@ public final class Server { private func run( _ builder: Builder, _ terminal: InteractiveWriter - ) -> AnyPublisher { - builder - .run() - .handleEvents(receiveCompletion: { [weak self] in - guard case .finished = $0, let self = self else { return } + ) async throws { + try await builder.run() - terminal.write("\nBuild completed successfully\n", inColor: .green, bold: false) - terminal.logLookup("The app is currently hosted at ", self.localURL) - self.connections.forEach { $0.send("reload") } - }) - .catch { _ in Empty().eraseToAnyPublisher() } - .eraseToAnyPublisher() + terminal.write("\nBuild completed successfully\n", inColor: .green, bold: false) + terminal.logLookup("The app is currently hosted at ", localURL) + connections.forEach { $0.send("reload") } } } @@ -248,15 +289,15 @@ extension Server { } extension Server: LifecycleHandler { - public func didBoot(_ application: Application) throws { - guard !skipAutoOpen else { return } + public nonisolated func didBoot(_ application: Application) throws { + guard !shouldSkipAutoOpen else { return } openInSystemBrowser(url: localURL) } /// Attempts to open the specified URL string in system browser on macOS and Linux. /// - Returns: true if launching command returns successfully. @discardableResult - private func openInSystemBrowser(url: String) -> Bool { + private nonisolated func openInSystemBrowser(url: String) -> Bool { #if os(macOS) let openCommand = "open" #elseif os(Linux) diff --git a/Sources/SwiftToolchain/Builder.swift b/Sources/SwiftToolchain/Builder.swift index 83b1c84..ff71cbc 100644 --- a/Sources/SwiftToolchain/Builder.swift +++ b/Sources/SwiftToolchain/Builder.swift @@ -13,11 +13,6 @@ // limitations under the License. import CartonHelpers -#if canImport(Combine) -import Combine -#else -import OpenCombine -#endif import Foundation import TSCBasic import WasmTransformer @@ -25,13 +20,10 @@ import WasmTransformer public final class Builder { public let mainWasmPath: AbsolutePath public let pathsToWatch: [AbsolutePath] - - private var currentProcess: ProcessRunner? private let arguments: [String] private let flavor: BuildFlavor private let terminal: InteractiveWriter private let fileSystem: FileSystem - private var subscription: AnyCancellable? public init( arguments: [String], @@ -49,70 +41,57 @@ public final class Builder { self.fileSystem = fileSystem } - private func processPublisher(builderArguments: [String]) -> AnyPublisher { + private func buildWithoutSanitizing(builderArguments: [String]) async throws { let buildStarted = Date() - let process = ProcessRunner( + try await Process.run( builderArguments, loadingMessage: "Compiling...", parser: nil, terminal ) - currentProcess = process - return process - .publisher - .handleEvents(receiveCompletion: { [weak self] in - guard case .finished = $0, let self = self else { return } + self.terminal.logLookup( + "`swift build` completed in ", + String(format: "%.2f seconds", abs(buildStarted.timeIntervalSinceNow)) + ) - self.terminal.logLookup( - "`swift build` completed in ", - String(format: "%.2f seconds", abs(buildStarted.timeIntervalSinceNow)) - ) + var transformers: [(inout InputByteStream, inout InMemoryOutputWriter) throws -> ()] = [] + if self.flavor.environment != .other { + transformers.append(I64ImportTransformer().transform) + } - var transformers: [(inout InputByteStream, inout InMemoryOutputWriter) throws -> ()] = [] - if self.flavor.environment != .other { - transformers.append(I64ImportTransformer().transform) - } + switch self.flavor.sanitize { + case .stackOverflow: + transformers.append(StackOverflowSanitizer().transform) + case .none: + break + } - switch self.flavor.sanitize { - case .stackOverflow: - transformers.append(StackOverflowSanitizer().transform) - case .none: - break - } + guard !transformers.isEmpty else { return } - guard !transformers.isEmpty else { return } + let binary = try self.fileSystem.readFileContents(self.mainWasmPath) - // FIXME: errors from these `try` expressions should be recoverable, not sure how to - // do that in `handleEvents`, and `flatMap` doesn't fit here as we need to track - // publisher completion. - // swiftlint:disable force_try - let binary = try! self.fileSystem.readFileContents(self.mainWasmPath) + let transformStarted = Date() + var inputBinary = binary.contents + for transformer in transformers { + var input = InputByteStream(bytes: inputBinary) + var writer = InMemoryOutputWriter(reservingCapacity: inputBinary.count) + try! transformer(&input, &writer) + inputBinary = writer.bytes() + } - let transformStarted = Date() - var inputBinary = binary.contents - for transformer in transformers { - var input = InputByteStream(bytes: inputBinary) - var writer = InMemoryOutputWriter(reservingCapacity: inputBinary.count) - try! transformer(&input, &writer) - inputBinary = writer.bytes() - } + self.terminal.logLookup( + "Binary transformation for Safari compatibility completed in ", + String(format: "%.2f seconds", abs(transformStarted.timeIntervalSinceNow)) + ) - self.terminal.logLookup( - "Binary transformation for Safari compatibility completed in ", - String(format: "%.2f seconds", abs(transformStarted.timeIntervalSinceNow)) - ) - - try! self.fileSystem.writeFileContents(self.mainWasmPath, bytes: .init(inputBinary)) - // swiftlint:enable force_try - }) - .eraseToAnyPublisher() + try self.fileSystem.writeFileContents(self.mainWasmPath, bytes: .init(inputBinary)) } - public func run() -> AnyPublisher { + public func run() async throws { switch flavor.sanitize { case .none: - return processPublisher(builderArguments: arguments) + return try await buildWithoutSanitizing(builderArguments: arguments) case .stackOverflow: let sanitizerFile = fileSystem.homeDirectory.appending(components: ".carton", "static", "so_sanitizer.wasm") @@ -123,17 +102,7 @@ public final class Builder { // stack-overflow-sanitizer depends on "--stack-first" "-Xlinker", "--stack-first", ]) - return processPublisher(builderArguments: modifiedArguments) - } - } - - public func runAndWaitUntilFinished() throws { - try tsc_await { completion in - subscription = run() - .sink( - receiveCompletion: { completion($0.result) }, - receiveValue: { _ in } - ) + return try await buildWithoutSanitizing(builderArguments: modifiedArguments) } } } diff --git a/Sources/SwiftToolchain/ProgressAnimation.swift b/Sources/SwiftToolchain/ProgressAnimation.swift deleted file mode 100644 index e280a6a..0000000 --- a/Sources/SwiftToolchain/ProgressAnimation.swift +++ /dev/null @@ -1,47 +0,0 @@ -// Copyright 2020 Carton contributors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#if canImport(Combine) -import Combine -#else -import OpenCombine -#endif -import TSCBasic -import TSCUtility - -struct Progress { - let step: Int - let total: Int - let text: String -} - -extension Publisher where Output == Progress { - func handle( - with progressAnimation: ProgressAnimationProtocol - ) -> Publishers.HandleEvents { - handleEvents( - receiveOutput: { - progressAnimation.update(step: $0.step, total: $0.total, text: $0.text) - }, - receiveCompletion: { - switch $0 { - case .finished: - progressAnimation.complete(success: true) - case .failure: - progressAnimation.complete(success: false) - } - } - ) - } -} diff --git a/Sources/SwiftToolchain/Toolchain.swift b/Sources/SwiftToolchain/Toolchain.swift index dacd720..941d8ee 100644 --- a/Sources/SwiftToolchain/Toolchain.swift +++ b/Sources/SwiftToolchain/Toolchain.swift @@ -23,7 +23,6 @@ public let compatibleJSKitVersion = Version(0, 11, 1) enum ToolchainError: Error, CustomStringConvertible { case directoryDoesNotExist(AbsolutePath) - case invalidResponseCode(UInt) case invalidInstallationArchive(AbsolutePath) case noExecutableProduct case failedToBuild(product: String) @@ -38,10 +37,6 @@ enum ToolchainError: Error, CustomStringConvertible { switch self { case let .directoryDoesNotExist(path): return "Directory at path \(path.pathString) does not exist and could not be created" - case let .invalidResponseCode(code): - return """ - While attempting to download an archive, the server returned an invalid response code \(code) - """ case let .invalidInstallationArchive(path): return "Invalid toolchain/SDK archive was installed at path \(path)" case .noExecutableProduct: @@ -105,9 +100,9 @@ public final class Toolchain { for versionSpec: String? = nil, _ fileSystem: FileSystem, _ terminal: InteractiveWriter - ) throws { + ) async throws { let toolchainSystem = ToolchainSystem(fileSystem: fileSystem) - let (swiftPath, version) = try toolchainSystem.inferSwiftPath(from: versionSpec, terminal) + let (swiftPath, version) = try await toolchainSystem.inferSwiftPath(from: versionSpec, terminal) self.swiftPath = swiftPath self.version = version self.fileSystem = fileSystem @@ -207,7 +202,7 @@ public final class Toolchain { public func buildCurrentProject( product: String?, flavor: BuildFlavor - ) throws -> BuildDescription { + ) async throws -> BuildDescription { guard let product = try inferDevProduct(hint: product) else { throw ToolchainError.noExecutableProduct } @@ -264,20 +259,19 @@ public final class Toolchain { builderArguments.append("--enable-test-discovery") } - // SwiftWasm 5.5 requires explicit linking arguments in certain configurations, + // SwiftWasm 5.5 requires explicit linking arguments in certain configurations, // see https://github.com/swiftwasm/swift/issues/3891 if version.starts(with: "wasm-5.5") { builderArguments.append(contentsOf: ["-Xlinker", "-licuuc", "-Xlinker", "-licui18n"]) } - try Builder( + try await Builder( arguments: builderArguments, mainWasmPath: mainWasmPath, flavor, fileSystem, terminal - ) - .runAndWaitUntilFinished() + ).run() guard fileSystem.exists(mainWasmPath) else { terminal.write( @@ -293,7 +287,7 @@ public final class Toolchain { /// Returns an absolute path to the resulting test bundle public func buildTestBundle( flavor: BuildFlavor - ) throws -> AbsolutePath { + ) async throws -> AbsolutePath { let manifest = try self.manifest.get() let binPath = try inferBinPath(isRelease: flavor.isRelease) let testProductName = "\(manifest.name)PackageTests" @@ -308,7 +302,7 @@ public final class Toolchain { var builderArguments = [ swiftPath.pathString, "build", "-c", flavor.isRelease ? "release" : "debug", "--product", testProductName, "--triple", "wasm32-unknown-wasi", - "-Xswiftc", "-color-diagnostics" + "-Xswiftc", "-color-diagnostics", ] // Versions later than 5.3.x have test discovery enabled by default and the explicit flag @@ -317,20 +311,19 @@ public final class Toolchain { builderArguments.append("--enable-test-discovery") } - // SwiftWasm 5.5 requires explicit linking arguments in certain configurations, + // SwiftWasm 5.5 requires explicit linking arguments in certain configurations, // see https://github.com/swiftwasm/swift/issues/3891 if version.starts(with: "wasm-5.5") { builderArguments.append(contentsOf: ["-Xlinker", "-licuuc", "-Xlinker", "-licui18n"]) } - try Builder( + try await Builder( arguments: builderArguments, mainWasmPath: testBundlePath, flavor, fileSystem, terminal - ) - .runAndWaitUntilFinished() + ).run() guard fileSystem.exists(testBundlePath) else { terminal.write( @@ -343,7 +336,7 @@ public final class Toolchain { return testBundlePath } - public func packageInit(name: String, type: PackageType, inPlace: Bool) throws { + public func runPackageInit(name: String, type: PackageType, inPlace: Bool) async throws { var initArgs = [ swiftPath.pathString, "package", "init", "--type", type.rawValue, @@ -351,13 +344,11 @@ public final class Toolchain { if !inPlace { initArgs.append(contentsOf: ["--name", name]) } - try ProcessRunner(initArgs, terminal) - .waitUntilFinished() + try await TSCBasic.Process.run(initArgs, terminal) } - public func runPackage(_ arguments: [String]) throws { + public func runPackage(_ arguments: [String]) async throws { let args = [swiftPath.pathString, "package"] + arguments - try ProcessRunner(args, terminal) - .waitUntilFinished() + try await TSCBasic.Process.run(args, terminal) } } diff --git a/Sources/SwiftToolchain/ToolchainInstallation.swift b/Sources/SwiftToolchain/ToolchainInstallation.swift index 9988e66..bd3ab60 100644 --- a/Sources/SwiftToolchain/ToolchainInstallation.swift +++ b/Sources/SwiftToolchain/ToolchainInstallation.swift @@ -15,16 +15,17 @@ import AsyncHTTPClient import CartonHelpers import Foundation -#if canImport(Combine) -import Combine -#else -import OpenCombine -#endif import TSCBasic import TSCUtility private let expectedArchiveSize = 891_856_371 +private extension FileDownloadDelegate.Progress { + var totalOrEstimatedBytes: Int { + totalBytes ?? expectedArchiveSize + } +} + extension ToolchainSystem { func installSDK( version: String, @@ -32,7 +33,7 @@ extension ToolchainSystem { to sdkPath: AbsolutePath, _ client: HTTPClient, _ terminal: InteractiveWriter - ) throws -> AbsolutePath { + ) async throws -> AbsolutePath { if !fileSystem.exists(sdkPath, followSymlink: true) { try fileSystem.createDirectory(sdkPath, recursive: true) } @@ -44,10 +45,6 @@ extension ToolchainSystem { let ext = url.pathExtension let archivePath = sdkPath.appending(component: "\(version).\(ext)") - let (delegate, subject) = try downloadDelegate(path: archivePath.pathString, terminal) - - var subscriptions = [AnyCancellable]() - let request = try HTTPClient.Request.get(url: url) // Clean up the downloaded file (especially important for failed downloads, otherwise running // `carton` again will fail trying to pick up the broken download). @@ -59,40 +56,41 @@ extension ToolchainSystem { } } - _ = try tsc_await { (completion: @escaping (Result<(), Error>) -> ()) in - client.execute(request: request, delegate: delegate).futureResult.whenComplete { - switch $0 { - case .success: - subject.send(completion: .finished) - case let .failure(error): - subject.send(completion: .failure(error)) + do { + let fileDownload = AsyncFileDownload( + path: archivePath.pathString, + url, + client, + onTotalBytes: { + terminal.write("Archive size is \($0 / 1_000_000) MB\n", inColor: .yellow) } - } + ) - subject - .removeDuplicates { - // only report values that differ in more than 1% - $1.step - $0.step < ($0.total / 100) + let animation = PercentProgressAnimation( + stream: stdoutStream, + header: "Downloading the archive" + ) + var previouslyReceived = 0 + for try await progress in fileDownload.progressStream { + guard progress.receivedBytes - previouslyReceived >= (progress.totalOrEstimatedBytes / 100) + else { + continue } - .handle( - with: PercentProgressAnimation(stream: stdoutStream, header: "Downloading the archive") + defer { previouslyReceived = progress.receivedBytes } + + animation.update( + step: progress.receivedBytes, + total: progress.totalOrEstimatedBytes, + text: "saving to \(archivePath.pathString)" ) - .sink( - receiveCompletion: { - switch $0 { - case .finished: - terminal.write("Download completed successfully\n", inColor: .green) - completion(.success(())) - case let .failure(error): - terminal.write("Download failed\n", inColor: .red) - completion(.failure(error)) - } - }, - receiveValue: { _ in } - ) - .store(in: &subscriptions) + } + } catch { + terminal.write("Download failed with error \(error)\n", inColor: .red) + throw error } + terminal.write("Download completed successfully\n", inColor: .green) + let installationPath: AbsolutePath let arguments: [String] @@ -118,30 +116,4 @@ extension ToolchainSystem { return installationPath } - - private func downloadDelegate( - path: String, - _ terminal: InteractiveWriter - ) throws -> (FileDownloadDelegate, PassthroughSubject) { - let subject = PassthroughSubject() - return try (FileDownloadDelegate( - path: path, - reportHead: { - guard $0.status == .ok, - let totalBytes = $0.headers.first(name: "Content-Length").flatMap(Int.init) - else { - subject.send(completion: .failure(ToolchainError.invalidResponseCode($0.status.code))) - return - } - terminal.write("Archive size is \(totalBytes / 1_000_000) MB\n", inColor: .yellow) - }, - reportProgress: { - subject.send(.init( - step: $0.receivedBytes, - total: $0.totalBytes ?? expectedArchiveSize, - text: "saving to \(path)" - )) - } - ), subject) - } } diff --git a/Sources/SwiftToolchain/ToolchainManagement.swift b/Sources/SwiftToolchain/ToolchainManagement.swift index 163ae7a..15aa99b 100644 --- a/Sources/SwiftToolchain/ToolchainManagement.swift +++ b/Sources/SwiftToolchain/ToolchainManagement.swift @@ -210,7 +210,7 @@ public class ToolchainSystem { func inferSwiftPath( from versionSpec: String? = nil, _ terminal: InteractiveWriter - ) throws -> (AbsolutePath, String) { + ) async throws -> (AbsolutePath, String) { let specURL = versionSpec.flatMap { (string: String) -> Foundation.URL? in guard let url = Foundation.URL(string: string), @@ -251,7 +251,7 @@ public class ToolchainSystem { inColor: .yellow ) terminal.logLookup("Swift toolchain/SDK download URL: ", downloadURL) - let installationPath = try installSDK( + let installationPath = try await installSDK( version: swiftVersion, from: downloadURL, to: cartonToolchainResolver.cartonSDKPath, diff --git a/Sources/carton-release/HashArchive.swift b/Sources/carton-release/HashArchive.swift index bd6d38e..dcd806c 100644 --- a/Sources/carton-release/HashArchive.swift +++ b/Sources/carton-release/HashArchive.swift @@ -17,15 +17,14 @@ import CartonHelpers import TSCBasic import WasmTransformer -struct HashArchive: ParsableCommand { - /** Converts a hexadecimal hash string to Swift code that represents a static +struct HashArchive: AsyncParsableCommand { + /** Converts a hexadecimal hash string to Swift code that represents an archive of static assets. */ private func arrayString(from hash: String) -> String { precondition(hash.count == 64) let commaSeparated = stride(from: 0, to: hash.count, by: 2) - .map { hash.dropLast(hash.count - $0 - 2).suffix(2) } - .map { "0x\($0)" } + .map { "0x\(hash.dropLast(hash.count - $0 - 2).suffix(2))" } .joined(separator: ", ") precondition(commaSeparated.count == 190) @@ -36,7 +35,7 @@ struct HashArchive: ParsableCommand { """ } - func run() throws { + func run() async throws { let terminal = InteractiveWriter.stdout let cwd = localFileSystem.currentWorkingDirectory! let staticPath = AbsolutePath(cwd, "static") @@ -46,8 +45,8 @@ struct HashArchive: ParsableCommand { ) try localFileSystem.createDirectory(dotFilesStaticPath, recursive: true) - let hashes = try ["dev", "bundle", "test"].map { entrypoint -> (String, String) in - try ProcessRunner(["npm", "run", entrypoint], terminal).waitUntilFinished() + let hashes = try await ["dev", "bundle", "test"].asyncMap { entrypoint -> (String, String) in + try await Process.run(["npm", "run", entrypoint], terminal) let entrypointPath = AbsolutePath(staticPath, "\(entrypoint).js") let dotFilesEntrypointPath = dotFilesStaticPath.appending(component: "\(entrypoint).js") try localFileSystem.removeFileTree(dotFilesEntrypointPath) @@ -68,7 +67,7 @@ struct HashArchive: ParsableCommand { .dropFirst() .map(\.pathString) - try ProcessRunner(["zip", "-j", "static.zip"] + archiveSources, terminal).waitUntilFinished() + try await Process.run(["zip", "-j", "static.zip"] + archiveSources, terminal) let archiveHash = try SHA256().hash( localFileSystem.readFileContents(AbsolutePath( diff --git a/Sources/carton-release/main.swift b/Sources/carton-release/Main.swift similarity index 90% rename from Sources/carton-release/main.swift rename to Sources/carton-release/Main.swift index 092c0b4..efda46a 100644 --- a/Sources/carton-release/main.swift +++ b/Sources/carton-release/Main.swift @@ -13,6 +13,7 @@ // limitations under the License. import ArgumentParser +import CartonHelpers struct CartonRelease: ParsableCommand { static let configuration = CommandConfiguration( @@ -21,4 +22,7 @@ struct CartonRelease: ParsableCommand { ) } -CartonRelease.main() +@main +struct Main: AsyncMain { + typealias Command = CartonRelease +} diff --git a/Tests/Fixtures/Milk/Package.resolved b/Tests/Fixtures/Milk/Package.resolved index a09886e..df012f2 100644 --- a/Tests/Fixtures/Milk/Package.resolved +++ b/Tests/Fixtures/Milk/Package.resolved @@ -6,8 +6,8 @@ "repositoryURL": "https://github.com/swiftwasm/JavaScriptKit.git", "state": { "branch": null, - "revision": "b19e7c8b10a2750ed47753e31ed13613171f3294", - "version": "0.10.1" + "revision": "309e63c03d8116210ad0437f5d1f09a26d4de48b", + "version": "0.11.1" } }, { @@ -24,8 +24,8 @@ "repositoryURL": "https://github.com/swiftwasm/OpenCombineJS.git", "state": { "branch": null, - "revision": "eaf324ce78710f53b52fb82e9a8de4693633e33a", - "version": "0.1.1" + "revision": "f1f1799ddbb9876a0ef8c5700a3b78d352d0b969", + "version": "0.1.2" } }, { @@ -51,8 +51,8 @@ "repositoryURL": "https://github.com/TokamakUI/Tokamak", "state": { "branch": null, - "revision": "c2ed28ca40445b1b63bfdfe45de2507e5eb24a21", - "version": "0.8.0" + "revision": "32616fe9d4d8dcdd6a48519e2ce8d7b0ba744434", + "version": "0.9.0" } } ]