From e37a6cd2df95482b0c730a225c276b358863eced Mon Sep 17 00:00:00 2001 From: "Chris (SPG) McGee" Date: Thu, 26 Dec 2024 14:02:32 -0500 Subject: [PATCH 1/9] Adopt the static linux sdk --- Package.resolved | 11 +---------- Package.swift | 4 ++-- Sources/Swiftly/Install.swift | 16 ++++++++-------- Sources/Swiftly/SelfUpdate.swift | 16 ++++++++-------- .../BuildSwiftlyRelease.swift | 13 +++++++++++-- 5 files changed, 30 insertions(+), 30 deletions(-) diff --git a/Package.resolved b/Package.resolved index a0e1f8e1..11a9893a 100644 --- a/Package.resolved +++ b/Package.resolved @@ -1,5 +1,5 @@ { - "originHash" : "9b0d92b6fbb59080a05ce00f87dc9c6277b32d78e56905abba4c40947edf6d7d", + "originHash" : "cc03f1f46471dab3d83dd2c3efecbcbb0807f138854b4d0f03bcee316fe44231", "pins" : [ { "identity" : "async-http-client", @@ -145,15 +145,6 @@ "version" : "1.2.1" } }, - { - "identity" : "swift-tools-support-core", - "kind" : "remoteSourceControl", - "location" : "https://github.com/apple/swift-tools-support-core.git", - "state" : { - "revision" : "5b130e04cc939373c4713b91704b0c47ceb36170", - "version" : "0.7.1" - } - }, { "identity" : "swiftformat", "kind" : "remoteSourceControl", diff --git a/Package.swift b/Package.swift index 795d4c7a..3eb0d091 100644 --- a/Package.swift +++ b/Package.swift @@ -17,7 +17,7 @@ let package = Package( .package(url: "https://github.com/apple/swift-argument-parser", from: "1.3.0"), .package(url: "https://github.com/swift-server/async-http-client", from: "1.21.2"), .package(url: "https://github.com/apple/swift-nio.git", from: "2.64.0"), - .package(url: "https://github.com/apple/swift-tools-support-core.git", from: "0.7.1"), + //.package(url: "https://github.com/apple/swift-tools-support-core.git", from: "0.7.1"), .package(url: "https://github.com/apple/swift-docc-plugin", from: "1.3.0"), // This dependency provides the correct version of the formatter so that you can run `swift run swiftformat Package.swift Plugins/ Sources/ Tests/` .package(url: "https://github.com/nicklockwood/SwiftFormat", exact: "0.49.18"), @@ -30,7 +30,7 @@ let package = Package( .target(name: "SwiftlyCore"), .target(name: "LinuxPlatform", condition: .when(platforms: [.linux])), .target(name: "MacOSPlatform", condition: .when(platforms: [.macOS])), - .product(name: "SwiftToolsSupport-auto", package: "swift-tools-support-core"), + //.product(name: "SwiftToolsSupport-auto", package: "swift-tools-support-core"), ] ), .target( diff --git a/Sources/Swiftly/Install.swift b/Sources/Swiftly/Install.swift index 7de678fa..6449ba46 100644 --- a/Sources/Swiftly/Install.swift +++ b/Sources/Swiftly/Install.swift @@ -1,8 +1,8 @@ import _StringProcessing import ArgumentParser import Foundation -import TSCBasic -import TSCUtility +//import TSCBasic +//import TSCUtility import SwiftlyCore @@ -192,10 +192,10 @@ struct Install: SwiftlyCommand { throw Error(message: "Invalid toolchain URL: \(url)") } - let animation = PercentProgressAnimation( + /*let animation = PercentProgressAnimation( stream: stdoutStream, header: "Downloading \(version)" - ) + )*/ var lastUpdate = Date() @@ -215,20 +215,20 @@ struct Install: SwiftlyCommand { lastUpdate = Date() - animation.update( + /*animation.update( step: progress.receivedBytes, total: progress.totalBytes!, text: "Downloaded \(String(format: "%.1f", downloadedMiB)) MiB of \(String(format: "%.1f", totalMiB)) MiB" - ) + )*/ } ) } catch let notFound as SwiftlyHTTPClient.DownloadNotFoundError { throw Error(message: "\(version) does not exist at URL \(notFound.url), exiting") } catch { - animation.complete(success: false) + //animation.complete(success: false) throw error } - animation.complete(success: true) + //animation.complete(success: true) if verifySignature { try await Swiftly.currentPlatform.verifySignature( diff --git a/Sources/Swiftly/SelfUpdate.swift b/Sources/Swiftly/SelfUpdate.swift index e41dea60..db703515 100644 --- a/Sources/Swiftly/SelfUpdate.swift +++ b/Sources/Swiftly/SelfUpdate.swift @@ -1,7 +1,7 @@ import ArgumentParser import Foundation -import TSCBasic -import TSCUtility +//import TSCBasic +//import TSCUtility import SwiftlyCore @@ -70,10 +70,10 @@ internal struct SelfUpdate: SwiftlyCommand { try? FileManager.default.removeItem(at: tmpFile) } - let animation = PercentProgressAnimation( + /*let animation = PercentProgressAnimation( stream: stdoutStream, header: "Downloading swiftly \(version)" - ) + )*/ do { try await SwiftlyCore.httpClient.downloadFile( url: downloadURL, @@ -82,18 +82,18 @@ internal struct SelfUpdate: SwiftlyCommand { let downloadedMiB = Double(progress.receivedBytes) / (1024.0 * 1024.0) let totalMiB = Double(progress.totalBytes!) / (1024.0 * 1024.0) - animation.update( + /*animation.update( step: progress.receivedBytes, total: progress.totalBytes!, text: "Downloaded \(String(format: "%.1f", downloadedMiB)) MiB of \(String(format: "%.1f", totalMiB)) MiB" - ) + )*/ } ) } catch { - animation.complete(success: false) + //animation.complete(success: false) throw error } - animation.complete(success: true) + //animation.complete(success: true) try await Swiftly.currentPlatform.verifySignature(httpClient: SwiftlyCore.httpClient, archiveDownloadURL: downloadURL, archive: tmpFile, verbose: verbose) try Swiftly.currentPlatform.extractSwiftlyAndInstall(from: tmpFile) diff --git a/Tools/build-swiftly-release/BuildSwiftlyRelease.swift b/Tools/build-swiftly-release/BuildSwiftlyRelease.swift index e8cb1a96..3869c043 100644 --- a/Tools/build-swiftly-release/BuildSwiftlyRelease.swift +++ b/Tools/build-swiftly-release/BuildSwiftlyRelease.swift @@ -371,8 +371,17 @@ struct BuildSwiftlyRelease: AsyncParsableCommand { FileManager.default.changeCurrentDirectoryPath(cwd) - // Statically link standard libraries for maximum portability of the swiftly binary - try runProgram(swift, "build", "--product=swiftly", "--pkg-config-path=\(pkgConfigPath)/lib/pkgconfig", "--static-swift-stdlib", "--configuration=release") + // Statically link standard libraries and use the static sdk for maximum portability +#if arch(arm64) + let sdkName = "aarch64-swift-linux-musl" +#else + let sdkName = "x86_64-swift-linux-musl" +#endif + + // FIXME: Adjust the URL and checksum to match the toolchain that is being used + try runProgram(swift, "sdk", "install" "https://download.swift.org/swift-6.0.3-release/static-sdk/swift-6.0.3-RELEASE/swift-6.0.3-RELEASE_static-linux-0.0.1.artifactbundle.tar.gz", "--checksum", "67f765e0030e661a7450f7e4877cfe008db4f57f177d5a08a6e26fd661cdd0bd") + + try runProgram(swift, "build", "--swift-sdk", sdkName, "--product=swiftly", "--pkg-config-path=\(pkgConfigPath)/lib/pkgconfig", "--static-swift-stdlib", "--configuration=release") let releaseDir = cwd + "/.build/release" From 01a9af288c49680c851381892555da2e0483a198 Mon Sep 17 00:00:00 2001 From: "Chris (SPG) McGee" Date: Thu, 26 Dec 2024 14:04:59 -0500 Subject: [PATCH 2/9] Fix missing comma --- Tools/build-swiftly-release/BuildSwiftlyRelease.swift | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Tools/build-swiftly-release/BuildSwiftlyRelease.swift b/Tools/build-swiftly-release/BuildSwiftlyRelease.swift index 3869c043..ed23aa7c 100644 --- a/Tools/build-swiftly-release/BuildSwiftlyRelease.swift +++ b/Tools/build-swiftly-release/BuildSwiftlyRelease.swift @@ -379,7 +379,7 @@ struct BuildSwiftlyRelease: AsyncParsableCommand { #endif // FIXME: Adjust the URL and checksum to match the toolchain that is being used - try runProgram(swift, "sdk", "install" "https://download.swift.org/swift-6.0.3-release/static-sdk/swift-6.0.3-RELEASE/swift-6.0.3-RELEASE_static-linux-0.0.1.artifactbundle.tar.gz", "--checksum", "67f765e0030e661a7450f7e4877cfe008db4f57f177d5a08a6e26fd661cdd0bd") + try runProgram(swift, "sdk", "install", "https://download.swift.org/swift-6.0.3-release/static-sdk/swift-6.0.3-RELEASE/swift-6.0.3-RELEASE_static-linux-0.0.1.artifactbundle.tar.gz", "--checksum", "67f765e0030e661a7450f7e4877cfe008db4f57f177d5a08a6e26fd661cdd0bd") try runProgram(swift, "build", "--swift-sdk", sdkName, "--product=swiftly", "--pkg-config-path=\(pkgConfigPath)/lib/pkgconfig", "--static-swift-stdlib", "--configuration=release") From ab2354c52d035908b23a9d662c254965cb123e03 Mon Sep 17 00:00:00 2001 From: "Chris (SPG) McGee" Date: Thu, 26 Dec 2024 15:58:25 -0500 Subject: [PATCH 3/9] Compile libarchive with musl-gcc wrapper to fix linking errors --- .../BuildSwiftlyRelease.swift | 71 ++++--------------- 1 file changed, 15 insertions(+), 56 deletions(-) diff --git a/Tools/build-swiftly-release/BuildSwiftlyRelease.swift b/Tools/build-swiftly-release/BuildSwiftlyRelease.swift index ed23aa7c..7a76bd9e 100644 --- a/Tools/build-swiftly-release/BuildSwiftlyRelease.swift +++ b/Tools/build-swiftly-release/BuildSwiftlyRelease.swift @@ -127,54 +127,6 @@ public func getShell() async throws -> String { #endif public func isSupportedLinux(useRhelUbi9: Bool) -> Bool { - let osReleaseFiles = ["/etc/os-release", "/usr/lib/os-release"] - var releaseFile: String? - for file in osReleaseFiles { - if FileManager.default.fileExists(atPath: file) { - releaseFile = file - break - } - } - - guard let releaseFile = releaseFile else { - return false - } - - guard let data = FileManager.default.contents(atPath: releaseFile) else { - return false - } - - guard let releaseInfo = String(data: data, encoding: .utf8) else { - return false - } - - var id: String? - var idlike: String? - var versionID: String? - for info in releaseInfo.split(separator: "\n").map(String.init) { - if info.hasPrefix("ID=") { - id = String(info.dropFirst("ID=".count)).replacingOccurrences(of: "\"", with: "") - } else if info.hasPrefix("ID_LIKE=") { - idlike = String(info.dropFirst("ID_LIKE=".count)).replacingOccurrences(of: "\"", with: "") - } else if info.hasPrefix("VERSION_ID=") { - versionID = String(info.dropFirst("VERSION_ID=".count)).replacingOccurrences(of: "\"", with: "") - } - } - - guard let id = id, let idlike = idlike else { - return false - } - - if useRhelUbi9 { - guard let versionID, versionID.hasPrefix("9"), (id + idlike).contains("rhel") else { - return false - } - } else { - guard let versionID = versionID, versionID == "2", (id + idlike).contains("amzn") else { - return false - } - } - return true } @@ -340,8 +292,13 @@ struct BuildSwiftlyRelease: AsyncParsableCommand { let cwd = FileManager.default.currentDirectoryPath FileManager.default.changeCurrentDirectoryPath(libArchivePath) + let sdkName = "swift-6.0.3-RELEASE_static-linux-0.0.1" + + // FIXME: Adjust the URL and checksum to match the toolchain that is being used + try runProgram(swift, "sdk", "install", "https://download.swift.org/swift-6.0.3-release/static-sdk/swift-6.0.3-RELEASE/swift-6.0.3-RELEASE_static-linux-0.0.1.artifactbundle.tar.gz", "--checksum", "67f765e0030e661a7450f7e4877cfe008db4f57f177d5a08a6e26fd661cdd0bd") + var customEnv = ProcessInfo.processInfo.environment - customEnv["CC"] = "clang" + customEnv["CC"] = "musl-gcc" try runProgramEnv( "./configure", @@ -371,17 +328,19 @@ struct BuildSwiftlyRelease: AsyncParsableCommand { FileManager.default.changeCurrentDirectoryPath(cwd) - // Statically link standard libraries and use the static sdk for maximum portability #if arch(arm64) - let sdkName = "aarch64-swift-linux-musl" + let muslTriple = "aarch64-swift-linux-musl" #else - let sdkName = "x86_64-swift-linux-musl" + let muslTriple = "x86_64-swift-linux-musl" #endif - // FIXME: Adjust the URL and checksum to match the toolchain that is being used - try runProgram(swift, "sdk", "install", "https://download.swift.org/swift-6.0.3-release/static-sdk/swift-6.0.3-RELEASE/swift-6.0.3-RELEASE_static-linux-0.0.1.artifactbundle.tar.gz", "--checksum", "67f765e0030e661a7450f7e4877cfe008db4f57f177d5a08a6e26fd661cdd0bd") - - try runProgram(swift, "build", "--swift-sdk", sdkName, "--product=swiftly", "--pkg-config-path=\(pkgConfigPath)/lib/pkgconfig", "--static-swift-stdlib", "--configuration=release") + do { + try runProgram(swift, "build", "--swift-sdk", sdkName, "--triple=\(muslTriple)", "--product=swiftly", "--pkg-config-path=\(pkgConfigPath)/lib/pkgconfig", "--static-swift-stdlib", "--configuration=release") + } catch { + try runProgram(swift, "sdk", "remove", sdkName) + throw error + } + try runProgram(swift, "sdk", "remove", sdkName) let releaseDir = cwd + "/.build/release" From 95cd4b69a77ec3ec9e47dd91eafa1b99a8919d12 Mon Sep 17 00:00:00 2001 From: "Chris (SPG) McGee" Date: Thu, 26 Dec 2024 21:32:59 -0500 Subject: [PATCH 4/9] Compile libarchive with the same clang and musl as used with swiftly --- .../BuildSwiftlyRelease.swift | 27 ++++----- Tools/build-swiftly-release/musl-clang | 57 +++++++++++++++++++ 2 files changed, 71 insertions(+), 13 deletions(-) create mode 100755 Tools/build-swiftly-release/musl-clang diff --git a/Tools/build-swiftly-release/BuildSwiftlyRelease.swift b/Tools/build-swiftly-release/BuildSwiftlyRelease.swift index 7a76bd9e..c499f392 100644 --- a/Tools/build-swiftly-release/BuildSwiftlyRelease.swift +++ b/Tools/build-swiftly-release/BuildSwiftlyRelease.swift @@ -13,6 +13,7 @@ public struct Error: LocalizedError { } public func runProgramEnv(_ args: String..., quiet: Bool = false, env: [String: String]?) throws { + print("\(args.joined(separator: " "))") let process = Process() process.executableURL = URL(fileURLWithPath: "/usr/bin/env") process.arguments = args @@ -40,6 +41,7 @@ public func runProgramEnv(_ args: String..., quiet: Bool = false, env: [String: } public func runProgram(_ args: String..., quiet: Bool = false) throws { + print("\(args.joined(separator: " "))") let process = Process() process.executableURL = URL(fileURLWithPath: "/usr/bin/env") process.arguments = args @@ -295,10 +297,18 @@ struct BuildSwiftlyRelease: AsyncParsableCommand { let sdkName = "swift-6.0.3-RELEASE_static-linux-0.0.1" // FIXME: Adjust the URL and checksum to match the toolchain that is being used + +#if arch(arm64) + let arch = "aarch64" +#else + let arch = "x86_64" +#endif + try runProgram(swift, "sdk", "install", "https://download.swift.org/swift-6.0.3-release/static-sdk/swift-6.0.3-RELEASE/swift-6.0.3-RELEASE_static-linux-0.0.1.artifactbundle.tar.gz", "--checksum", "67f765e0030e661a7450f7e4877cfe008db4f57f177d5a08a6e26fd661cdd0bd") var customEnv = ProcessInfo.processInfo.environment - customEnv["CC"] = "musl-gcc" + customEnv["CC"] = "\(cwd)/Tools/build-swiftly-release/musl-clang" + customEnv["MUSL_PREFIX"] = "\(FileManager.default.homeDirectoryForCurrentUser.path)/.swiftpm/swift-sdks/\(sdkName).artifactbundle/\(sdkName)/swift-linux-musl/musl-1.2.5.sdk/\(arch)/usr" try runProgramEnv( "./configure", @@ -328,18 +338,9 @@ struct BuildSwiftlyRelease: AsyncParsableCommand { FileManager.default.changeCurrentDirectoryPath(cwd) -#if arch(arm64) - let muslTriple = "aarch64-swift-linux-musl" -#else - let muslTriple = "x86_64-swift-linux-musl" -#endif - - do { - try runProgram(swift, "build", "--swift-sdk", sdkName, "--triple=\(muslTriple)", "--product=swiftly", "--pkg-config-path=\(pkgConfigPath)/lib/pkgconfig", "--static-swift-stdlib", "--configuration=release") - } catch { - try runProgram(swift, "sdk", "remove", sdkName) - throw error - } + // FIXME: running this twice fixes certain linker errors + try? runProgram(swift, "build", "--swift-sdk", sdkName, "--product=swiftly", "--pkg-config-path=\(pkgConfigPath)/lib/pkgconfig", "--static-swift-stdlib", "--configuration=release") + try runProgram(swift, "build", "--swift-sdk", sdkName, "--product=swiftly", "--pkg-config-path=\(pkgConfigPath)/lib/pkgconfig", "--static-swift-stdlib", "--configuration=release") try runProgram(swift, "sdk", "remove", sdkName) let releaseDir = cwd + "/.build/release" diff --git a/Tools/build-swiftly-release/musl-clang b/Tools/build-swiftly-release/musl-clang new file mode 100755 index 00000000..644fdb3f --- /dev/null +++ b/Tools/build-swiftly-release/musl-clang @@ -0,0 +1,57 @@ +#!/bin/sh + +PREFIX=${MUSL_PREFIX:-"/usr/local/musl"} +if [ ! -d "${PREFIX}" ]; then + echo "invalid prefix: ${PREFIX}" + return 1 +fi + +CLANG=${REALCLANG:-"clang"} + +hasNo() { + pat="$1" + shift 1 + + for e in "$@"; do + if [ "$e" = "${pat}" ]; then + return 1 + fi + done + return 0 +} + +ARGS="-nostdinc" +TAIL="" + +if hasNo '-nostdinc' "$@"; then + ARGS="${ARGS} -isystem ${PREFIX}/include" +fi + +if \ + hasNo '-c' "$@" && \ + hasNo '-S' "$@" && \ + hasNo '-E' "$@" +then + ARGS="${ARGS} -nostdlib" + ARGS="${ARGS} -Wl,-dynamic-linker=${PREFIX}/lib/libc.so" + ARGS="${ARGS} -L${PREFIX}/lib -L${PREFIX}/lib/swift/clang/lib/linux" + + if hasNo '-nostartfiles' "$@" && \ + hasNo '-nostdlib' "$@" && \ + hasNo '-nodefaultlibs' "$@" + then + ARGS="${ARGS} ${PREFIX}/lib/crt1.o" + ARGS="${ARGS} ${PREFIX}/lib/crti.o" + + TAIL="${TAIL} ${PREFIX}/lib/crtn.o" + fi + + if hasNo '-nostdlib' "$@" && \ + hasNo '-nodefaultlibs' "$@" + then + TAIL="${TAIL} -lc -lclang_rt.builtins-$(uname -m)" + fi +fi + +exec ${CLANG} ${ARGS} "$@" ${TAIL} -static + From 04d881cd6c9f414aaf21fc3ad6568a40568a6f8c Mon Sep 17 00:00:00 2001 From: "Chris (SPG) McGee" Date: Fri, 27 Dec 2024 07:28:46 -0500 Subject: [PATCH 5/9] Fix linker errors by using the right swift-sdk value --- Tools/build-swiftly-release/BuildSwiftlyRelease.swift | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/Tools/build-swiftly-release/BuildSwiftlyRelease.swift b/Tools/build-swiftly-release/BuildSwiftlyRelease.swift index c499f392..a7b4155e 100644 --- a/Tools/build-swiftly-release/BuildSwiftlyRelease.swift +++ b/Tools/build-swiftly-release/BuildSwiftlyRelease.swift @@ -338,9 +338,7 @@ struct BuildSwiftlyRelease: AsyncParsableCommand { FileManager.default.changeCurrentDirectoryPath(cwd) - // FIXME: running this twice fixes certain linker errors - try? runProgram(swift, "build", "--swift-sdk", sdkName, "--product=swiftly", "--pkg-config-path=\(pkgConfigPath)/lib/pkgconfig", "--static-swift-stdlib", "--configuration=release") - try runProgram(swift, "build", "--swift-sdk", sdkName, "--product=swiftly", "--pkg-config-path=\(pkgConfigPath)/lib/pkgconfig", "--static-swift-stdlib", "--configuration=release") + try runProgram(swift, "build", "--swift-sdk", "\(arch)-swift-linux-musl", "--product=swiftly", "--pkg-config-path=\(pkgConfigPath)/lib/pkgconfig", "--static-swift-stdlib", "--configuration=release") try runProgram(swift, "sdk", "remove", sdkName) let releaseDir = cwd + "/.build/release" From ca7aeb72ea5fcb970395e06a59498e81f9e58540 Mon Sep 17 00:00:00 2001 From: "Chris (SPG) McGee" Date: Fri, 27 Dec 2024 08:20:17 -0500 Subject: [PATCH 6/9] Re-add the progress animation code --- Package.resolved | 11 ++++++++++- Package.swift | 4 ++-- Sources/Swiftly/Install.swift | 16 ++++++++-------- Sources/Swiftly/SelfUpdate.swift | 16 ++++++++-------- .../BuildSwiftlyRelease.swift | 4 ++-- 5 files changed, 30 insertions(+), 21 deletions(-) diff --git a/Package.resolved b/Package.resolved index 11a9893a..edb24aa5 100644 --- a/Package.resolved +++ b/Package.resolved @@ -1,5 +1,5 @@ { - "originHash" : "cc03f1f46471dab3d83dd2c3efecbcbb0807f138854b4d0f03bcee316fe44231", + "originHash" : "531e10b955219c0de91ada74260f59bff8033189f1a9f9f78b199480c61f466a", "pins" : [ { "identity" : "async-http-client", @@ -145,6 +145,15 @@ "version" : "1.2.1" } }, + { + "identity" : "swift-tools-support-core", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-tools-support-core.git", + "state" : { + "revision" : "b464fcd8d884e599e3202d9bd1eee29a9e504069", + "version" : "0.7.2" + } + }, { "identity" : "swiftformat", "kind" : "remoteSourceControl", diff --git a/Package.swift b/Package.swift index 3eb0d091..71bed523 100644 --- a/Package.swift +++ b/Package.swift @@ -17,7 +17,7 @@ let package = Package( .package(url: "https://github.com/apple/swift-argument-parser", from: "1.3.0"), .package(url: "https://github.com/swift-server/async-http-client", from: "1.21.2"), .package(url: "https://github.com/apple/swift-nio.git", from: "2.64.0"), - //.package(url: "https://github.com/apple/swift-tools-support-core.git", from: "0.7.1"), + .package(url: "https://github.com/apple/swift-tools-support-core.git", from: "0.7.2"), .package(url: "https://github.com/apple/swift-docc-plugin", from: "1.3.0"), // This dependency provides the correct version of the formatter so that you can run `swift run swiftformat Package.swift Plugins/ Sources/ Tests/` .package(url: "https://github.com/nicklockwood/SwiftFormat", exact: "0.49.18"), @@ -30,7 +30,7 @@ let package = Package( .target(name: "SwiftlyCore"), .target(name: "LinuxPlatform", condition: .when(platforms: [.linux])), .target(name: "MacOSPlatform", condition: .when(platforms: [.macOS])), - //.product(name: "SwiftToolsSupport-auto", package: "swift-tools-support-core"), + .product(name: "SwiftToolsSupport-auto", package: "swift-tools-support-core"), ] ), .target( diff --git a/Sources/Swiftly/Install.swift b/Sources/Swiftly/Install.swift index 6449ba46..7de678fa 100644 --- a/Sources/Swiftly/Install.swift +++ b/Sources/Swiftly/Install.swift @@ -1,8 +1,8 @@ import _StringProcessing import ArgumentParser import Foundation -//import TSCBasic -//import TSCUtility +import TSCBasic +import TSCUtility import SwiftlyCore @@ -192,10 +192,10 @@ struct Install: SwiftlyCommand { throw Error(message: "Invalid toolchain URL: \(url)") } - /*let animation = PercentProgressAnimation( + let animation = PercentProgressAnimation( stream: stdoutStream, header: "Downloading \(version)" - )*/ + ) var lastUpdate = Date() @@ -215,20 +215,20 @@ struct Install: SwiftlyCommand { lastUpdate = Date() - /*animation.update( + animation.update( step: progress.receivedBytes, total: progress.totalBytes!, text: "Downloaded \(String(format: "%.1f", downloadedMiB)) MiB of \(String(format: "%.1f", totalMiB)) MiB" - )*/ + ) } ) } catch let notFound as SwiftlyHTTPClient.DownloadNotFoundError { throw Error(message: "\(version) does not exist at URL \(notFound.url), exiting") } catch { - //animation.complete(success: false) + animation.complete(success: false) throw error } - //animation.complete(success: true) + animation.complete(success: true) if verifySignature { try await Swiftly.currentPlatform.verifySignature( diff --git a/Sources/Swiftly/SelfUpdate.swift b/Sources/Swiftly/SelfUpdate.swift index db703515..e41dea60 100644 --- a/Sources/Swiftly/SelfUpdate.swift +++ b/Sources/Swiftly/SelfUpdate.swift @@ -1,7 +1,7 @@ import ArgumentParser import Foundation -//import TSCBasic -//import TSCUtility +import TSCBasic +import TSCUtility import SwiftlyCore @@ -70,10 +70,10 @@ internal struct SelfUpdate: SwiftlyCommand { try? FileManager.default.removeItem(at: tmpFile) } - /*let animation = PercentProgressAnimation( + let animation = PercentProgressAnimation( stream: stdoutStream, header: "Downloading swiftly \(version)" - )*/ + ) do { try await SwiftlyCore.httpClient.downloadFile( url: downloadURL, @@ -82,18 +82,18 @@ internal struct SelfUpdate: SwiftlyCommand { let downloadedMiB = Double(progress.receivedBytes) / (1024.0 * 1024.0) let totalMiB = Double(progress.totalBytes!) / (1024.0 * 1024.0) - /*animation.update( + animation.update( step: progress.receivedBytes, total: progress.totalBytes!, text: "Downloaded \(String(format: "%.1f", downloadedMiB)) MiB of \(String(format: "%.1f", totalMiB)) MiB" - )*/ + ) } ) } catch { - //animation.complete(success: false) + animation.complete(success: false) throw error } - //animation.complete(success: true) + animation.complete(success: true) try await Swiftly.currentPlatform.verifySignature(httpClient: SwiftlyCore.httpClient, archiveDownloadURL: downloadURL, archive: tmpFile, verbose: verbose) try Swiftly.currentPlatform.extractSwiftlyAndInstall(from: tmpFile) diff --git a/Tools/build-swiftly-release/BuildSwiftlyRelease.swift b/Tools/build-swiftly-release/BuildSwiftlyRelease.swift index a7b4155e..89d0e559 100644 --- a/Tools/build-swiftly-release/BuildSwiftlyRelease.swift +++ b/Tools/build-swiftly-release/BuildSwiftlyRelease.swift @@ -128,8 +128,8 @@ public func getShell() async throws -> String { } #endif -public func isSupportedLinux(useRhelUbi9: Bool) -> Bool { - return true +public func isSupportedLinux(useRhelUbi9 _: Bool) -> Bool { + true } @main From 999f0e2a3e94e7ecd3931224c8c852c285209c3d Mon Sep 17 00:00:00 2001 From: "Chris (SPG) McGee" Date: Fri, 27 Dec 2024 08:31:55 -0500 Subject: [PATCH 7/9] Force a newline to display the swiftly prompts for user input --- Sources/SwiftlyCore/SwiftlyCore.swift | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Sources/SwiftlyCore/SwiftlyCore.swift b/Sources/SwiftlyCore/SwiftlyCore.swift index d3eb231a..5d2754a2 100644 --- a/Sources/SwiftlyCore/SwiftlyCore.swift +++ b/Sources/SwiftlyCore/SwiftlyCore.swift @@ -45,7 +45,7 @@ public protocol InputProvider { public var inputProvider: (any InputProvider)? public func readLine(prompt: String) -> String? { - print(prompt, terminator: ": ") + print(prompt, terminator: ": \n") guard let provider = SwiftlyCore.inputProvider else { return Swift.readLine(strippingNewline: true) } From 134b46e3864858e7609c481aa97411502b12335f Mon Sep 17 00:00:00 2001 From: "Chris (SPG) McGee" Date: Fri, 27 Dec 2024 12:10:37 -0500 Subject: [PATCH 8/9] Install static sdk matching current swift version Remove supported linux distribution check and deprecate command-line flag --- .../BuildSwiftlyRelease.swift | 46 +++++++++++++------ 1 file changed, 31 insertions(+), 15 deletions(-) diff --git a/Tools/build-swiftly-release/BuildSwiftlyRelease.swift b/Tools/build-swiftly-release/BuildSwiftlyRelease.swift index 89d0e559..dcf67bac 100644 --- a/Tools/build-swiftly-release/BuildSwiftlyRelease.swift +++ b/Tools/build-swiftly-release/BuildSwiftlyRelease.swift @@ -1,6 +1,16 @@ import ArgumentParser import Foundation +public struct SwiftPlatform: Codable { + public var name: String? + public var checksum: String? +} + +public struct SwiftRelease: Codable { + public var name: String? + public var platforms: [SwiftPlatform]? +} + // These functions are cloned and adapted from SwiftlyCore until we can do better bootstrapping public struct Error: LocalizedError { public let message: String @@ -128,10 +138,6 @@ public func getShell() async throws -> String { } #endif -public func isSupportedLinux(useRhelUbi9 _: Bool) -> Bool { - true -} - @main struct BuildSwiftlyRelease: AsyncParsableCommand { static let configuration = CommandConfiguration( @@ -149,7 +155,7 @@ struct BuildSwiftlyRelease: AsyncParsableCommand { @Option(help: "Package identifier of macOS package") var identifier: String = "org.swift.swiftly" #elseif os(Linux) - @Flag(name: .long, help: "Use RHEL UBI9 as the supported Linux to build a release instead of Amazon Linux 2") + @Flag(name: .long, help: "Deprecated option since releases can be built on any swift supported Linux distribution.") var useRhelUbi9: Bool = false #endif @@ -249,13 +255,6 @@ struct BuildSwiftlyRelease: AsyncParsableCommand { } func buildLinuxRelease() async throws { -#if os(Linux) - // Check system requirements - guard isSupportedLinux(useRhelUbi9: self.useRhelUbi9) else { - throw Error(message: "Linux releases must be made from specific distributions so that the binary can be used everyone else because it has the oldest version of glibc for maximum compatibility with other versions of Linux. Please try again with \(!self.useRhelUbi9 ? "Amazon Linux 2" : "RedHat UBI 9").") - } -#endif - // TODO: turn these into checks that the system meets the criteria for being capable of using the toolchain + checking for packages, not tools let curl = try await self.assertTool("curl", message: "Please install curl with `yum install curl`") let tar = try await self.assertTool("tar", message: "Please install tar with `yum install tar`") @@ -294,9 +293,15 @@ struct BuildSwiftlyRelease: AsyncParsableCommand { let cwd = FileManager.default.currentDirectoryPath FileManager.default.changeCurrentDirectoryPath(libArchivePath) - let sdkName = "swift-6.0.3-RELEASE_static-linux-0.0.1" + let swiftVerRegex: Regex<(Substring, Substring)> = try! Regex("Swift version (\\d+\\.\\d+\\.\\d+) ") + let swiftVerOutput = (try await runProgramOutput(swift, "--version")) ?? "" + guard let swiftVerMatch = try swiftVerRegex.firstMatch(in: swiftVerOutput) else { + throw Error(message: "Unable to detect swift version") + } - // FIXME: Adjust the URL and checksum to match the toolchain that is being used + let swiftVersion = swiftVerMatch.output.1 + + let sdkName = "swift-\(swiftVersion)-RELEASE_static-linux-0.0.1" #if arch(arm64) let arch = "aarch64" @@ -304,7 +309,18 @@ struct BuildSwiftlyRelease: AsyncParsableCommand { let arch = "x86_64" #endif - try runProgram(swift, "sdk", "install", "https://download.swift.org/swift-6.0.3-release/static-sdk/swift-6.0.3-RELEASE/swift-6.0.3-RELEASE_static-linux-0.0.1.artifactbundle.tar.gz", "--checksum", "67f765e0030e661a7450f7e4877cfe008db4f57f177d5a08a6e26fd661cdd0bd") + let swiftReleasesJson = (try await runProgramOutput(curl, "https://www.swift.org/api/v1/install/releases.json")) ?? "[]" + let swiftReleases = try JSONDecoder().decode([SwiftRelease].self, from: swiftReleasesJson.data(using: .utf8)!) + + guard let swiftRelease = swiftReleases.first(where: { ($0.name ?? "") == swiftVersion }) else { + throw Error(message: "Unable to find swift release using swift.org API: \(swiftVersion)") + } + + guard let sdkPlatform = (swiftRelease.platforms ?? [SwiftPlatform]()).first(where: { ($0.name ?? "") == "Static SDK" }) else { + throw Error(message: "Swift release \(swiftVersion) has no Static SDK offering") + } + + try runProgram(swift, "sdk", "install", "https://download.swift.org/swift-\(swiftVersion)-release/static-sdk/swift-\(swiftVersion)-RELEASE/swift-\(swiftVersion)-RELEASE_static-linux-0.0.1.artifactbundle.tar.gz", "--checksum", sdkPlatform.checksum ?? "deadbeef") var customEnv = ProcessInfo.processInfo.environment customEnv["CC"] = "\(cwd)/Tools/build-swiftly-release/musl-clang" From e41fd1377218304c6897ec325081023ea4ac87ce Mon Sep 17 00:00:00 2001 From: "Chris (SPG) McGee" Date: Tue, 7 Jan 2025 10:47:51 -0500 Subject: [PATCH 9/9] Honour the quiet parameter for runProgram and runProgramEnv in the release script --- Tools/build-swiftly-release/BuildSwiftlyRelease.swift | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/Tools/build-swiftly-release/BuildSwiftlyRelease.swift b/Tools/build-swiftly-release/BuildSwiftlyRelease.swift index dcf67bac..320918c5 100644 --- a/Tools/build-swiftly-release/BuildSwiftlyRelease.swift +++ b/Tools/build-swiftly-release/BuildSwiftlyRelease.swift @@ -23,7 +23,8 @@ public struct Error: LocalizedError { } public func runProgramEnv(_ args: String..., quiet: Bool = false, env: [String: String]?) throws { - print("\(args.joined(separator: " "))") + if !quiet { print("\(args.joined(separator: " "))") } + let process = Process() process.executableURL = URL(fileURLWithPath: "/usr/bin/env") process.arguments = args @@ -51,7 +52,8 @@ public func runProgramEnv(_ args: String..., quiet: Bool = false, env: [String: } public func runProgram(_ args: String..., quiet: Bool = false) throws { - print("\(args.joined(separator: " "))") + if !quiet { print("\(args.joined(separator: " "))") } + let process = Process() process.executableURL = URL(fileURLWithPath: "/usr/bin/env") process.arguments = args