diff --git a/.swift-version b/.swift-version index f9ce5a96..e0ea36fe 100644 --- a/.swift-version +++ b/.swift-version @@ -1 +1 @@ -5.10 +6.0 diff --git a/Package.swift b/Package.swift index a4a95146..ae23796f 100644 --- a/Package.swift +++ b/Package.swift @@ -67,6 +67,13 @@ let package = Package( ], path: "Tools/generate-docs-reference" ), + .executableTarget( + name: "build-swiftly-release", + dependencies: [ + .product(name: "ArgumentParser", package: "swift-argument-parser"), + ], + path: "Tools/build-swiftly-release" + ), .target( name: "LinuxPlatform", dependencies: [ diff --git a/RELEASING.md b/RELEASING.md index 36895b40..b09f729c 100644 --- a/RELEASING.md +++ b/RELEASING.md @@ -1,8 +1,6 @@ # Releasing -Swiftly and the swiftly-install release script have different release schedules and their version numbers do not correspond. Below is instructions for releasing each. - -## Releasing swiftly +Swift has a tool for producing final product packages, suitable for distribution. Follow these steps to complete a release and test the packaging. 1. Check out the commit you wish to create a release for. Ensure no other local modifications or changes are present. @@ -14,20 +12,9 @@ Swiftly and the swiftly-install release script have different release schedules 5. Create a tag on that commit with the format "x.y.z". Do not omit "z", even if its value is 0. -6. Build the executables for the release by running ./scripts/build_release.sh from the root of the swiftly repository (do this once on an x86_64 machine and once on an aarch64 one) +6. Build the executables for the release by running `swift run build-swiftly-release ` from the root of the swiftly repository + * Build on a Apple silicon macOS machine to produce a universal package for x86_64 and arm64 + * Build on an Amazon Linux 2 image for x86_64 + * Build on an Amazon Linux 2 image for arm64 7. Push the tag to `origin`. `git push origin ` - -8. Go to the GitHub page for the new tag, click edit tag, add an appropriate description, attach the prebuilt executables, and click "Publish Release". - -## Releasing swiftly-install - -1. Check out the commit you wish to create a release for. Ensure no other local modifications or changes are present. - -2. Ensure the version string `SWIFTLY_INSTALL_VERSION` in `install/swiftly-install.sh` is accurate. If it is not, push another commit updating it to the proper value. - -3. Create a tag on that commit with the format "swiftly-install-x.y.z". Do not omit "z", even if its value is 0. - -4. Push the tag to `origin`. `git push origin ` - -5. Copy `install/swiftly-install.sh` to website branch of repository diff --git a/Sources/LinuxPlatform/Linux.swift b/Sources/LinuxPlatform/Linux.swift index 776b723a..6deb26c2 100644 --- a/Sources/LinuxPlatform/Linux.swift +++ b/Sources/LinuxPlatform/Linux.swift @@ -193,44 +193,21 @@ public struct Linux: Platform { throw Error(message: msg) } - let foundKeys = (try? self.runProgram( - "gpg", - "--list-keys", - "swift-infrastructure@forums.swift.org", - "swift-infrastructure@swift.org", - quiet: true - )) != nil - if !foundKeys { - // Import the swift keys if they aren't here already + // Import the latest swift keys, but only once per session, which will help with the performance in tests + if !swiftGPGKeysRefreshed { let tmpFile = self.getTempFilePath() FileManager.default.createFile(atPath: tmpFile.path, contents: nil, attributes: [.posixPermissions: 0o600]) defer { try? FileManager.default.removeItem(at: tmpFile) } - guard let url = URL(string: "https://swift.org/keys/all-keys.asc") else { + guard let url = URL(string: "https://www.swift.org/keys/all-keys.asc") else { throw Error(message: "malformed URL to the swift gpg keys") } try await httpClient.downloadFile(url: url, to: tmpFile) try self.runProgram("gpg", "--import", tmpFile.path, quiet: true) - } - // We only need to refresh the keys once per session, which will help with performance in tests - if !swiftGPGKeysRefreshed { - SwiftlyCore.print("Refreshing Swift PGP keys...") - do { - try self.runProgram( - "gpg", - "--quiet", - "--keyserver", - "hkp://keyserver.ubuntu.com", - "--refresh-keys", - "Swift" - ) - } catch { - throw Error(message: "Failed to refresh PGP keys: \(error)") - } swiftGPGKeysRefreshed = true } } @@ -290,6 +267,23 @@ public struct Linux: Platform { } } + public func extractSwiftlyAndInstall(from archive: URL) throws { + guard archive.fileExists() else { + throw Error(message: "\(archive) doesn't exist") + } + + let tmpDir = self.getTempFilePath() + try FileManager.default.createDirectory(atPath: tmpDir.path, withIntermediateDirectories: true) + + SwiftlyCore.print("Extracting new swiftly...") + try extractArchive(atPath: archive) { name in + // Extract to the temporary directory + tmpDir.appendingPathComponent(String(name)) + } + + try self.runProgram(tmpDir.appendingPathComponent("swiftly").path, "init") + } + public func uninstall(_ toolchain: ToolchainVersion) throws { let toolchainDir = self.swiftlyToolchainsDir.appendingPathComponent(toolchain.name) try FileManager.default.removeItem(at: toolchainDir) @@ -410,7 +404,7 @@ public struct Linux: Platform { do { try self.runProgram("gpg", "--verify", sigFile.path, archive.path) } catch { - throw Error(message: "Toolchain signature verification failed: \(error).") + throw Error(message: "Signature verification failed: \(error).") } } diff --git a/Sources/MacOSPlatform/MacOS.swift b/Sources/MacOSPlatform/MacOS.swift index 978cab8b..619e0a93 100644 --- a/Sources/MacOSPlatform/MacOS.swift +++ b/Sources/MacOSPlatform/MacOS.swift @@ -80,6 +80,43 @@ public struct MacOS: Platform { } } + public func extractSwiftlyAndInstall(from archive: URL) throws { + guard archive.fileExists() else { + throw Error(message: "\(archive) doesn't exist") + } + + let homeDir: URL + + if SwiftlyCore.mockedHomeDir == nil { + homeDir = FileManager.default.homeDirectoryForCurrentUser + + SwiftlyCore.print("Extracting the swiftly package...") + try runProgram("installer", "-pkg", archive.path, "-target", "CurrentUserHomeDirectory") + try? runProgram("pkgutil", "--volume", homeDir.path, "--forget", "org.swift.swiftly") + } else { + homeDir = SwiftlyCore.mockedHomeDir ?? FileManager.default.homeDirectoryForCurrentUser + + let installDir = homeDir.appendingPathComponent("usr/local") + try FileManager.default.createDirectory(atPath: installDir.path, withIntermediateDirectories: true) + + // In the case of a mock for testing purposes we won't use the installer, perferring a manual process because + // the installer will not install to an arbitrary path, only a volume or user home directory. + let tmpDir = self.getTempFilePath() + try runProgram("pkgutil", "--expand", archive.path, tmpDir.path) + + // There's a slight difference in the location of the special Payload file between official swift packages + // and the ones that are mocked here in the test framework. + let payload = tmpDir.appendingPathComponent("Payload") + guard payload.fileExists() else { + throw Error(message: "Payload file could not be found at \(tmpDir).") + } + + try runProgram("tar", "-C", installDir.path, "-xf", payload.path) + } + + try self.runProgram(homeDir.appendingPathComponent("usr/local/bin/swiftly").path, "init") + } + public func uninstall(_ toolchain: ToolchainVersion) throws { SwiftlyCore.print("Uninstalling package in user home directory...") diff --git a/Sources/Swiftly/SelfUpdate.swift b/Sources/Swiftly/SelfUpdate.swift index 72743eee..b8534f48 100644 --- a/Sources/Swiftly/SelfUpdate.swift +++ b/Sources/Swiftly/SelfUpdate.swift @@ -14,27 +14,52 @@ internal struct SelfUpdate: SwiftlyCommand { internal mutating func run() async throws { try validateSwiftly() - SwiftlyCore.print("Checking for swiftly updates...") - let release: SwiftlyGitHubRelease = try await SwiftlyCore.httpClient.getFromGitHub( - url: "https://api.github.com/repos/swift-server/swiftly/releases/latest" - ) + let swiftlyBin = Swiftly.currentPlatform.swiftlyBinDir.appendingPathComponent("swiftly") + guard FileManager.default.fileExists(atPath: swiftlyBin.path) else { + throw Error(message: "Self update doesn't work when swiftly has been installed externally. Please keep it updated from the source where you installed it in the first place.") + } + + let _ = try await Self.execute() + } + + public static func execute() async throws -> SwiftlyVersion { + SwiftlyCore.print("Checking for swiftly updates...") - let version = try SwiftlyVersion(parsing: release.tag) + let swiftlyRelease = try await SwiftlyCore.httpClient.getSwiftlyRelease() - guard version > SwiftlyCore.version else { + guard swiftlyRelease.version > SwiftlyCore.version else { SwiftlyCore.print("Already up to date.") - return + return SwiftlyCore.version } - SwiftlyCore.print("A new version is available: \(version)") + var downloadURL: Foundation.URL? + for platform in swiftlyRelease.platforms { +#if os(macOS) + guard platform.platform == .Darwin else { + continue + } +#elseif os(Linux) + guard platform.platform == .Linux else { + continue + } +#endif + +#if arch(x86_64) + downloadURL = platform.x86_64 +#elseif arch(arm64) + downloadURL = platform.arm64 +#endif + } - let executableName = Swiftly.currentPlatform.getExecutableName() - let urlString = "https://github.com/swift-server/swiftly/versions/latest/download/\(executableName)" - guard let downloadURL = URL(string: urlString) else { - throw Error(message: "Invalid download url: \(urlString)") + guard let downloadURL = downloadURL else { + throw Error(message: "The newest release of swiftly is incompatible with your current OS and/or processor architecture.") } + let version = swiftlyRelease.version + + SwiftlyCore.print("A new version is available: \(version)") + let tmpFile = Swiftly.currentPlatform.getTempFilePath() FileManager.default.createFile(atPath: tmpFile.path, contents: nil) defer { @@ -66,11 +91,10 @@ internal struct SelfUpdate: SwiftlyCommand { } animation.complete(success: true) - let swiftlyExecutable = Swiftly.currentPlatform.swiftlyBinDir.appendingPathComponent("swiftly", isDirectory: false) - try FileManager.default.removeItem(at: swiftlyExecutable) - try FileManager.default.moveItem(at: tmpFile, to: swiftlyExecutable) - try FileManager.default.setAttributes([.posixPermissions: 0o755], ofItemAtPath: swiftlyExecutable.path) + try await Swiftly.currentPlatform.verifySignature(httpClient: SwiftlyCore.httpClient, archiveDownloadURL: downloadURL, archive: tmpFile) + try Swiftly.currentPlatform.extractSwiftlyAndInstall(from: tmpFile) SwiftlyCore.print("Successfully updated swiftly to \(version) (was \(SwiftlyCore.version))") + return version } } diff --git a/Sources/SwiftlyCore/HTTPClient.swift b/Sources/SwiftlyCore/HTTPClient.swift index 397d7c7a..c1da0649 100644 --- a/Sources/SwiftlyCore/HTTPClient.swift +++ b/Sources/SwiftlyCore/HTTPClient.swift @@ -22,13 +22,26 @@ private func makeRequest(url: String) -> HTTPClientRequest { return request } -struct SwiftOrgSwiftlyRelease: Codable { - var name: String +public enum SwiftOrgSwiftlyPlatformType: String, Codable { + case Darwin + case Linux + case Windows +} + +public struct SwiftOrgSwiftlyPlatform: Codable { + public var platform: SwiftOrgSwiftlyPlatformType + public var x86_64: URL + public var arm64: URL +} + +public struct SwiftOrgSwiftlyRelease: Codable { + public var version: SwiftlyVersion + public var platforms: [SwiftOrgSwiftlyPlatform] } struct SwiftOrgPlatform: Codable { var name: String - var archs: [String] + var archs: [String]? /// platform is a mapping from the 'name' field of the swift.org platform object /// to swiftly's PlatformDefinition, if possible. @@ -170,6 +183,13 @@ public struct SwiftlyHTTPClient { return try JSONDecoder().decode(type.self, from: response.buffer) } + /// Return the current Swiftly release using the swift.org API. + public func getSwiftlyRelease() async throws -> SwiftOrgSwiftlyRelease { + let url = "https://www.swift.org/api/v1/swiftly.json" + let swiftlyRelease: SwiftOrgSwiftlyRelease = try await self.getFromJSON(url: url, type: SwiftOrgSwiftlyRelease.self) + return swiftlyRelease + } + /// Return an array of released Swift versions that match the given filter, up to the provided /// limit (default unlimited). public func getReleaseToolchains( @@ -190,7 +210,7 @@ public struct SwiftlyHTTPClient { a! } - let url = "https://swift.org/api/v1/install/releases.json" + let url = "https://www.swift.org/api/v1/install/releases.json" let swiftOrgReleases: [SwiftOrgRelease] = try await self.getFromJSON(url: url, type: [SwiftOrgRelease].self) var swiftOrgFiltered: [ToolchainVersion.StableRelease] = try swiftOrgReleases.compactMap { swiftOrgRelease in @@ -200,7 +220,7 @@ public struct SwiftlyHTTPClient { return nil } - guard swiftOrgPlatform.archs.contains(arch) else { + guard let archs = swiftOrgPlatform.archs, archs.contains(arch) else { return nil } } @@ -277,7 +297,7 @@ public struct SwiftlyHTTPClient { platform.name } - let url = "https://swift.org/api/v1/install/dev/\(branch.name)/\(platformName).json" + let url = "https://www.swift.org/api/v1/install/dev/\(branch.name)/\(platformName).json" // For a particular branch and platform the snapshots are listed underneath their architecture let swiftOrgSnapshotArchs: SwiftOrgSnapshotList = try await self.getFromJSON(url: url, type: SwiftOrgSnapshotList.self) diff --git a/Sources/SwiftlyCore/Platform.swift b/Sources/SwiftlyCore/Platform.swift index 32d2e13a..b4a64459 100644 --- a/Sources/SwiftlyCore/Platform.swift +++ b/Sources/SwiftlyCore/Platform.swift @@ -57,6 +57,10 @@ public protocol Platform { /// After this completes, a user can “use” the toolchain. func install(from: URL, version: ToolchainVersion) throws + /// Extract swiftly from the provided downloaded archive and install + /// ourselves from that. + func extractSwiftlyAndInstall(from archive: URL) throws + /// Uninstalls a toolchain associated with the given version. /// If this version is in use, the next latest version will be used afterwards. func uninstall(_ version: ToolchainVersion) throws diff --git a/Tests/SwiftlyTests/HTTPClientTests.swift b/Tests/SwiftlyTests/HTTPClientTests.swift index e72155ae..1d666d68 100644 --- a/Tests/SwiftlyTests/HTTPClientTests.swift +++ b/Tests/SwiftlyTests/HTTPClientTests.swift @@ -7,7 +7,7 @@ final class HTTPClientTests: SwiftlyTests { // GIVEN: we have a swiftly http client // WHEN: we make get request for a particular type of JSON var releases: [SwiftOrgRelease] = try await SwiftlyCore.httpClient.getFromJSON( - url: "https://swift.org/api/v1/install/releases.json", + url: "https://www.swift.org/api/v1/install/releases.json", type: [SwiftOrgRelease].self, headers: [:] ) @@ -19,7 +19,7 @@ final class HTTPClientTests: SwiftlyTests { var exceptionThrown = false do { releases = try await SwiftlyCore.httpClient.getFromJSON( - url: "https://swift.org/api/v1/install/releases-invalid.json", + url: "https://www.swift.org/api/v1/install/releases-invalid.json", type: [SwiftOrgRelease].self, headers: [:] ) diff --git a/Tests/SwiftlyTests/SelfUpdateTests.swift b/Tests/SwiftlyTests/SelfUpdateTests.swift index b405d294..1adb49a2 100644 --- a/Tests/SwiftlyTests/SelfUpdateTests.swift +++ b/Tests/SwiftlyTests/SelfUpdateTests.swift @@ -6,56 +6,23 @@ import NIO import XCTest final class SelfUpdateTests: SwiftlyTests { - private static var newMajorVersion: String { - "\(SwiftlyCore.version.major + 1).0.0" + private static var newMajorVersion: SwiftlyVersion { + SwiftlyVersion(major: SwiftlyCore.version.major + 1, minor: 0, patch: 0) } - private static var newMinorVersion: String { - "\(SwiftlyCore.version.major).\(SwiftlyCore.version.minor + 1).0" + private static var newMinorVersion: SwiftlyVersion { + SwiftlyVersion(major: SwiftlyCore.version.major, minor: SwiftlyCore.version.minor + 1, patch: 0) } - private static var newPatchVersion: String { - "\(SwiftlyCore.version.major).\(SwiftlyCore.version.minor).\(SwiftlyCore.version.patch + 1)" + private static var newPatchVersion: SwiftlyVersion { + SwiftlyVersion(major: SwiftlyCore.version.major, minor: SwiftlyCore.version.minor, patch: SwiftlyCore.version.patch + 1) } - private static func mockHTTPHandler(latestVersion: String) -> ((HTTPClientRequest) async throws -> HTTPClientResponse) { - return { request in - guard let url = URL(string: request.url) else { - throw SwiftlyTestError(message: "invalid url \(request.url)") - } - - switch url.host { - case "api.github.com": - let nextRelease = SwiftlyGitHubRelease(tag: latestVersion) - var buffer = ByteBuffer() - try buffer.writeJSONEncodable(nextRelease) - return HTTPClientResponse(body: .bytes(buffer)) - case "github.com": - let buffer = ByteBuffer(string: latestVersion) - return HTTPClientResponse(body: .bytes(buffer)) - default: - throw SwiftlyTestError(message: "unknown url host: \(String(describing: url.host))") - } - } - } - - func runSelfUpdateTest(latestVersion: String, shouldUpdate: Bool = true) async throws { + func runSelfUpdateTest(latestVersion: SwiftlyVersion) async throws { try await self.withTestHome { - let swiftlyURL = Swiftly.currentPlatform.swiftlyBinDir.appendingPathComponent("swiftly", isDirectory: false) - try Data("old".utf8).write(to: swiftlyURL) - - var update = try self.parseCommand(SelfUpdate.self, ["self-update"]) - - try await self.withMockedHTTPRequests(Self.mockHTTPHandler(latestVersion: latestVersion)) { - try await update.run() - } - - let swiftly = try Data(contentsOf: swiftlyURL) - - if shouldUpdate { - XCTAssertEqual(String(decoding: swiftly, as: UTF8.self), latestVersion) - } else { - XCTAssertEqual(String(decoding: swiftly, as: UTF8.self), "old") + try await self.withMockedSwiftlyVersion(latestSwiftlyVersion: latestVersion) { + let updatedVersion = try await SelfUpdate.execute() + XCTAssertEqual(latestVersion, updatedVersion) } } } @@ -68,17 +35,6 @@ final class SelfUpdateTests: SwiftlyTests { /// Verify updating the most up-to-date toolchain has no effect. func testSelfUpdateAlreadyUpToDate() async throws { - try await self.runSelfUpdateTest(latestVersion: String(describing: SwiftlyCore.version), shouldUpdate: false) - } - - /// Tests that attempting to self-update using the actual GitHub API works as expected. - func testSelfUpdateIntegration() async throws { - try await self.withTestHome { - let swiftlyURL = Swiftly.currentPlatform.swiftlyBinDir.appendingPathComponent("swiftly", isDirectory: false) - try Data("old".utf8).write(to: swiftlyURL) - - var update = try self.parseCommand(SelfUpdate.self, ["self-update"]) - try await update.run() - } + try await self.runSelfUpdateTest(latestVersion: SwiftlyCore.version) } } diff --git a/Tests/SwiftlyTests/SwiftlyTests.swift b/Tests/SwiftlyTests/SwiftlyTests.swift index 9f469765..a384e37e 100644 --- a/Tests/SwiftlyTests/SwiftlyTests.swift +++ b/Tests/SwiftlyTests/SwiftlyTests.swift @@ -196,6 +196,17 @@ class SwiftlyTests: XCTestCase { } } + func withMockedSwiftlyVersion(latestSwiftlyVersion: SwiftlyVersion = SwiftlyCore.version, _ f: () async throws -> Void) async throws { + let prevExecutor = SwiftlyCore.httpRequestExecutor + let mockDownloader = MockToolchainDownloader(executables: ["swift"], latestSwiftlyVersion: latestSwiftlyVersion, delegate: prevExecutor) + SwiftlyCore.httpRequestExecutor = mockDownloader + defer { + SwiftlyCore.httpRequestExecutor = prevExecutor + } + + try await f() + } + func withMockedToolchain(executables: [String]? = nil, f: () async throws -> Void) async throws { let prevExecutor = SwiftlyCore.httpRequestExecutor let mockDownloader = MockToolchainDownloader(executables: executables, delegate: prevExecutor) @@ -544,12 +555,15 @@ public class MockToolchainDownloader: HTTPRequestExecutor { #endif private let delegate: HTTPRequestExecutor - public init(executables: [String]? = nil, delegate: HTTPRequestExecutor) { + private let latestSwiftlyVersion: SwiftlyVersion + + public init(executables: [String]? = nil, latestSwiftlyVersion: SwiftlyVersion = SwiftlyCore.version, delegate: HTTPRequestExecutor) { self.executables = executables ?? ["swift"] #if os(Linux) self.signatures = [:] #endif self.delegate = delegate + self.latestSwiftlyVersion = latestSwiftlyVersion } public func execute(_ request: HTTPClientRequest, timeout: TimeAmount) async throws -> HTTPClientResponse { @@ -557,7 +571,11 @@ public class MockToolchainDownloader: HTTPRequestExecutor { throw SwiftlyTestError(message: "invalid request URL: \(request.url)") } - if url.host == "download.swift.org" { + if url.host == "download.swift.org" && url.path.hasPrefix("/swiftly-") { + // Download a swiftly bundle + return try self.makeSwiftlyDownloadResponse(from: url) + } else if url.host == "download.swift.org" && (url.path.hasPrefix("/swift-") || url.path.hasPrefix("/development")) { + // Download a toolchain return try self.makeToolchainDownloadResponse(from: url) } else if url.host == "api.github.com" { if url.path == "/repos/apple/swift/tags" { @@ -565,7 +583,33 @@ public class MockToolchainDownloader: HTTPRequestExecutor { } else { throw SwiftlyTestError(message: "unxpected github API request URL: \(request.url)") } - } else if url.host == "swift.org" { + } else if url.host == "www.swift.org" && url.path == "/api/v1/swiftly.json" { + // Mock a response that would represent the swiftly offerings using the latest version + let decoder = JSONDecoder() + let json = """ + { + "version": "\(self.latestSwiftlyVersion)", + "platforms": [ + { + "platform": "Darwin", + "x86_64": "https://download.swift.org/swiftly-darwin.pkg", + "arm64": "https://download.swift.org/swiftly-darwin.pkg" + }, + { + "platform": "Linux", + "x86_64": "https://download.swift.org/swiftly-linux.tar.gz", + "arm64": "https://download.swift.org/swiftly-linux.tar.gz" + } + ] + } + """.data(using: .utf8)! + + let swiftlyRelease = try decoder.decode(SwiftOrgSwiftlyRelease.self, from: json) + var buffer = ByteBuffer() + try buffer.writeJSONEncodable(swiftlyRelease) + return HTTPClientResponse(body: .bytes(buffer)) + } else if url.host == "www.swift.org" { + // Delegate any API requests to swift.org return try await self.delegate.execute(request, timeout: timeout) } else { throw SwiftlyTestError(message: "unmocked URL: \(request)") @@ -649,7 +693,96 @@ public class MockToolchainDownloader: HTTPRequestExecutor { return HTTPClientResponse(body: .bytes(ByteBuffer(data: mockedToolchain))) } + private func makeSwiftlyDownloadResponse(from url: URL) throws -> HTTPClientResponse { + let mockedSwiftly = try self.makeMockedSwiftly(from: url) + return HTTPClientResponse(body: .bytes(ByteBuffer(data: mockedSwiftly))) + } + #if os(Linux) + public func makeMockedSwiftly(from url: URL) throws -> Data { + // Check our cache if this is a signature request + if url.path.hasSuffix(".sig") { + // Signatures will either be in the cache or this don't exist + guard let signature = self.signatures["swiftly"] else { + throw SwiftlyTestError(message: "swiftly signature wasn't found in the cache") + } + + return try Data(contentsOf: signature) + } + + let tmp = FileManager.default.temporaryDirectory.appendingPathComponent("swiftly-\(UUID())") + let swiftlyDir = tmp.appendingPathComponent("swiftly", isDirectory: true) + + try FileManager.default.createDirectory( + at: swiftlyDir, + withIntermediateDirectories: true + ) + + for executable in ["swiftly"] { + let executablePath = swiftlyDir.appendingPathComponent(executable) + + let script = """ + #!/usr/bin/env sh + + echo 'Installed' + """ + + let data = Data(script.utf8) + try data.write(to: executablePath) + + // make the file executable + try FileManager.default.setAttributes([.posixPermissions: 0o755], ofItemAtPath: executablePath.path) + } + + let archive = tmp.appendingPathComponent("swiftly.tar.gz") + + let task = Process() + task.executableURL = URL(fileURLWithPath: "/usr/bin/env") + task.arguments = ["bash", "-c", "tar -C \(swiftlyDir.path) -czf \(archive.path) swiftly"] + + try task.run() + task.waitUntilExit() + + // Extra step involves generating a gpg signature and putting that in a cache for a later request. We will + // use a local key for this to avoid running into entropy problems in CI. + let gpgKeyFile = FileManager.default.temporaryDirectory.appendingPathComponent("swiftly-\(UUID())") + try Data(PackageResources.mock_signing_key_private_pgp).write(to: gpgKeyFile) + let importKey = Process() + importKey.executableURL = URL(fileURLWithPath: "/usr/bin/env") + importKey.arguments = ["bash", "-c", """ + mkdir -p $HOME/.gnupg + touch $HOME/.gnupg/gpg.conf + gpg --batch --import \(gpgKeyFile.path) >/dev/null 2>&1 || echo -n + """] + try importKey.run() + importKey.waitUntilExit() + if importKey.terminationStatus != 0 { + throw SwiftlyTestError(message: "unable to import test gpg signing key") + } + + let detachSign = Process() + detachSign.executableURL = URL(fileURLWithPath: "/usr/bin/env") + detachSign.arguments = ["bash", "-c", """ + export GPG_TTY=$(tty) + gpg --version | grep '2.0.' > /dev/null + if [ "$?" == "0" ]; then + gpg --default-key "A2A645E5249D25845C43954E7D210032D2F670B7" --detach-sign "\(archive.path)" + else + gpg --pinentry-mode loopback --default-key "A2A645E5249D25845C43954E7D210032D2F670B7" --detach-sign "\(archive.path)" + fi + """] + try detachSign.run() + detachSign.waitUntilExit() + + if detachSign.terminationStatus != 0 { + throw SwiftlyTestError(message: "unable to sign archive using the test user's gpg key") + } + + self.signatures["swiftly"] = archive.appendingPathExtension("sig") + + return try Data(contentsOf: archive) + } + public func makeMockedToolchain(toolchain: ToolchainVersion, name: String) throws -> Data { // Check our cache if this is a signature request if name.hasSuffix(".sig") { @@ -738,6 +871,57 @@ public class MockToolchainDownloader: HTTPRequestExecutor { } #elseif os(macOS) + public func makeMockedSwiftly(from _: URL) throws -> Data { + let tmp = FileManager.default.temporaryDirectory.appendingPathComponent("swiftly-\(UUID())") + let swiftlyDir = tmp.appendingPathComponent("swiftly", isDirectory: true) + let swiftlyBinDir = swiftlyDir.appendingPathComponent("bin") + + try FileManager.default.createDirectory( + at: swiftlyBinDir, + withIntermediateDirectories: true + ) + + defer { + try? FileManager.default.removeItem(at: tmp) + } + + for executable in ["swiftly"] { + let executablePath = swiftlyBinDir.appendingPathComponent(executable) + + let script = """ + #!/usr/bin/env sh + + echo 'Installed.' + """ + + let data = Data(script.utf8) + try data.write(to: executablePath) + + // make the file executable + try FileManager.default.setAttributes([.posixPermissions: 0o755], ofItemAtPath: executablePath.path) + } + + let pkg = tmp.appendingPathComponent("swiftly.pkg") + + let task = Process() + task.executableURL = URL(fileURLWithPath: "/usr/bin/env") + task.arguments = [ + "pkgbuild", + "--root", + swiftlyDir.path, + "--install-location", + "usr/local", + "--version", + "\(self.latestSwiftlyVersion)", + "--identifier", + "org.swift.swiftly", + pkg.path, + ] + try task.run() + task.waitUntilExit() + + return try Data(contentsOf: pkg) + } public func makeMockedToolchain(toolchain: ToolchainVersion, name _: String) throws -> Data { let tmp = FileManager.default.temporaryDirectory.appendingPathComponent("swiftly-\(UUID())") diff --git a/Tools/build-swiftly-release/BuildSwiftlyRelease.swift b/Tools/build-swiftly-release/BuildSwiftlyRelease.swift new file mode 100644 index 00000000..d5bc7efb --- /dev/null +++ b/Tools/build-swiftly-release/BuildSwiftlyRelease.swift @@ -0,0 +1,405 @@ +import ArgumentParser +import Foundation + +// These functions are cloned and adapted from SwiftlyCore until we can do better bootstrapping +public struct Error: LocalizedError { + public let message: String + + public init(message: String) { + self.message = message + } + + public var errorDescription: String? { self.message } +} + +public func runProgramEnv(_ args: String..., quiet: Bool = false, env: [String: String]?) throws { + let process = Process() + process.executableURL = URL(fileURLWithPath: "/usr/bin/env") + process.arguments = args + + if let env = env { + process.environment = env + } + + if quiet { + process.standardOutput = nil + process.standardError = nil + } + + try process.run() + // Attach this process to our process group so that Ctrl-C and other signals work + let pgid = tcgetpgrp(STDOUT_FILENO) + if pgid != -1 { + tcsetpgrp(STDOUT_FILENO, process.processIdentifier) + } + process.waitUntilExit() + + guard process.terminationStatus == 0 else { + throw Error(message: "\(args.first!) exited with non-zero status: \(process.terminationStatus)") + } +} + +public func runProgram(_ args: String..., quiet: Bool = false) throws { + let process = Process() + process.executableURL = URL(fileURLWithPath: "/usr/bin/env") + process.arguments = args + + if quiet { + process.standardOutput = nil + process.standardError = nil + } + + try process.run() + // Attach this process to our process group so that Ctrl-C and other signals work + let pgid = tcgetpgrp(STDOUT_FILENO) + if pgid != -1 { + tcsetpgrp(STDOUT_FILENO, process.processIdentifier) + } + process.waitUntilExit() + + guard process.terminationStatus == 0 else { + throw Error(message: "\(args.first!) exited with non-zero status: \(process.terminationStatus)") + } +} + +public func runProgramOutput(_ program: String, _ args: String...) async throws -> String? { + let process = Process() + process.executableURL = URL(fileURLWithPath: "/usr/bin/env") + process.arguments = [program] + args + + let outPipe = Pipe() + process.standardInput = FileHandle.nullDevice + process.standardError = FileHandle.nullDevice + process.standardOutput = outPipe + + try process.run() + // Attach this process to our process group so that Ctrl-C and other signals work + let pgid = tcgetpgrp(STDOUT_FILENO) + if pgid != -1 { + tcsetpgrp(STDOUT_FILENO, process.processIdentifier) + } + let outData = try outPipe.fileHandleForReading.readToEnd() + + process.waitUntilExit() + + guard process.terminationStatus == 0 else { + throw Error(message: "\(args.first!) exited with non-zero status: \(process.terminationStatus)") + } + + if let outData = outData { + return String(data: outData, encoding: .utf8) + } else { + return nil + } +} + +#if os(macOS) +public func getShell() async throws -> String { + if let directoryInfo = try await runProgramOutput("dscl", ".", "-read", FileManager.default.homeDirectoryForCurrentUser.path) { + for line in directoryInfo.components(separatedBy: "\n") { + if line.hasPrefix("UserShell: ") { + if case let comps = line.components(separatedBy: ": "), comps.count == 2 { + return comps[1] + } + } + } + } + + // Fall back to zsh on macOS + return "/bin/zsh" +} + +#elseif os(Linux) +public func getShell() async throws -> String { + if let passwds = try await runProgramOutput("getent", "passwd") { + for line in passwds.components(separatedBy: "\n") { + if line.hasPrefix("root:") { + if case let comps = line.components(separatedBy: ":"), comps.count > 1 { + return comps[comps.count - 1] + } + } + } + } + + // Fall back on bash on Linux and other Unixes + return "/bin/bash" +} +#endif + +public func isAmazonLinux2() -> Bool { + let osReleaseFiles = ["/etc/os-release", "/usr/lib/os-release"] + var releaseFile: String? + for file in osReleaseFiles { + if FileManager.default.fileExists(atPath: file) { + releaseFile = file + break + } + } + + guard let releaseFile = releaseFile else { + return false + } + + guard let data = FileManager.default.contents(atPath: releaseFile) else { + return false + } + + guard let releaseInfo = String(data: data, encoding: .utf8) else { + return false + } + + var id: String? + var idlike: String? + var versionID: String? + for info in releaseInfo.split(separator: "\n").map(String.init) { + if info.hasPrefix("ID=") { + id = String(info.dropFirst("ID=".count)).replacingOccurrences(of: "\"", with: "") + } else if info.hasPrefix("ID_LIKE=") { + idlike = String(info.dropFirst("ID_LIKE=".count)).replacingOccurrences(of: "\"", with: "") + } else if info.hasPrefix("VERSION_ID=") { + versionID = String(info.dropFirst("VERSION_ID=".count)).replacingOccurrences(of: "\"", with: "") + } + } + + guard let id = id, let idlike = idlike else { + return false + } + + guard let versionID = versionID, versionID == "2", (id + idlike).contains("amzn") else { + return false + } + + return true +} + +@main +struct BuildSwiftlyRelease: AsyncParsableCommand { + static let configuration = CommandConfiguration( + commandName: "build-swiftly-release", + abstract: "Build final swiftly product for a release." + ) + + @Flag(name: .long, help: "Skip the git repo checks and proceed.") + var skip: Bool = false + + @Argument(help: "Version of swiftly to build the release.") + var version: String + + func validate() throws {} + + func run() async throws { +#if os(Linux) + try await self.buildLinuxRelease() +#elseif os(macOS) + try await self.buildMacOSRelease() +#else + #error("Unsupported OS") +#endif + } + + func assertTool(_ name: String, message: String) async throws -> String { + guard let _ = try? await runProgramOutput(getShell(), "-c", "which which") else { + throw Error(message: "The which command could not be found. Please install it with your package manager.") + } + + guard let location = try? await runProgramOutput(getShell(), "-c", "which \(name)") else { + throw Error(message: message) + } + + return location.replacingOccurrences(of: "\n", with: "") + } + + func findSwiftVersion() throws -> String? { + var cwd = URL(fileURLWithPath: FileManager.default.currentDirectoryPath) + + while cwd.path != "" && cwd.path != "/" { + guard FileManager.default.fileExists(atPath: cwd.path) else { + break + } + + let svFile = cwd.appendingPathComponent(".swift-version") + + if FileManager.default.fileExists(atPath: svFile.path) { + let selector = try? String(contentsOf: svFile, encoding: .utf8) + if let selector = selector { + return selector.replacingOccurrences(of: "\n", with: "") + } + return selector + } + + cwd = cwd.deletingLastPathComponent() + } + + return nil + } + + func checkSwiftRequirement() async throws -> String { + guard let requiredSwiftVersion = try? self.findSwiftVersion() else { + throw Error(message: "Unable to determine the required swift version for this version of swiftly. Please make sure that you `cd ` and there is a .swift-version file there.") + } + + let swift = try await self.assertTool("swift", message: "Please install swift \(requiredSwiftVersion) and make sure that it is added to your path.") + + // We also need a swift toolchain with the correct version + guard let swiftVersion = try await runProgramOutput(swift, "--version"), swiftVersion.contains("Swift version \(requiredSwiftVersion)") else { + throw Error(message: "Swiftly releases require a Swift \(requiredSwiftVersion) toolchain available on the path") + } + + return swift + } + + func checkGitRepoStatus(_ git: String) async throws { + guard !self.skip else { + return + } + + guard let gitTags = try await runProgramOutput(git, "log", "-n1", "--pretty=format:%d"), gitTags.contains("tag: \(self.version)") else { + throw Error(message: "Git repo is not yet tagged for release \(self.version). Please tag this commit with that version and push it to GitHub.") + } + + do { + try runProgram(git, "diff-index", "--quiet", "HEAD") + } catch { + throw Error(message: "Git repo has local changes. First commit these changes, tag the commit with release \(self.version) and push the tag to GitHub.") + } + } + + func collectLicenses(_ licenseDir: String) async throws { + try FileManager.default.createDirectory(atPath: licenseDir, withIntermediateDirectories: true) + + let cwd = FileManager.default.currentDirectoryPath + + // Copy the swiftly license to the bundle + try FileManager.default.copyItem(atPath: cwd + "/LICENSE.txt", toPath: licenseDir + "/LICENSE.txt") + } + + func buildLinuxRelease() async throws { + // Check system requirements + guard isAmazonLinux2() else { + // TODO: see if docker can be used to spawn an Amazon Linux 2 container to continue the release building process + throw Error(message: "Linux releases must be made from Amazon Linux 2 because it has the oldest version of glibc for maximum compatibility with other versions of Linux") + } + + // TODO: turn these into checks that the system meets the criteria for being capable of using the toolchain + checking for packages, not tools + let curl = try await self.assertTool("curl", message: "Please install curl with `yum install curl`") + let tar = try await self.assertTool("tar", message: "Please install tar with `yum install tar`") + let make = try await self.assertTool("make", message: "Please install make with `yum install make`") + let git = try await self.assertTool("git", message: "Please install git with `yum install git`") + let strip = try await self.assertTool("strip", message: "Please install strip with `yum install binutils`") + + let swift = try await self.checkSwiftRequirement() + + try await self.checkGitRepoStatus(git) + + // Build a specific version of libarchive + let libArchiveVersion = "3.7.4" + let buildCheckoutsDir = FileManager.default.currentDirectoryPath + "/.build/checkouts" + let libArchivePath = buildCheckoutsDir + "/libarchive-\(libArchiveVersion)" + let pkgConfigPath = libArchivePath + "/pkgconfig" + + try? FileManager.default.createDirectory(atPath: buildCheckoutsDir, withIntermediateDirectories: true) + try? FileManager.default.createDirectory(atPath: pkgConfigPath, withIntermediateDirectories: true) + + try? FileManager.default.removeItem(atPath: libArchivePath) + try runProgram(curl, "-o", "\(buildCheckoutsDir + "/libarchive-\(libArchiveVersion).tar.gz")", "--remote-name", "--location", "https://github.com/libarchive/libarchive/releases/download/v\(libArchiveVersion)/libarchive-\(libArchiveVersion).tar.gz") + try runProgram(tar, "--directory=\(buildCheckoutsDir)", "-xzf", "\(buildCheckoutsDir)/libarchive-\(libArchiveVersion).tar.gz") + + let cwd = FileManager.default.currentDirectoryPath + FileManager.default.changeCurrentDirectoryPath(libArchivePath) + + var customEnv = ProcessInfo.processInfo.environment + customEnv["CC"] = "clang" + + try runProgramEnv( + "./configure", + "--prefix=\(pkgConfigPath)", + "--enable-shared=no", + "--with-pic", + "--without-nettle", + "--without-openssl", + "--without-lzo2", + "--without-expat", + "--without-xml2", + "--without-bz2lib", + "--without-libb2", + "--without-iconv", + "--without-zstd", + "--without-lzma", + "--without-lz4", + "--disable-acl", + "--disable-bsdtar", + "--disable-bsdcat", + env: customEnv + ) + + try runProgramEnv(make, env: customEnv) + + try runProgram(make, "install") + + FileManager.default.changeCurrentDirectoryPath(cwd) + + try runProgram(swift, "package", "clean") + + // Statically link standard libraries for maximum portability of the swiftly binary + try runProgram(swift, "build", "--product=swiftly", "--pkg-config-path=\(pkgConfigPath)/lib/pkgconfig", "--static-swift-stdlib", "--configuration=release") + + let releaseDir = cwd + "/.build/release" + + // Strip the symbols from the binary to decrease its size + try runProgram(strip, releaseDir + "/swiftly") + + try await self.collectLicenses(releaseDir) + +#if arch(arm64) + let releaseArchive = "\(releaseDir)/swiftly-\(version)-aarch64.tar.gz" +#else + let releaseArchive = "\(releaseDir)/swiftly-\(version).tar.gz" +#endif + + try runProgram(tar, "--directory=\(releaseDir)", "-czf", releaseArchive, "swiftly", "LICENSE.txt") + + print(releaseArchive) + } + + func buildMacOSRelease() async throws { + // Check system requirements + let git = try await self.assertTool("git", message: "Please install git with either `xcode-select --install` or `brew install git`") + + let swift = try await checkSwiftRequirement() + + try await self.checkGitRepoStatus(git) + + let lipo = try await self.assertTool("lipo", message: "In order to make a universal binary there needs to be the `lipo` tool that is installed on macOS.") + let pkgbuild = try await self.assertTool("pkgbuild", message: "In order to make pkg installers there needs to be the `pkgbuild` tool that is installed on macOS.") + let strip = try await self.assertTool("strip", message: "In order to strip binaries there needs to be the `strip` tool that is installed on macOS.") + + try runProgram(swift, "package", "clean") + + for arch in ["x86_64", "arm64"] { + try runProgram(swift, "build", "--product=swiftly", "--configuration=release", "--arch=\(arch)") + try runProgram(strip, ".build/\(arch)-apple-macosx/release/swiftly") + } + + let swiftlyBinDir = FileManager.default.currentDirectoryPath + "/.build/release/usr/local/bin" + try? FileManager.default.createDirectory(atPath: swiftlyBinDir, withIntermediateDirectories: true) + + try runProgram(lipo, ".build/x86_64-apple-macosx/release/swiftly", ".build/arm64-apple-macosx/release/swiftly", "-create", "-o", "\(swiftlyBinDir)/swiftly") + + let swiftlyLicenseDir = FileManager.default.currentDirectoryPath + "/.build/release/usr/local/share/doc/swiftly/license" + try? FileManager.default.createDirectory(atPath: swiftlyLicenseDir, withIntermediateDirectories: true) + try await self.collectLicenses(swiftlyLicenseDir) + + try runProgram( + pkgbuild, + "--root", + swiftlyBinDir + "/..", + "--install-location", + "usr/local", + "--version", + self.version, + "--identifier", + "org.swift.swiftly", + ".build/release/swiftly-\(self.version).pkg" + ) + } +} diff --git a/Tools/generate-docs-reference/GenerateDocsReference.swift b/Tools/generate-docs-reference/GenerateDocsReference.swift index 77cc2cde..06e5e8ed 100644 --- a/Tools/generate-docs-reference/GenerateDocsReference.swift +++ b/Tools/generate-docs-reference/GenerateDocsReference.swift @@ -3,7 +3,7 @@ import ArgumentParserToolInfo import Foundation @main -struct GenerateDocsReferencel: ParsableCommand { +struct GenerateDocsReference: ParsableCommand { enum Error: Swift.Error { case failedToRunSubprocess(error: Swift.Error) case unableToParseToolOutput(error: Swift.Error) diff --git a/scripts/build.dockerfile b/scripts/build.dockerfile deleted file mode 100644 index 58bb7e6e..00000000 --- a/scripts/build.dockerfile +++ /dev/null @@ -1,25 +0,0 @@ -# Dockerfile used to build a statically-linked swiftly executable for generic GNU/Linux platforms. -# See RELEASING.md for information on how to use this file. - -FROM swift:5.10-amazonlinux2 - -# swiftly build depdenencies -RUN yum install -y \ - curl \ - gcc \ - make - -COPY . /tmp/swiftly -WORKDIR /tmp/swiftly - -RUN ./scripts/install-libarchive.sh - -RUN swift build \ - --static-swift-stdlib \ - --configuration release - -RUN mv .build/release/swiftly /swiftly -RUN strip /swiftly - -RUN /swiftly --version -RUN ldd /swiftly diff --git a/scripts/build_release.sh b/scripts/build_release.sh deleted file mode 100755 index fa6e283e..00000000 --- a/scripts/build_release.sh +++ /dev/null @@ -1,44 +0,0 @@ -#!/usr/bin/env bash - -set -o errexit - -version="$1" - -if [[ -z "$version" ]]; then - echo "Usage: build_release.sh " - exit 1 -fi - -raw_arch="$(uname -m)" -case "$raw_arch" in - "x86_64") - arch="x86_64" - ;; - - "aarch64" | "arm64") - arch="aarch64" - ;; - - *) - echo "Error: Unsupported CPU architecture: $raw_arch" - ;; -esac - -git checkout "$version" - -if [[ ! -z "$(git status --porcelain=v1 2>/dev/null)" ]]; then - echo "There are uncommitted changes in the local tree, please commit or discard them" - exit 1 -fi - -image_name="swiftly-$version" -binary_name="swiftly-$arch-unknown-linux-gnu" -docker build -t "$image_name" -f scripts/build.dockerfile . -container_id=$(docker create "$image_name") -docker cp "$container_id:/swiftly" "$binary_name" -docker rm -v "$container_id" -docker image rm -f "$image_name" - -"./$binary_name" --version > /dev/null - -echo "$binary_name has been successfully built!"