Enhancements to the release build script

Package.swift

@@ -67,6 +67,13 @@ let package = Package(
             ],
             path: "Tools/generate-docs-reference"
         ),
+        .executableTarget(
+            name: "build-swiftly-release",
+            dependencies: [
+                .product(name: "ArgumentParser", package: "swift-argument-parser"),
+            ],
+            path: "Tools/build-swiftly-release"
+        ),
         .target(
             name: "LinuxPlatform",
             dependencies: [

RELEASING.md

@@ -1,8 +1,6 @@
 # Releasing
 
-Swiftly and the swiftly-install release script have different release schedules and their version numbers do not correspond. Below is instructions for releasing each.
-
-## Releasing swiftly
+Swift has a tool for producing final product packages, suitable for distribution. Follow these steps to complete a release and test the packaging.
 
 1. Check out the commit you wish to create a release for. Ensure no other local modifications or changes are present.
 
@@ -14,20 +12,9 @@ Swiftly and the swiftly-install release script have different release schedules
 
 5. Create a tag on that commit with the format "x.y.z". Do not omit "z", even if its value is 0.
 
-6. Build the executables for the release by running ./scripts/build_release.sh from the root of the swiftly repository (do this once on an x86_64 machine and once on an aarch64 one)
+6. Build the executables for the release by running `swift run build-swiftly-release <version>` from the root of the swiftly repository
+  * Build on a Apple silicon macOS machine to produce a universal package for x86_64 and arm64
+  * Build on an Amazon Linux 2 image for x86_64
+  * Build on an Amazon Linux 2 image for arm64
 
 7. Push the tag to `origin`. `git push origin <tag_name>`
-
-8. Go to the GitHub page for the new tag, click edit tag, add an appropriate description, attach the prebuilt executables, and click "Publish Release".
-
-## Releasing swiftly-install
-
-1. Check out the commit you wish to create a release for. Ensure no other local modifications or changes are present.
-
-2. Ensure the version string `SWIFTLY_INSTALL_VERSION` in `install/swiftly-install.sh` is accurate. If it is not, push another commit updating it to the proper value.
-
-3. Create a tag on that commit with the format "swiftly-install-x.y.z". Do not omit "z", even if its value is 0.
-
-4. Push the tag to `origin`. `git push origin <tag_name>`
-
-5. Copy `install/swiftly-install.sh` to website branch of repository

Sources/LinuxPlatform/Linux.swift

@@ -193,44 +193,21 @@ public struct Linux: Platform {
                 throw Error(message: msg)
             }
 
-            let foundKeys = (try? self.runProgram(
-                "gpg",
-                "--list-keys",
-                "[email protected]",
-                "[email protected]",
-                quiet: true
-            )) != nil
-            if !foundKeys {
-                // Import the swift keys if they aren't here already
+            // Import the latest swift keys, but only once per session, which will help with the performance in tests
+            if !swiftGPGKeysRefreshed {
                 let tmpFile = self.getTempFilePath()
                 FileManager.default.createFile(atPath: tmpFile.path, contents: nil, attributes: [.posixPermissions: 0o600])
                 defer {
                     try? FileManager.default.removeItem(at: tmpFile)
                 }
 
-                guard let url = URL(string: "https://swift.org/keys/all-keys.asc") else {
+                guard let url = URL(string: "https://www.swift.org/keys/all-keys.asc") else {
                     throw Error(message: "malformed URL to the swift gpg keys")
                 }
 
                 try await httpClient.downloadFile(url: url, to: tmpFile)
                 try self.runProgram("gpg", "--import", tmpFile.path, quiet: true)
-            }
 
-            // We only need to refresh the keys once per session, which will help with performance in tests
-            if !swiftGPGKeysRefreshed {
-                SwiftlyCore.print("Refreshing Swift PGP keys...")
-                do {
-                    try self.runProgram(
-                        "gpg",
-                        "--quiet",
-                        "--keyserver",
-                        "hkp://keyserver.ubuntu.com",
-                        "--refresh-keys",
-                        "Swift"
-                    )
-                } catch {
-                    throw Error(message: "Failed to refresh PGP keys: \(error)")
-                }
                 swiftGPGKeysRefreshed = true
             }
         }
@@ -290,6 +267,23 @@ public struct Linux: Platform {
         }
     }
 
+    public func extractSwiftlyAndInstall(from archive: URL) throws {
+        guard archive.fileExists() else {
+            throw Error(message: "\(archive) doesn't exist")
+        }
+
+        let tmpDir = self.getTempFilePath()
+        try FileManager.default.createDirectory(atPath: tmpDir.path, withIntermediateDirectories: true)
+
+        SwiftlyCore.print("Extracting new swiftly...")
+        try extractArchive(atPath: archive) { name in
+            // Extract to the temporary directory
+            tmpDir.appendingPathComponent(String(name))
+        }
+
+        try self.runProgram(tmpDir.appendingPathComponent("swiftly").path, "init")
+    }
+
     public func uninstall(_ toolchain: ToolchainVersion) throws {
         let toolchainDir = self.swiftlyToolchainsDir.appendingPathComponent(toolchain.name)
         try FileManager.default.removeItem(at: toolchainDir)
@@ -410,7 +404,7 @@ public struct Linux: Platform {
         do {
             try self.runProgram("gpg", "--verify", sigFile.path, archive.path)
         } catch {
-            throw Error(message: "Toolchain signature verification failed: \(error).")
+            throw Error(message: "Signature verification failed: \(error).")
         }
     }
 

Sources/MacOSPlatform/MacOS.swift

@@ -80,6 +80,43 @@ public struct MacOS: Platform {
         }
     }
 
+    public func extractSwiftlyAndInstall(from archive: URL) throws {
+        guard archive.fileExists() else {
+            throw Error(message: "\(archive) doesn't exist")
+        }
+
+        let homeDir: URL
+
+        if SwiftlyCore.mockedHomeDir == nil {
+            homeDir = FileManager.default.homeDirectoryForCurrentUser
+
+            SwiftlyCore.print("Extracting the swiftly package...")
+            try runProgram("installer", "-pkg", archive.path, "-target", "CurrentUserHomeDirectory")
+            try? runProgram("pkgutil", "--volume", homeDir.path, "--forget", "org.swift.swiftly")
+        } else {
+            homeDir = SwiftlyCore.mockedHomeDir ?? FileManager.default.homeDirectoryForCurrentUser
+
+            let installDir = homeDir.appendingPathComponent("usr/local")
+            try FileManager.default.createDirectory(atPath: installDir.path, withIntermediateDirectories: true)
+
+            // In the case of a mock for testing purposes we won't use the installer, perferring a manual process because
+            //  the installer will not install to an arbitrary path, only a volume or user home directory.
+            let tmpDir = self.getTempFilePath()
+            try runProgram("pkgutil", "--expand", archive.path, tmpDir.path)
+
+            // There's a slight difference in the location of the special Payload file between official swift packages
+            // and the ones that are mocked here in the test framework.
+            let payload = tmpDir.appendingPathComponent("Payload")
+            guard payload.fileExists() else {
+                throw Error(message: "Payload file could not be found at \(tmpDir).")
+            }
+
+            try runProgram("tar", "-C", installDir.path, "-xf", payload.path)
+        }
+
+        try self.runProgram(homeDir.appendingPathComponent("usr/local/bin/swiftly").path, "init")
+    }
+
     public func uninstall(_ toolchain: ToolchainVersion) throws {
         SwiftlyCore.print("Uninstalling package in user home directory...")
 

Sources/Swiftly/SelfUpdate.swift

@@ -14,27 +14,52 @@ internal struct SelfUpdate: SwiftlyCommand {
 
     internal mutating func run() async throws {
         try validateSwiftly()
-        SwiftlyCore.print("Checking for swiftly updates...")
 
-        let release: SwiftlyGitHubRelease = try await SwiftlyCore.httpClient.getFromGitHub(
-            url: "https://api.github.com/repos/swift-server/swiftly/releases/latest"
-        )
+        let swiftlyBin = Swiftly.currentPlatform.swiftlyBinDir.appendingPathComponent("swiftly")
+        guard FileManager.default.fileExists(atPath: swiftlyBin.path) else {
+            throw Error(message: "Self update doesn't work when swiftly has been installed externally. Please keep it updated from the source where you installed it in the first place.")
+        }
+
+        let _ = try await Self.execute()
+    }
+
+    public static func execute() async throws -> SwiftlyVersion {
+        SwiftlyCore.print("Checking for swiftly updates...")
 
-        let version = try SwiftlyVersion(parsing: release.tag)
+        let swiftlyRelease = try await SwiftlyCore.httpClient.getSwiftlyRelease()
 
-        guard version > SwiftlyCore.version else {
+        guard swiftlyRelease.version > SwiftlyCore.version else {
             SwiftlyCore.print("Already up to date.")
-            return
+            return SwiftlyCore.version
         }
 
-        SwiftlyCore.print("A new version is available: \(version)")
+        var downloadURL: Foundation.URL?
+        for platform in swiftlyRelease.platforms {
+#if os(macOS)
+            guard platform.platform == .Darwin else {
+                continue
+            }
+#elseif os(Linux)
+            guard platform.platform == .Linux else {
+                continue
+            }
+#endif
+
+#if arch(x86_64)
+            downloadURL = platform.x86_64
+#elseif arch(arm64)
+            downloadURL = platform.arm64
+#endif
+        }
 
-        let executableName = Swiftly.currentPlatform.getExecutableName()
-        let urlString = "https://github.com/swift-server/swiftly/versions/latest/download/\(executableName)"
-        guard let downloadURL = URL(string: urlString) else {
-            throw Error(message: "Invalid download url: \(urlString)")
+        guard let downloadURL = downloadURL else {
+            throw Error(message: "The newest release of swiftly is incompatible with your current OS and/or processor architecture.")
         }
 
+        let version = swiftlyRelease.version
+
+        SwiftlyCore.print("A new version is available: \(version)")
+
         let tmpFile = Swiftly.currentPlatform.getTempFilePath()
         FileManager.default.createFile(atPath: tmpFile.path, contents: nil)
         defer {
@@ -66,11 +91,10 @@ internal struct SelfUpdate: SwiftlyCommand {
         }
         animation.complete(success: true)
 
-        let swiftlyExecutable = Swiftly.currentPlatform.swiftlyBinDir.appendingPathComponent("swiftly", isDirectory: false)
-        try FileManager.default.removeItem(at: swiftlyExecutable)
-        try FileManager.default.moveItem(at: tmpFile, to: swiftlyExecutable)
-        try FileManager.default.setAttributes([.posixPermissions: 0o755], ofItemAtPath: swiftlyExecutable.path)
+        try await Swiftly.currentPlatform.verifySignature(httpClient: SwiftlyCore.httpClient, archiveDownloadURL: downloadURL, archive: tmpFile)
+        try Swiftly.currentPlatform.extractSwiftlyAndInstall(from: tmpFile)
 
         SwiftlyCore.print("Successfully updated swiftly to \(version) (was \(SwiftlyCore.version))")
+        return version
     }
 }

Sources/SwiftlyCore/HTTPClient.swift

@@ -22,13 +22,26 @@ private func makeRequest(url: String) -> HTTPClientRequest {
     return request
 }
 
-struct SwiftOrgSwiftlyRelease: Codable {
-    var name: String
+public enum SwiftOrgSwiftlyPlatformType: String, Codable {
+    case Darwin
+    case Linux
+    case Windows
+}
+
+public struct SwiftOrgSwiftlyPlatform: Codable {
+    public var platform: SwiftOrgSwiftlyPlatformType
+    public var x86_64: URL
+    public var arm64: URL
+}
+
+public struct SwiftOrgSwiftlyRelease: Codable {
+    public var version: SwiftlyVersion
+    public var platforms: [SwiftOrgSwiftlyPlatform]
 }
 
 struct SwiftOrgPlatform: Codable {
     var name: String
-    var archs: [String]
+    var archs: [String]?
 
     /// platform is a mapping from the 'name' field of the swift.org platform object
     /// to swiftly's PlatformDefinition, if possible.
@@ -170,6 +183,13 @@ public struct SwiftlyHTTPClient {
         return try JSONDecoder().decode(type.self, from: response.buffer)
     }
 
+    /// Return the current Swiftly release using the swift.org API.
+    public func getSwiftlyRelease() async throws -> SwiftOrgSwiftlyRelease {
+        let url = "https://www.swift.org/api/v1/swiftly.json"
+        let swiftlyRelease: SwiftOrgSwiftlyRelease = try await self.getFromJSON(url: url, type: SwiftOrgSwiftlyRelease.self)
+        return swiftlyRelease
+    }
+
     /// Return an array of released Swift versions that match the given filter, up to the provided
     /// limit (default unlimited).
     public func getReleaseToolchains(
@@ -190,7 +210,7 @@ public struct SwiftlyHTTPClient {
             a!
         }
 
-        let url = "https://swift.org/api/v1/install/releases.json"
+        let url = "https://www.swift.org/api/v1/install/releases.json"
         let swiftOrgReleases: [SwiftOrgRelease] = try await self.getFromJSON(url: url, type: [SwiftOrgRelease].self)
 
         var swiftOrgFiltered: [ToolchainVersion.StableRelease] = try swiftOrgReleases.compactMap { swiftOrgRelease in
@@ -200,7 +220,7 @@ public struct SwiftlyHTTPClient {
                     return nil
                 }
 
-                guard swiftOrgPlatform.archs.contains(arch) else {
+                guard let archs = swiftOrgPlatform.archs, archs.contains(arch) else {
                     return nil
                 }
             }
@@ -277,7 +297,7 @@ public struct SwiftlyHTTPClient {
             platform.name
         }
 
-        let url = "https://swift.org/api/v1/install/dev/\(branch.name)/\(platformName).json"
+        let url = "https://www.swift.org/api/v1/install/dev/\(branch.name)/\(platformName).json"
 
         // For a particular branch and platform the snapshots are listed underneath their architecture
         let swiftOrgSnapshotArchs: SwiftOrgSnapshotList = try await self.getFromJSON(url: url, type: SwiftOrgSnapshotList.self)

Sources/SwiftlyCore/Platform.swift

@@ -57,6 +57,10 @@ public protocol Platform {
     /// After this completes, a user can “use” the toolchain.
     func install(from: URL, version: ToolchainVersion) throws
 
+    /// Extract swiftly from the provided downloaded archive and install
+    /// ourselves from that.
+    func extractSwiftlyAndInstall(from archive: URL) throws
+
     /// Uninstalls a toolchain associated with the given version.
     /// If this version is in use, the next latest version will be used afterwards.
     func uninstall(_ version: ToolchainVersion) throws

Tests/SwiftlyTests/HTTPClientTests.swift

@@ -7,7 +7,7 @@ final class HTTPClientTests: SwiftlyTests {
         // GIVEN: we have a swiftly http client
         // WHEN: we make get request for a particular type of JSON
         var releases: [SwiftOrgRelease] = try await SwiftlyCore.httpClient.getFromJSON(
-            url: "https://swift.org/api/v1/install/releases.json",
+            url: "https://www.swift.org/api/v1/install/releases.json",
             type: [SwiftOrgRelease].self,
             headers: [:]
         )
@@ -19,7 +19,7 @@ final class HTTPClientTests: SwiftlyTests {
         var exceptionThrown = false
         do {
             releases = try await SwiftlyCore.httpClient.getFromJSON(
-                url: "https://swift.org/api/v1/install/releases-invalid.json",
+                url: "https://www.swift.org/api/v1/install/releases-invalid.json",
                 type: [SwiftOrgRelease].self,
                 headers: [:]
             )