Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
16 commits
Select commit Hold shift + click to select a range
883e638
Add new Linux platforms: Ubuntu 2404,2310, Debian 12, Fedora 39
cmcgee1024 Oct 19, 2024
0cc129c
Add the new entries to the --platform option
cmcgee1024 Oct 19, 2024
afbd797
Pull list of linux platforms out into a var.
cmcgee1024 Oct 19, 2024
d301ef3
Remove all of the platform def environment variables to force auto-de…
cmcgee1024 Oct 19, 2024
393f9d5
Remove last remnants of the special platform environment variables fr…
cmcgee1024 Oct 20, 2024
6fc24f0
Remove mention of CentOS from the README.md since it is not particula…
cmcgee1024 Oct 20, 2024
be65db1
Add system package dependency list for Ubuntu 23.10
cmcgee1024 Oct 23, 2024
4e7ca7c
Merge branch 'main' of github.com:cmcgee1024/swiftly into add_new_lin…
cmcgee1024 Oct 25, 2024
ba9bfb3
Remove Ubuntu 23.10 as it is EOL
cmcgee1024 Nov 1, 2024
2620466
Merge branch 'main' of github.com:cmcgee1024/swiftly into add_new_lin…
cmcgee1024 Nov 7, 2024
b052c2b
Add new Linux distributions to the GitHub workflow.
cmcgee1024 Nov 7, 2024
e5c1da0
Add appropriate libstdc++ to images that are missing them, and gnupg
cmcgee1024 Nov 7, 2024
4896e64
Re-add the list of linux os versions
cmcgee1024 Nov 7, 2024
3ea709f
Bump the tsc dependency to fix compile errors on Linux with newer glibc
cmcgee1024 Nov 8, 2024
c202b02
Add libstdc++-static package to fedora39 to fix a linker error
cmcgee1024 Nov 8, 2024
4ff2eb2
Skip tests that rely on snapshots for platforms where snapshots are n…
cmcgee1024 Nov 8, 2024
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 5 additions & 6 deletions .github/workflows/pull_request.yml
Original file line number Diff line number Diff line change
Expand Up @@ -21,12 +21,11 @@ jobs:
name: Test
uses: swiftlang/github-workflows/.github/workflows/swift_package_test.yml@main
with:
# linux_os_versions: "[\"jammy\", \"noble\", \"focal\", \"amazonlinux2\", \"rhel-ubi9\", \"bookworm\", \"fedora39\"]"
# Not working: noble (compile error in TSC FileSystem), bookworm (missing memory.h), fedora39 (missing memory.h)
linux_os_versions: "[\"jammy\", \"focal\", \"rhel-ubi9\"]"
# Amazon Linux 2 won't work with GH infrastructure
linux_os_versions: "[\"jammy\", \"focal\", \"rhel-ubi9\", \"noble\", \"bookworm\", \"fedora39\"]"
# We only care about the current stable release, because that's where we make our swiftly releases
linux_exclude_swift_versions: "[{\"swift_version\": \"nightly-main\"},{\"swift_version\": \"nightly-6.0\"},{\"swift_version\": \"5.8\"},{\"swift_version\": \"5.9\"},{\"swift_version\": \"5.10\"}]"
linux_pre_build_command: ((apt-get update && apt-get -y install curl make) || ((curl --help || yum -y install curl) && yum -y install make)) && ./scripts/install-libarchive.sh
linux_pre_build_command: ./scripts/prep-gh-action.sh && ./scripts/install-libarchive.sh
enable_windows_checks: false

releasebuild:
Expand All @@ -36,7 +35,7 @@ jobs:
linux_os_versions: "[\"rhel-ubi9\"]"
# We only care about the current stable release, because that's where we make our swiftly releases
linux_exclude_swift_versions: "[{\"swift_version\": \"nightly-main\"},{\"swift_version\": \"nightly-6.0\"},{\"swift_version\": \"5.8\"},{\"swift_version\": \"5.9\"},{\"swift_version\": \"5.10\"}]"
linux_pre_build_command: echo ""
linux_pre_build_command: ./scripts/prep-gh-action.sh
linux_build_command: swift run build-swiftly-release --skip 0.4.0
enable_windows_checks: false

Expand All @@ -47,6 +46,6 @@ jobs:
# We only need to run this with one swift release and on one of the linux distributions
linux_os_versions: "[\"jammy\"]"
linux_exclude_swift_versions: "[{\"swift_version\": \"nightly-main\"},{\"swift_version\": \"nightly-6.0\"},{\"swift_version\": \"5.8\"},{\"swift_version\": \"5.9\"},{\"swift_version\": \"5.10\"}]"
linux_pre_build_command: echo ""
linux_pre_build_command: ./scripts/prep-gh-action.sh
linux_build_command: swift run swiftformat --lint --dryrun .
enable_windows_checks: false
6 changes: 3 additions & 3 deletions Package.resolved

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 1 addition & 1 deletion Package.swift
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ let package = Package(
.package(url: "https://github.com/apple/swift-argument-parser", from: "1.3.0"),
.package(url: "https://github.com/swift-server/async-http-client", from: "1.21.2"),
.package(url: "https://github.com/apple/swift-nio.git", from: "2.64.0"),
.package(url: "https://github.com/apple/swift-tools-support-core.git", from: "0.6.1"),
.package(url: "https://github.com/apple/swift-tools-support-core.git", from: "0.7.1"),
.package(url: "https://github.com/apple/swift-docc-plugin", from: "1.3.0"),
// This dependency provides the correct version of the formatter so that you can run `swift run swiftformat Package.swift Plugins/ Sources/ Tests/`
.package(url: "https://github.com/nicklockwood/SwiftFormat", exact: "0.49.18"),
Expand Down
1 change: 0 additions & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,6 @@ Target: x86_64-unknown-linux-gnu
## Platform support

- Linux-based platforms listed on https://swift.org/download
- CentOS 7 will not be supported due to some dependencies of swiftly not supporting it, however.
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Is it worth noting here that 23.10 isn't supported for 5.10 releases? technically that is listed on swift.org/download? Just thinking out loud, it doesn't matter much


Right now, swiftly is in early stages of development and is supported on Linux and macOS. For more detailed information about swiftly's intended features and implementation, check out the [design document](DESIGN.md).

Expand Down
192 changes: 105 additions & 87 deletions Sources/LinuxPlatform/Linux.swift
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,17 @@ var swiftGPGKeysRefreshed = false
/// This implementation can be reused for any supported Linux platform.
/// TODO: replace dummy implementations
public struct Linux: Platform {
let linuxPlatforms = [
PlatformDefinition.ubuntu2404,
PlatformDefinition.ubuntu2204,
PlatformDefinition.ubuntu2004,
PlatformDefinition.ubuntu1804,
PlatformDefinition.fedora39,
PlatformDefinition.rhel9,
PlatformDefinition.amazonlinux2,
PlatformDefinition.debian12,
]

public init() {}

public var appDataDirectory: URL {
Expand Down Expand Up @@ -125,6 +136,26 @@ public struct Linux: Platform {
"tzdata",
"zlib1g-dev",
]
case "ubuntu2404":
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think we're missing 2310 from this list

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I couldn't find a list of packages for ubuntu 23.10 from the Dockerfiles. If someone knows where the docker file is, I can add it here.

This is where I usually look for them: https://github.com/swiftlang/swift-docker/tree/main/6.0/ubuntu

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I manually pieced together the list by looking at the differences between ubuntu 22.04, and 24.04. Tested it out in docker and it looks to be working fine. There's only the swift 5.10.1 release available for that one though, so I wonder how useful and used this version of ubuntu will be.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I have removed 23.10 from the list because we can't bring up infrastructure for a platform that is EOL.

[
"binutils",
"git",
"unzip",
"gnupg2",
"libc6-dev",
"libcurl4-openssl-dev",
"libedit2",
"libgcc-13-dev",
"libpython3-dev",
"libsqlite3-0",
"libstdc++-13-dev",
"libxml2-dev",
"libncurses-dev",
"libz3-dev",
"pkg-config",
"tzdata",
"zlib1g-dev",
]
case "amazonlinux2":
[
"binutils",
Expand Down Expand Up @@ -158,6 +189,39 @@ public struct Linux: Platform {
"unzip",
"zip",
]
case "fedora39":
[
"binutils",
"gcc",
"git",
"unzip",
"libcurl-devel",
"libedit-devel",
"libicu-devel",
"sqlite-devel",
"libuuid-devel",
"libxml2-devel",
"python3-devel",
"libstdc++-devel",
"libstdc++-static",
]
case "debian12":
[
"binutils-gold",
"libicu-dev",
"libcurl4-openssl-dev",
"libedit-dev",
"libsqlite3-dev",
"libncurses-dev",
"libpython3-dev",
"libxml2-dev",
"pkg-config",
"uuid-dev",
"tzdata",
"git",
"gcc",
"libstdc++-12-dev",
]
default:
[]
}
Expand All @@ -169,10 +233,16 @@ public struct Linux: Platform {
"apt-get"
case "ubuntu2204":
"apt-get"
case "ubuntu2404":
"apt-get"
case "amazonlinux2":
"yum"
case "ubi9":
"yum"
case "fedora39":
"yum"
case "debian12":
"apt-get"
default:
nil
}
Expand All @@ -196,7 +266,7 @@ public struct Linux: Platform {
// Import the latest swift keys, but only once per session, which will help with the performance in tests
if !swiftGPGKeysRefreshed {
let tmpFile = self.getTempFilePath()
FileManager.default.createFile(atPath: tmpFile.path, contents: nil, attributes: [.posixPermissions: 0o600])
let _ = FileManager.default.createFile(atPath: tmpFile.path, contents: nil, attributes: [.posixPermissions: 0o600])
defer {
try? FileManager.default.removeItem(at: tmpFile)
}
Expand Down Expand Up @@ -407,7 +477,7 @@ public struct Linux: Platform {
public func verifySignature(httpClient: SwiftlyHTTPClient, archiveDownloadURL: URL, archive: URL) async throws {
SwiftlyCore.print("Downloading toolchain signature...")
let sigFile = self.getTempFilePath()
FileManager.default.createFile(atPath: sigFile.path, contents: nil)
let _ = FileManager.default.createFile(atPath: sigFile.path, contents: nil)
defer {
try? FileManager.default.removeItem(at: sigFile)
}
Expand All @@ -425,59 +495,43 @@ public struct Linux: Platform {
}
}

private func manualSelectPlatform(_ platformPretty: String?) -> PlatformDefinition {
private func manualSelectPlatform(_ platformPretty: String?) async -> PlatformDefinition {
if let platformPretty = platformPretty {
print("\(platformPretty) is not an officially supported platform, but the toolchains for another platform may still work on it.")
} else {
print("This platform could not be detected, but a toolchain for one of the supported platforms may work on it.")
}

let selections = self.linuxPlatforms.enumerated().map { "\($0 + 1)) \($1.namePretty)" }.joined(separator: "\n")

print("""
Please select the platform to use for toolchain downloads:

0) Cancel
1) Ubuntu 22.04
2) Ubuntu 20.04
3) Ubuntu 18.04
4) RHEL 9
5) Amazon Linux 2
\(selections)
""")

let choice = SwiftlyCore.readLine(prompt: "> ") ?? "0"
let choice = SwiftlyCore.readLine(prompt: "Pick one of the available selections [0-\(self.linuxPlatforms.count)] ") ?? "0"

switch choice {
case "1":
return PlatformDefinition.ubuntu2204
case "2":
return PlatformDefinition.ubuntu2004
case "3":
return PlatformDefinition.ubuntu1804
case "4":
return PlatformDefinition.rhel9
case "5":
return PlatformDefinition.amazonlinux2
default:
guard let choiceNum = Int(choice) else {
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think right now if you were to run this and enter 0 for cancel, it would explode, it would go on to run
return self.linuxPlatforms[choiceNum - 1] and you'd get an index exception

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think that this guard will guard against non-integer input, such as "foo", or "bar". The guard below should check for 0, or something larger than the linuxPlatforms count and cancel the installation. The final array subscript at the return statement should be safely in range.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

oh I see, the guard clause tripped me up, Makes sense

fatalError("Installation canceled")
}

guard choiceNum > 0 && choiceNum <= self.linuxPlatforms.count else {
fatalError("Installation canceled")
}

return self.linuxPlatforms[choiceNum - 1]
}

public func detectPlatform(disableConfirmation: Bool, platform: String?) async throws -> PlatformDefinition {
// We've been given a hint to use
if let platform = platform {
switch platform {
case "ubuntu22.04":
return PlatformDefinition.ubuntu2204
case "ubuntu20.04":
return PlatformDefinition.ubuntu2004
case "ubuntu18.04":
return PlatformDefinition.ubuntu1804
case "amazonlinux2":
return PlatformDefinition.amazonlinux2
case "rhel9":
return PlatformDefinition.rhel9
default:
fatalError("Unrecognized platform \(platform)")
if let platform {
guard let pd = linuxPlatforms.first(where: { $0.nameFull == platform }) else {
fatalError("Unrecognized platform \(platform). Recognized values: \(self.linuxPlatforms.map(\.nameFull).joined(separator: ", ")).")
}

return pd
}

let osReleaseFiles = ["/etc/os-release", "/usr/lib/os-release"]
Expand All @@ -498,98 +552,62 @@ public struct Linux: Platform {
} else {
print(message)
}
return self.manualSelectPlatform(platformPretty)
}

let data = FileManager.default.contents(atPath: releaseFile)
guard let data = data else {
let message = "Unable to read OS release information from file \(releaseFile)"
if disableConfirmation {
throw Error(message: message)
} else {
print(message)
}
return self.manualSelectPlatform(platformPretty)
return await self.manualSelectPlatform(platformPretty)
}

guard let releaseInfo = String(data: data, encoding: .utf8) else {
let message = "Unable to read OS release information from file \(releaseFile)"
if disableConfirmation {
throw Error(message: message)
} else {
print(message)
}
return self.manualSelectPlatform(platformPretty)
}
let releaseInfo = try String(contentsOfFile: releaseFile, encoding: .utf8)
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

should we still wrap this in an equivalent try/catch which calls to self.manualSelectPlatform if the info can't be read correctly?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think that we want to heavily emphasize the platform auto-detection because it will make the user's life a ton easier. If there's some kind of problem reading the /etc/os-release or /usr/lib/os-release to the point where we can't even read the bytes into a String with the expected UTF-8 encoding the user should probably be made aware of this corruption.

From the linux manual it says that "[a]ll strings should be in UTF-8 encoding, and non-printable characters should not be used."

https://man7.org/linux/man-pages/man5/os-release.5.html


var id: String?
var idlike: String?
var versionID: String?
var ubuntuCodeName: String?
for info in releaseInfo.split(separator: "\n").map(String.init) {
if info.hasPrefix("ID=") {
id = String(info.dropFirst("ID=".count)).replacingOccurrences(of: "\"", with: "")
} else if info.hasPrefix("ID_LIKE=") {
idlike = String(info.dropFirst("ID_LIKE=".count)).replacingOccurrences(of: "\"", with: "")
} else if info.hasPrefix("VERSION_ID=") {
versionID = String(info.dropFirst("VERSION_ID=".count)).replacingOccurrences(of: "\"", with: "")
} else if info.hasPrefix("UBUNTU_CODENAME=") {
ubuntuCodeName = String(info.dropFirst("UBUNTU_CODENAME=".count)).replacingOccurrences(of: "\"", with: "")
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Do we still need to account for stings which start with "UBUNTU_CODENAME" here?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We're reading this file line-by-line and we don't have any need for the ubuntu codename anymore. I think it's safe to just skip ubuntu codename lines.

versionID = String(info.dropFirst("VERSION_ID=".count)).replacingOccurrences(of: "\"", with: "").replacingOccurrences(of: ".", with: "")
} else if info.hasPrefix("PRETTY_NAME=") {
platformPretty = String(info.dropFirst("PRETTY_NAME=".count)).replacingOccurrences(of: "\"", with: "")
}
}

guard let id = id, let idlike = idlike else {
guard let id, let versionID else {
let message = "Unable to find release information from file \(releaseFile)"
if disableConfirmation {
throw Error(message: message)
} else {
print(message)
}
return self.manualSelectPlatform(platformPretty)
return await self.manualSelectPlatform(platformPretty)
}

if (id + idlike).contains("amzn") {
guard let versionID = versionID, versionID == "2" else {
if (id + (idlike ?? "")).contains("amzn") {
guard versionID == "2" else {
let message = "Unsupported version of Amazon Linux"
if disableConfirmation {
throw Error(message: message)
} else {
print(message)
}
return self.manualSelectPlatform(platformPretty)
return await self.manualSelectPlatform(platformPretty)
}

return PlatformDefinition(name: "amazonlinux2", nameFull: "amazonlinux2", namePretty: "Amazon Linux 2")
} else if (id + idlike).contains("ubuntu") {
if ubuntuCodeName == "jammy" {
return PlatformDefinition(name: "ubuntu2204", nameFull: "ubuntu22.04", namePretty: "Ubuntu 22.04")
} else if ubuntuCodeName == "focal" {
return PlatformDefinition(name: "ubuntu2004", nameFull: "ubuntu20.04", namePretty: "Ubuntu 20.04")
} else if ubuntuCodeName == "bionic" {
return PlatformDefinition(name: "ubuntu1804", nameFull: "ubuntu18.04", namePretty: "Ubuntu 18.04")
} else {
let message = "Unsupported version of Ubuntu Linux"
if disableConfirmation {
throw Error(message: message)
} else {
print(message)
}
return self.manualSelectPlatform(platformPretty)
}
} else if (id + idlike).contains("rhel") {
guard let versionID = versionID, versionID.hasPrefix("9") else {
return PlatformDefinition.amazonlinux2
} else if (id + (idlike ?? "")).contains("rhel") {
guard versionID.hasPrefix("9") else {
let message = "Unsupported version of RHEL"
if disableConfirmation {
throw Error(message: message)
} else {
print(message)
}
return self.manualSelectPlatform(platformPretty)
return await self.manualSelectPlatform(platformPretty)
}

return PlatformDefinition(name: "ubi9", nameFull: "ubi9", namePretty: "RHEL 9")
return PlatformDefinition.rhel9
} else if let pd = [PlatformDefinition.ubuntu1804, .ubuntu2004, .ubuntu2204, .ubuntu2404, .debian12, .fedora39].first(where: { $0.name == id + versionID }) {
return pd
}

let message = "Unsupported Linux platform"
Expand All @@ -598,7 +616,7 @@ public struct Linux: Platform {
} else {
print(message)
}
return self.manualSelectPlatform(platformPretty)
return await self.manualSelectPlatform(platformPretty)
}

public func getShell() async throws -> String {
Expand Down
2 changes: 0 additions & 2 deletions Sources/SwiftlyCore/HTTPClient.swift
Original file line number Diff line number Diff line change
Expand Up @@ -72,8 +72,6 @@ struct SwiftOrgPlatform: Codable {
PlatformDefinition.ubuntu2204
case "Red Hat Universal Base Image 9":
PlatformDefinition.rhel9
case "Ubuntu 23.10":
PlatformDefinition(name: "ubuntu2310", nameFull: "ubuntu23.10", namePretty: "Ubuntu 23.10")
case "Ubuntu 24.04":
PlatformDefinition(name: "ubuntu2404", nameFull: "ubuntu24.04", namePretty: "Ubuntu 24.04")
case "Debian 12":
Expand Down
Loading