From 0ccd65f5c7477073b3d47a27c4630fd9c40422d3 Mon Sep 17 00:00:00 2001 From: AdmiringWorm Date: Wed, 2 Sep 2020 00:26:28 +0200 Subject: [PATCH 1/2] (GH-628) Copy system libgit2 files to cake.git and gitversion directories On some linux distros it is not possible to use the embedded libgit2 binaries for these two softwares, and as such they need to either be linked or be copied to the necessary directories. As we are moving toward better support regarding .NET Core cake, we can no longer fully provide the support of patch DLLMaps as these do not work using .NET Core. As such this commit makes the necessary changes to copy the system libgit2 library to the existing libgit2*.so locactions underneeth the tools directory. --- Cake.Recipe/Content/gitversion.cake | 21 ++----------- Cake.Recipe/Content/parameters.cake | 48 +++++++++++++++++++++++++++-- 2 files changed, 48 insertions(+), 21 deletions(-) diff --git a/Cake.Recipe/Content/gitversion.cake b/Cake.Recipe/Content/gitversion.cake index a522b2d6..31720d24 100644 --- a/Cake.Recipe/Content/gitversion.cake +++ b/Cake.Recipe/Content/gitversion.cake @@ -51,7 +51,7 @@ public class BuildVersion if (BuildParameters.ShouldCalculateVersion) { - if (BuildParameters.BuildAgentOperatingSystem != PlatformFamily.Windows) { + if (BuildParameters.BuildAgentOperatingSystem == PlatformFamily.Linux) { PatchGitLibConfigFiles(context); } @@ -129,7 +129,7 @@ public class BuildVersion private static void PatchGitLibConfigFiles(ICakeContext context) { var configFiles = context.GetFiles("./tools/**/LibGit2Sharp.dll.config"); - var libgitPath = GetLibGit2Path(context); + var libgitPath = BuildParameters.GetLibGit2Path(context); if (string.IsNullOrEmpty(libgitPath)) { return; } foreach (var config in configFiles) { @@ -150,21 +150,4 @@ public class BuildVersion } } } - - private static string GetLibGit2Path(ICakeContext context) - { - var possiblePaths = new[] { - "/usr/lib*/libgit2.so*", - "/usr/lib/*/libgit2.so*" - }; - - foreach (var path in possiblePaths) { - var file = context.GetFiles(path).FirstOrDefault(); - if (file != null && !string.IsNullOrEmpty(file.ToString())) { - return file.ToString(); - } - } - - return null; - } } diff --git a/Cake.Recipe/Content/parameters.cake b/Cake.Recipe/Content/parameters.cake index fb4b6d81..f1ae4553 100644 --- a/Cake.Recipe/Content/parameters.cake +++ b/Cake.Recipe/Content/parameters.cake @@ -420,6 +420,13 @@ public static class BuildParameters PreferredBuildAgentOperatingSystem = preferredBuildAgentOperatingSystem; PreferredBuildProviderType = preferredBuildProviderType; + BuildAgentOperatingSystem = context.Environment.Platform.Family; + + if (BuildAgentOperatingSystem == PlatformFamily.Linux) + { + CopyLibGit2Binaries(context, "./tools/Addins/**/netstandard*/**/libgit2*.so"); + } + BuildProvider = GetBuildProvider(context, buildSystem); EmailRecipient = emailRecipient; @@ -553,8 +560,6 @@ public static class BuildParameters ); TreatWarningsAsErrors = treatWarningsAsErrors; - BuildAgentOperatingSystem = context.Environment.Platform.Family; - GitHub = GetGitHubCredentials(context); MicrosoftTeams = GetMicrosoftTeamsCredentials(context); Email = GetEmailCredentials(context); @@ -650,4 +655,43 @@ public static class BuildParameters } } } + + private static void CopyLibGit2Binaries(ICakeContext context, params string[] copyToGlobs) + { + var copyToFiles = context.GetFiles(copyToGlobs); + + string libgit2Path = GetLibGit2Path(context); + + if (String.IsNullOrEmpty(libgit2Path)) + { + return; + } + + foreach (var file in copyToFiles) + { + if (context.FileExists(file)) + { + context.DeleteFile(file); + } + context.Information("Copying system library from '{0}' to '{1}'", libgit2Path, file); + context.CopyFile(libgit2Path, file); + } + } + + internal static string GetLibGit2Path(ICakeContext context) + { + var possiblePaths = new[] { + "/usr/lib*/libgit2.so*", + "/usr/lib/*/libgit2.so*" + }; + + foreach (var path in possiblePaths) { + var file = context.GetFiles(path).FirstOrDefault(); + if (file != null && !string.IsNullOrEmpty(file.ToString())) { + return file.ToString(); + } + } + + return null; + } } From 5157a7af70eef8dd79dc78693087f2760a316d4e Mon Sep 17 00:00:00 2001 From: AdmiringWorm Date: Wed, 2 Sep 2020 12:40:46 +0200 Subject: [PATCH 2/2] (maint) Move logic for patching and copying libgit to its own file This includes the ability to report which Build Platform is used (Ubuntu, Arch Linux, Windows 10, etc). --- Cake.Recipe/Content/build.cake | 1 + Cake.Recipe/Content/buildPlatform.cake | 233 +++++++++++++++++++++++++ Cake.Recipe/Content/gitversion.cake | 29 +-- Cake.Recipe/Content/parameters.cake | 50 +----- 4 files changed, 240 insertions(+), 73 deletions(-) create mode 100644 Cake.Recipe/Content/buildPlatform.cake diff --git a/Cake.Recipe/Content/build.cake b/Cake.Recipe/Content/build.cake index 482b9115..7e57f6a1 100644 --- a/Cake.Recipe/Content/build.cake +++ b/Cake.Recipe/Content/build.cake @@ -106,6 +106,7 @@ Teardown((context, buildVersion) => BuildParameters.Tasks.ShowInfoTask = Task("Show-Info") .Does(() => { + Information("Build Platform: {0}", BuildParameters.Platform); Information("Target: {0}", BuildParameters.Target); Information("Configuration: {0}", BuildParameters.Configuration); Information("PrepareLocalRelease: {0}", BuildParameters.PrepareLocalRelease); diff --git a/Cake.Recipe/Content/buildPlatform.cake b/Cake.Recipe/Content/buildPlatform.cake new file mode 100644 index 00000000..8d3df56b --- /dev/null +++ b/Cake.Recipe/Content/buildPlatform.cake @@ -0,0 +1,233 @@ +using System.Runtime.InteropServices; + +public class BuildPlatform +{ + protected ICakeContext context; + + public BuildPlatform(ICakeContext context) + { + this.context = context; + SystemVersion = System.Environment.OSVersion.Version.ToString(); + SystemName = RuntimeInformation.OSDescription.Trim(); + OperatingSystem = context.Environment.Platform.Family; + SystemArchitecture = RuntimeInformation.OSArchitecture.ToString(); + } + + public PlatformFamily OperatingSystem { get; private set; } + public string SystemName { get; private set; } + public string SystemDistro { get; protected set; } = ""; + public string SystemDistroId { get; protected set; } = ""; + public string SystemVersion { get; protected set; } + public string SystemArchitecture { get; private set; } + + public static BuildPlatform Create(ICakeContext context) + { + BuildPlatform result; + if (context.Environment.Platform.Family != PlatformFamily.Linux) + { + result = new BuildPlatform(context); + } + else + { + result = new LinuxBuildPlatform(context); + } + + return result; + } + + public virtual void CopyLibGit2Binaries(params string[] globbingPatterns) + { + } + + public virtual void PatchGitLib2ConfigFiles(string globbingPattern = "**/LibGit2Sharp.dll.config") + { + } + + public override string ToString() + { + if (!string.IsNullOrEmpty(SystemDistro)) + { + return string.Format("{0} {1} ({2})", SystemDistro, SystemVersion, SystemArchitecture); + } + else if (OperatingSystem == PlatformFamily.Windows) + { + // Windows have the system version in the SystemName + return string.Format("{0} ({1})", SystemName, SystemArchitecture); + } + else + { + return string.Format("{0} {1} ({2})", SystemName, SystemVersion, SystemArchitecture); + } + } +} + +public class LinuxBuildPlatform : BuildPlatform +{ + public LinuxBuildPlatform(ICakeContext context) + : base(context) + { + SetSystemInformation(); + } + + public override void CopyLibGit2Binaries(params string[] globbingPatterns) + { + var addinPath = GetAddinPath(context); + var copyToFiles = context.GetFiles(globbingPatterns.Select(g => addinPath.CombineWithFilePath(g).ToString()).ToArray()); + + string libgit2Path = GetLibGit2Path(context); + + if (String.IsNullOrEmpty(libgit2Path)) + { + return; + } + + foreach (var file in copyToFiles) + { + if (context.FileExists(file)) + { + context.DeleteFile(file); + } + context.Verbose("Copying system library from '{0}' to '{1}'", libgit2Path, file); + context.CopyFile(libgit2Path, file); + } + } + + public override void PatchGitLib2ConfigFiles(string globbingPattern = "**/LibGit2Sharp.dll.config") + { + var toolPath = context.Configuration.GetToolPath(".", context.Environment); + var configFiles = context.GetFiles(toolPath.CombineWithFilePath(globbingPattern).ToString()); + + var libgit2Path = GetLibGit2Path(context); + + if (String.IsNullOrEmpty(libgit2Path)) + { + return; + } + + // This is a workaround for Ubuntu 18.04 especially. + // as gitversion will fail if we add a dllmap to the system + // libgit2 library. + var relativeExpected = "lib/"; + if (!string.IsNullOrEmpty(SystemDistroId)) + { + if (!string.IsNullOrEmpty(SystemVersion) + && Version.TryParse(SystemVersion, out _)) + { + relativeExpected += string.Format("{0}.{1}-{2}", + SystemDistroId, + SystemVersion, + SystemArchitecture); + } + else + { + relativeExpected += string.Format("{0}-{1}", + SystemDistroId, + SystemArchitecture); + } + } + + foreach (var config in configFiles) + { + if (!relativeExpected.Equals("lib/")) + { + var parent = config.GetDirectory(); + var expectedPath = parent.Combine(relativeExpected); + if (context.DirectoryExists(expectedPath)) + { + continue; + } + } + + var xml = System.Xml.Linq.XDocument.Load(config.ToString()); + + if (xml.Element("configuration").Elements("dllmap").All(e => e.Attribute("target").Value != libgit2Path)) + { + var dllName = xml.Element("configuration").Elements("dllmap").First(e => e.Attribute("os").Value == "linux").Attribute("dll").Value; + xml.Element("configuration") + .Add(new System.Xml.Linq.XElement("dllmap", + new System.Xml.Linq.XAttribute("os", "linux"), + new System.Xml.Linq.XAttribute("dll", dllName), + new System.Xml.Linq.XAttribute("target", libgit2Path))); + + context.Information($"Patching '{config}' to use fallback system path on Linux..."); + xml.Save(config.ToString()); + } + } + } + + // Shamelessly stolen from + private static DirectoryPath GetAddinPath(ICakeContext context) + { + var addinPath = context.Configuration.GetValue("Paths_Addins"); + if (!string.IsNullOrEmpty(addinPath)) + { + return new DirectoryPath(addinPath).MakeAbsolute(context.Environment); + } + + var toolPath = context.Configuration.GetToolPath(".", context.Environment); + return toolPath.Combine("Addins").Collapse(); + } + + private static string GetLibGit2Path(ICakeContext context) + { + var possiblePaths = new[] { + "/usr/lib*/libgit2.so*", + "/usr/lib/*/libgit2.so*" + }; + + foreach (var path in possiblePaths) { + var file = context.GetFiles(path).FirstOrDefault(); + if (file != null && !string.IsNullOrEmpty(file.ToString())) { + return file.ToString(); + } + } + + return null; + } + + private void SetSystemInformation() + { + string file; + if (context.FileExists("/etc/lsb-release")) + { + file = "/etc/lsb-release"; + } + else if (context.FileExists("/etc/os-release")) + { + file = "/etc/os-release"; + } + else + { + return; + } + + var lines = System.IO.File.ReadAllLines(file); + var details = lines.Where(l => !string.IsNullOrEmpty(l)) + .Select(l => l.Split('=')) + .Where(s => s.Length == 2) + .ToDictionary(s => s[0].ToUpperInvariant(), s => s[1]); + + if (details.ContainsKey("DISTRIB_ID")) + { + SystemDistroId = SystemDistro = details["DISTRIB_ID"]; + } + else if(details.ContainsKey("ID")) + { + SystemDistroId = details["ID"]; + } + + if (details.ContainsKey("NAME")) + { + SystemDistro = details["NAME"]; + } + + if (details.ContainsKey("DISTRIB_RELEASE")) + { + SystemVersion = details["DISTRIB_RELEASE"]; + } + else if (details.ContainsKey("BUILD_ID")) + { + SystemVersion = details["BUILD_ID"]; + } + } +} \ No newline at end of file diff --git a/Cake.Recipe/Content/gitversion.cake b/Cake.Recipe/Content/gitversion.cake index 31720d24..d5a59b2c 100644 --- a/Cake.Recipe/Content/gitversion.cake +++ b/Cake.Recipe/Content/gitversion.cake @@ -51,9 +51,7 @@ public class BuildVersion if (BuildParameters.ShouldCalculateVersion) { - if (BuildParameters.BuildAgentOperatingSystem == PlatformFamily.Linux) { - PatchGitLibConfigFiles(context); - } + BuildParameters.Platform.PatchGitLib2ConfigFiles("**/GitVersion*/**/LibGit2Sharp.dll.config"); context.Information("Calculating Semantic Version..."); if (!BuildParameters.IsLocalBuild || BuildParameters.IsPublishBuild || BuildParameters.IsReleaseBuild || BuildParameters.PrepareLocalRelease) @@ -125,29 +123,4 @@ public class BuildVersion AssemblySemVer = assemblySemVer?.ToLowerInvariant() }; } - - private static void PatchGitLibConfigFiles(ICakeContext context) - { - var configFiles = context.GetFiles("./tools/**/LibGit2Sharp.dll.config"); - var libgitPath = BuildParameters.GetLibGit2Path(context); - if (string.IsNullOrEmpty(libgitPath)) { return; } - - foreach (var config in configFiles) { - var xml = System.Xml.Linq.XDocument.Load(config.ToString()); - - if (xml.Element("configuration").Elements("dllmap") - .All(e => e.Attribute("target").Value != libgitPath)) { - - var dllName = xml.Element("configuration").Elements("dllmap").First(e => e.Attribute("os").Value == "linux").Attribute("dll").Value; - xml.Element("configuration") - .Add(new System.Xml.Linq.XElement("dllmap", - new System.Xml.Linq.XAttribute("os", "linux"), - new System.Xml.Linq.XAttribute("dll", dllName), - new System.Xml.Linq.XAttribute("target", libgitPath))); - - context.Information($"Patching '{config}' to use fallback system path on Linux..."); - xml.Save(config.ToString()); - } - } - } } diff --git a/Cake.Recipe/Content/parameters.cake b/Cake.Recipe/Content/parameters.cake index f1ae4553..d9471f30 100644 --- a/Cake.Recipe/Content/parameters.cake +++ b/Cake.Recipe/Content/parameters.cake @@ -18,7 +18,8 @@ public static class BuildParameters public static string Configuration { get; private set; } public static Cake.Core.Configuration.ICakeConfiguration CakeConfiguration { get; private set; } public static bool IsLocalBuild { get; private set; } - public static PlatformFamily BuildAgentOperatingSystem { get; private set; } + public static PlatformFamily BuildAgentOperatingSystem => Platform.OperatingSystem; + public static BuildPlatform Platform { get; private set; } public static bool IsRunningOnAppVeyor { get; private set; } public static bool IsRunningOnTravisCI { get; private set; } public static bool IsPullRequest { get; private set; } @@ -420,12 +421,10 @@ public static class BuildParameters PreferredBuildAgentOperatingSystem = preferredBuildAgentOperatingSystem; PreferredBuildProviderType = preferredBuildProviderType; - BuildAgentOperatingSystem = context.Environment.Platform.Family; + Platform = BuildPlatform.Create(context); - if (BuildAgentOperatingSystem == PlatformFamily.Linux) - { - CopyLibGit2Binaries(context, "./tools/Addins/**/netstandard*/**/libgit2*.so"); - } + Platform.CopyLibGit2Binaries("**/netstandard*/**/libgit2*.so", "**/linux-x64/libgit2*.so"); + Platform.PatchGitLib2ConfigFiles(); BuildProvider = GetBuildProvider(context, buildSystem); @@ -655,43 +654,4 @@ public static class BuildParameters } } } - - private static void CopyLibGit2Binaries(ICakeContext context, params string[] copyToGlobs) - { - var copyToFiles = context.GetFiles(copyToGlobs); - - string libgit2Path = GetLibGit2Path(context); - - if (String.IsNullOrEmpty(libgit2Path)) - { - return; - } - - foreach (var file in copyToFiles) - { - if (context.FileExists(file)) - { - context.DeleteFile(file); - } - context.Information("Copying system library from '{0}' to '{1}'", libgit2Path, file); - context.CopyFile(libgit2Path, file); - } - } - - internal static string GetLibGit2Path(ICakeContext context) - { - var possiblePaths = new[] { - "/usr/lib*/libgit2.so*", - "/usr/lib/*/libgit2.so*" - }; - - foreach (var path in possiblePaths) { - var file = context.GetFiles(path).FirstOrDefault(); - if (file != null && !string.IsNullOrEmpty(file.ToString())) { - return file.ToString(); - } - } - - return null; - } }