diff --git a/.github/workflows/native_toolchain_c.yaml b/.github/workflows/native_toolchain_c.yaml index 4a05ade9f2..0daeefb7d9 100644 --- a/.github/workflows/native_toolchain_c.yaml +++ b/.github/workflows/native_toolchain_c.yaml @@ -47,6 +47,9 @@ jobs: - name: Install native toolchains run: sudo apt-get update && sudo apt-get install gcc-i686-linux-gnu gcc-aarch64-linux-gnu gcc-arm-linux-gnueabihf gcc-riscv64-linux-gnu + - name: Install rust for pkgs/native_toolchain_c/test/clinker/rust_test.dart + uses: actions-rust-lang/setup-rust-toolchain@fb51252c7ba57d633bc668f941da052e410add48 + - run: git clone https://chromium.googlesource.com/chromium/tools/depot_tools.git - run: echo "$PWD/depot_tools" >> $GITHUB_PATH - run: mkdir dart-sdk diff --git a/.github/workflows/publish.yaml b/.github/workflows/publish.yaml index 9283cd38ac..1d30881b15 100644 --- a/.github/workflows/publish.yaml +++ b/.github/workflows/publish.yaml @@ -25,3 +25,4 @@ jobs: with: write-comments: false sdk: dev # use beta/stable after 3.3.0 + ignore-packages: pkgs/objective_c,pkgs/ffigen,pkgs/jni,pkgs/jnigen diff --git a/pkgs/hooks_runner/test_data/treeshaking_native_libs/hook/link.dart b/pkgs/hooks_runner/test_data/treeshaking_native_libs/hook/link.dart index f8b91b4cb9..0b080a6c00 100644 --- a/pkgs/hooks_runner/test_data/treeshaking_native_libs/hook/link.dart +++ b/pkgs/hooks_runner/test_data/treeshaking_native_libs/hook/link.dart @@ -12,7 +12,7 @@ void main(List arguments) async { final linker = CLinker.library( name: input.packageName, assetName: input.assets.code.single.id.split('/').skip(1).join('/'), - linkerOptions: LinkerOptions.treeshake(symbols: ['add']), + linkerOptions: LinkerOptions.treeshake(symbolsToKeep: ['add']), sources: [input.assets.code.single.file!.toFilePath()], ); await linker.run( diff --git a/pkgs/native_toolchain_c/CHANGELOG.md b/pkgs/native_toolchain_c/CHANGELOG.md index ac49fa52b9..c5327dc251 100644 --- a/pkgs/native_toolchain_c/CHANGELOG.md +++ b/pkgs/native_toolchain_c/CHANGELOG.md @@ -1,3 +1,7 @@ +## 0.17.0 + +* Fix treeshaking on mac. + ## 0.16.8 * Support building assets for packages which are not the input package. diff --git a/pkgs/native_toolchain_c/lib/src/cbuilder/linker_options.dart b/pkgs/native_toolchain_c/lib/src/cbuilder/linker_options.dart index e6aba29440..ac8f8a450d 100644 --- a/pkgs/native_toolchain_c/lib/src/cbuilder/linker_options.dart +++ b/pkgs/native_toolchain_c/lib/src/cbuilder/linker_options.dart @@ -25,45 +25,48 @@ class LinkerOptions { /// See also the `ld` man page at https://linux.die.net/man/1/ld. final bool gcSections; - /// The linker script to be passed via `--version-script`. - /// - /// See also the `ld` man page at https://linux.die.net/man/1/ld. - final Uri? linkerScript; + final LinkerScriptMode? _linkerScriptMode; /// Whether to strip debugging symbols from the binary. final bool stripDebug; /// The symbols to keep in the resulting binaries. - /// - /// If null all symbols will be kept. - final List? _symbolsToKeep; + final List _symbols; - final bool _generateLinkerScript; + final bool _keepAllSymbols; /// Create linking options manually for fine-grained control. + /// + /// If [symbolsToKeep] is null, all symbols will be kept. LinkerOptions.manual({ List? flags, bool? gcSections, - this.linkerScript, + Uri? linkerScript, this.stripDebug = true, Iterable? symbolsToKeep, }) : _linkerFlags = flags ?? [], gcSections = gcSections ?? true, - _symbolsToKeep = symbolsToKeep?.toList(growable: false), - _generateLinkerScript = false; + _symbols = symbolsToKeep?.toList(growable: false) ?? const [], + _keepAllSymbols = symbolsToKeep == null, + _linkerScriptMode = linkerScript != null + ? ManualLinkerScript(script: linkerScript) + : null; /// Create linking options to tree-shake symbols from the input files. /// - /// The [symbols] specify the symbols which should be kept. + /// The [symbolsToKeep] specify the symbols which should be kept. Passing + /// `null` implies that all symbols should be kept. LinkerOptions.treeshake({ Iterable? flags, - required Iterable? symbols, + required Iterable? symbolsToKeep, this.stripDebug = true, }) : _linkerFlags = flags?.toList(growable: false) ?? [], - _symbolsToKeep = symbols?.toList(growable: false), + _symbols = symbolsToKeep?.toList(growable: false) ?? const [], + _keepAllSymbols = symbolsToKeep == null, gcSections = true, - linkerScript = null, - _generateLinkerScript = symbols != null; + _linkerScriptMode = symbolsToKeep != null + ? GenerateLinkerScript() + : null; Iterable _toLinkerSyntax(Tool linker, Iterable flagList) { if (linker.isClangLike) { @@ -76,6 +79,19 @@ class LinkerOptions { } } +sealed class LinkerScriptMode {} + +final class GenerateLinkerScript extends LinkerScriptMode {} + +final class ManualLinkerScript extends LinkerScriptMode { + /// The linker script to be passed via `--version-script`. + /// + /// See also the `ld` man page at https://linux.die.net/man/1/ld. + final Uri script; + + ManualLinkerScript({required this.script}); +} + extension LinkerOptionsExt on LinkerOptions { /// Takes [sourceFiles] and turns it into flags for the compiler driver while /// considering the current [LinkerOptions]. @@ -99,8 +115,6 @@ extension LinkerOptionsExt on LinkerOptions { } } - bool get _includeAllSymbols => _symbolsToKeep == null; - Iterable _sourceFilesToFlagsForClangLike( Tool tool, Iterable sourceFiles, @@ -109,33 +123,37 @@ extension LinkerOptionsExt on LinkerOptions { switch (targetOS) { case OS.macOS || OS.iOS: return [ - if (!_includeAllSymbols) ...sourceFiles, + if (!_keepAllSymbols) ...sourceFiles, ..._toLinkerSyntax(tool, [ - if (_includeAllSymbols) ...sourceFiles.map((e) => '-force_load,$e'), + if (_keepAllSymbols) ...sourceFiles.map((e) => '-force_load,$e'), ..._linkerFlags, - ..._symbolsToKeep?.map((symbol) => '-u,_$symbol') ?? [], + ..._symbols.map((symbol) => '-u,_$symbol'), if (stripDebug) '-S', if (gcSections) '-dead_strip', + if (_linkerScriptMode is ManualLinkerScript) + '-exported_symbols_list,${_linkerScriptMode.script.toFilePath()}' + else if (_linkerScriptMode is GenerateLinkerScript) + '-exported_symbols_list,${_createMacSymbolList(_symbols)}', ]), ]; case OS.android || OS.linux: final wholeArchiveSandwich = sourceFiles.any((source) => source.endsWith('.a')) || - _includeAllSymbols; + _keepAllSymbols; return [ if (wholeArchiveSandwich) ..._toLinkerSyntax(tool, ['--whole-archive']), ...sourceFiles, ..._toLinkerSyntax(tool, [ ..._linkerFlags, - ..._symbolsToKeep?.map((symbol) => '-u,$symbol') ?? [], + ..._symbols.map((symbol) => '-u,$symbol'), if (stripDebug) '--strip-debug', if (gcSections) '--gc-sections', - if (linkerScript != null) - '--version-script=${linkerScript!.toFilePath()}' - else if (_generateLinkerScript && _symbolsToKeep != null) - '--version-script=${_createClangLikeLinkScript(_symbolsToKeep)}', + if (_linkerScriptMode is ManualLinkerScript) + '--version-script=${_linkerScriptMode.script.toFilePath()}' + else if (_linkerScriptMode is GenerateLinkerScript) + '--version-script=${_createClangLikeLinkScript(_symbols)}', if (wholeArchiveSandwich) '--no-whole-archive', ]), ]; @@ -152,21 +170,34 @@ extension LinkerOptionsExt on LinkerOptions { ) => [ ...sourceFiles, '/link', - if (_includeAllSymbols) ...sourceFiles.map((e) => '/WHOLEARCHIVE:$e'), + if (_keepAllSymbols) ...sourceFiles.map((e) => '/WHOLEARCHIVE:$e'), ..._linkerFlags, - ..._symbolsToKeep?.map( - (symbol) => - '/INCLUDE:${targetArch == Architecture.ia32 ? '_' : ''}$symbol', - ) ?? - [], - if (linkerScript != null) - '/DEF:${linkerScript!.toFilePath()}' - else if (_generateLinkerScript && _symbolsToKeep != null) - '/DEF:${_createClLinkScript(_symbolsToKeep)}', + ..._symbols.map( + (symbol) => + '/INCLUDE:${targetArch == Architecture.ia32 ? '_' : ''}$symbol', + ), + if (_linkerScriptMode is ManualLinkerScript) + '/DEF:${_linkerScriptMode.script.toFilePath()}' + else if (_linkerScriptMode is GenerateLinkerScript) + '/DEF:${_createClLinkScript(_symbols)}', if (stripDebug) '/PDBSTRIPPED', if (gcSections) '/OPT:REF', ]; + /// This creates a list of exported symbols. + /// + /// If this is not set, some symbols might be kept. This can be inspected + /// using `ld -why_live`, see https://www.unix.com/man_page/osx/1/ld/, where + /// the reason will show up as `global-dont-strip`. + /// This might possibly be a Rust only feature. + static String _createMacSymbolList(Iterable symbols) { + final tempDir = Directory.systemTemp.createTempSync(); + final symbolsFileUri = tempDir.uri.resolve('exported_symbols_list.txt'); + final symbolsFile = File.fromUri(symbolsFileUri)..createSync(); + symbolsFile.writeAsStringSync(symbols.map((e) => '_$e').join('\n')); + return symbolsFileUri.toFilePath(); + } + static String _createClangLikeLinkScript(Iterable symbols) { final tempDir = Directory.systemTemp.createTempSync(); final symbolsFileUri = tempDir.uri.resolve('symbols.lds'); diff --git a/pkgs/native_toolchain_c/pubspec.yaml b/pkgs/native_toolchain_c/pubspec.yaml index edb42c771f..54de4073e1 100644 --- a/pkgs/native_toolchain_c/pubspec.yaml +++ b/pkgs/native_toolchain_c/pubspec.yaml @@ -1,7 +1,7 @@ name: native_toolchain_c description: >- A library to invoke the native C compiler installed on the host machine. -version: 0.16.8 +version: 0.17.0 repository: https://github.com/dart-lang/native/tree/main/pkgs/native_toolchain_c topics: diff --git a/pkgs/native_toolchain_c/test/clinker/rust_test.dart b/pkgs/native_toolchain_c/test/clinker/rust_test.dart new file mode 100644 index 0000000000..2223106eb4 --- /dev/null +++ b/pkgs/native_toolchain_c/test/clinker/rust_test.dart @@ -0,0 +1,123 @@ +// Copyright (c) 2025, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +import 'dart:io'; + +import 'package:code_assets/code_assets.dart'; +import 'package:hooks/hooks.dart'; +import 'package:native_toolchain_c/native_toolchain_c.dart'; +import 'package:test/test.dart'; + +import '../helpers.dart'; + +Future main() async { + late final LinkInput linkInput; + late final Uri staticLib; + final linkOutputBuilder = LinkOutputBuilder(); + final targetArchitecture = Architecture.current; + final targetOS = OS.current; + late final bool rustToolchainInstalled; + setUpAll(() async { + final tempUri = await tempDirForTest(); + final tempUri2 = await tempDirForTest(); + staticLib = tempUri.resolve(targetOS.staticlibFileName('libtest')); + final processResult = await Process.run('rustc', [ + '--crate-type=staticlib', + 'test/clinker/testfiles/linker/test.rs', + '-o', + staticLib.toFilePath(), + ]); + rustToolchainInstalled = processResult.exitCode == 0; + if (rustToolchainInstalled) { + await File.fromUri( + staticLib, + ).copy(tempUri.resolve('libtest.a').toFilePath()); + } + final linkInputBuilder = LinkInputBuilder() + ..setupShared( + packageName: 'testpackage', + packageRoot: tempUri, + outputFile: tempUri.resolve('output.json'), + outputDirectoryShared: tempUri2, + ) + ..setupLink(assets: [], recordedUsesFile: null) + ..addExtension( + CodeAssetExtension( + targetOS: targetOS, + targetArchitecture: targetArchitecture, + linkModePreference: LinkModePreference.dynamic, + ), + ); + + linkInput = linkInputBuilder.build(); + }); + test('link rust binary with script treeshakes', () async { + if (!rustToolchainInstalled) { + return; + } + final treeshakeOption = LinkerOptions.treeshake( + symbolsToKeep: ['my_other_func'], + ); + final symbols = await _link( + staticLib, + treeshakeOption, + linkInput, + linkOutputBuilder, + targetArchitecture, + targetOS, + ); + final skipReason = symbols == null + ? 'tool to extract symbols unavailable' + : false; + expect(symbols, contains('my_other_func'), skip: skipReason); + expect(symbols, isNot(contains('my_func')), skip: skipReason); + }); + + test('link rust binary without script keeps symbols', () async { + if (!rustToolchainInstalled) { + return; + } + final manualOption = LinkerOptions.manual( + symbolsToKeep: ['my_other_func'], + stripDebug: true, + gcSections: true, + ); + final symbols = await _link( + staticLib, + manualOption, + linkInput, + linkOutputBuilder, + targetArchitecture, + targetOS, + ); + final skipReason = symbols == null + ? 'tool to extract symbols unavailable' + : false; + expect(symbols, contains('my_other_func'), skip: skipReason); + expect(symbols, contains('my_func'), skip: skipReason); + }); +} + +Future _link( + Uri staticLib, + LinkerOptions manualOption, + LinkInput linkInput, + LinkOutputBuilder linkOutputBuilder, + Architecture targetArchitecture, + OS targetOS, +) async { + await CLinker.library( + name: 'mylibname', + assetName: '', + sources: [staticLib.toFilePath()], + linkerOptions: manualOption, + ).run(input: linkInput, output: linkOutputBuilder, logger: logger); + + final linkOutput = linkOutputBuilder.build(); + final asset = linkOutput.assets.code.first; + + await expectMachineArchitecture(asset.file!, targetArchitecture, targetOS); + + return await readSymbols(asset, targetOS); +} diff --git a/pkgs/native_toolchain_c/test/clinker/testfiles/linker/symbols.txt b/pkgs/native_toolchain_c/test/clinker/testfiles/linker/symbols.txt new file mode 100644 index 0000000000..0a74d2890e --- /dev/null +++ b/pkgs/native_toolchain_c/test/clinker/testfiles/linker/symbols.txt @@ -0,0 +1 @@ +_my_other_func diff --git a/pkgs/native_toolchain_c/test/clinker/testfiles/linker/test.rs b/pkgs/native_toolchain_c/test/clinker/testfiles/linker/test.rs new file mode 100644 index 0000000000..ebc7a4a6d5 --- /dev/null +++ b/pkgs/native_toolchain_c/test/clinker/testfiles/linker/test.rs @@ -0,0 +1,9 @@ +#[no_mangle] +pub extern "C" fn my_func() -> u64 { + return 41; +} + +#[no_mangle] +pub extern "C" fn my_other_func() -> u64 { + return 42; +} diff --git a/pkgs/native_toolchain_c/test/clinker/treeshake_helper.dart b/pkgs/native_toolchain_c/test/clinker/treeshake_helper.dart index a90319638b..44c8c765dc 100644 --- a/pkgs/native_toolchain_c/test/clinker/treeshake_helper.dart +++ b/pkgs/native_toolchain_c/test/clinker/treeshake_helper.dart @@ -39,22 +39,28 @@ void runTreeshakeTests( symbolsToKeep: ['my_other_func'], stripDebug: true, gcSections: true, - linkerScript: targetOS == OS.windows - ? packageUri.resolve('test/clinker/testfiles/linker/symbols.def') - : packageUri.resolve('test/clinker/testfiles/linker/symbols.lds'), + linkerScript: switch (targetOS) { + OS.windows => packageUri.resolve( + 'test/clinker/testfiles/linker/symbols.def', + ), + OS.macOS || OS.iOS => packageUri.resolve( + 'test/clinker/testfiles/linker/symbols.txt', + ), + _ => packageUri.resolve('test/clinker/testfiles/linker/symbols.lds'), + }, ), ); CLinker linkerAuto(List sources) => CLinker.library( name: 'mylibname', assetName: '', sources: sources, - linkerOptions: LinkerOptions.treeshake(symbols: ['my_other_func']), + linkerOptions: LinkerOptions.treeshake(symbolsToKeep: ['my_other_func']), ); - CLinker linkerAutoEmpty(List sources) => CLinker.library( + CLinker linkerAutoKeepAll(List sources) => CLinker.library( name: 'mylibname', assetName: '', sources: sources, - linkerOptions: LinkerOptions.treeshake(symbols: null), + linkerOptions: LinkerOptions.treeshake(symbolsToKeep: null), ); late Map sizes; @@ -62,7 +68,7 @@ void runTreeshakeTests( for (final clinker in [ (name: 'manual', linker: linkerManual), (name: 'auto', linker: linkerAuto), - (name: 'autoEmpty', linker: linkerAutoEmpty), + (name: 'autoEmpty', linker: linkerAutoKeepAll), ]) { test('link test with CLinker ${clinker.name}', () async { final tempUri = await tempDirForTest(); @@ -129,7 +135,7 @@ void runTreeshakeTests( final skipReason = symbols == null ? 'tool to extract symbols unavailable' : false; - if (clinker.linker != linkerAutoEmpty) { + if (clinker.linker != linkerAutoKeepAll) { expect(symbols, contains('my_other_func'), skip: skipReason); expect(symbols, isNot(contains('my_func')), skip: skipReason); } else { diff --git a/pkgs/native_toolchain_c/test/clinker/windows_module_definition_helper.dart b/pkgs/native_toolchain_c/test/clinker/windows_module_definition_helper.dart index 073269ff23..3dd8e8f809 100644 --- a/pkgs/native_toolchain_c/test/clinker/windows_module_definition_helper.dart +++ b/pkgs/native_toolchain_c/test/clinker/windows_module_definition_helper.dart @@ -59,7 +59,7 @@ void runWindowsModuleDefinitionTests(List architectures) { name: name, assetName: '', linkerOptions: LinkerOptions.treeshake( - symbols: ['my_func', 'my_unexported_func'], + symbolsToKeep: ['my_func', 'my_unexported_func'], ), sources: [uri.toFilePath()], ).run(input: linkInput, output: linkOutput, logger: logger); diff --git a/tool/ci.dart b/tool/ci.dart index 874a42783b..16dd954c3a 100644 --- a/tool/ci.dart +++ b/tool/ci.dart @@ -83,7 +83,7 @@ void main(List arguments) async { 'pkgs/hooks/example/build/$exampleWithTest/', ), 'dart', - ['--enable-experiment=native-assets', 'test'], + ['test'], ); } @@ -92,19 +92,14 @@ void main(List arguments) async { 'pkgs/hooks/example/build/native_add_app/', ), 'dart', - ['--enable-experiment=native-assets', 'run'], + ['run'], ); _runProcess( workingDirectory: repositoryRoot.resolve( 'pkgs/hooks/example/build/native_add_app/', ), 'dart', - [ - '--enable-experiment=native-assets', - 'build', - 'cli', - 'bin/native_add_app.dart', - ], + ['build', 'cli', 'bin/native_add_app.dart'], ); _runProcess( repositoryRoot