diff --git a/go.mod b/go.mod index 16affee914..8a7b84d4a2 100644 --- a/go.mod +++ b/go.mod @@ -12,6 +12,7 @@ require ( github.com/goplus/llvm v0.8.5 github.com/goplus/mod v0.17.1 github.com/qiniu/x v1.15.1 + gopkg.in/yaml.v3 v3.0.1 golang.org/x/tools v0.36.0 ) diff --git a/internal/build/build.go b/internal/build/build.go index fc67e5c579..6460debfd7 100644 --- a/internal/build/build.go +++ b/internal/build/build.go @@ -32,6 +32,7 @@ import ( "runtime" "slices" "strings" + "sync/atomic" "unsafe" "golang.org/x/tools/go/ssa" @@ -322,9 +323,7 @@ func Do(args []string, conf *Config) ([]Package, error) { altPkgs, err := packages.LoadEx(dedup, sizes, cfg, altPkgPaths...) check(err) - noRt := 1 prog.SetRuntime(func() *types.Package { - noRt = 0 return altPkgs[0].Types }) prog.SetPython(func() *types.Package { @@ -350,31 +349,36 @@ func Do(args []string, conf *Config) ([]Package, error) { output := conf.OutFile != "" ctx := &context{env: env, conf: cfg, progSSA: progSSA, prog: prog, dedup: dedup, patches: patches, built: make(map[string]none), initial: initial, mode: mode, - output: output, - needRt: make(map[*packages.Package]bool), - needPyInit: make(map[*packages.Package]bool), - buildConf: conf, - crossCompile: export, - cTransformer: cabi.NewTransformer(prog, export.LLVMTarget, export.TargetABI, conf.AbiMode, cabiOptimize), + fingerprinting: make(map[string]bool), + pkgs: map[*packages.Package]Package{}, + pkgByID: map[string]Package{}, + output: output, + buildConf: conf, + crossCompile: export, + cTransformer: cabi.NewTransformer(prog, export.LLVMTarget, export.TargetABI, conf.AbiMode, cabiOptimize), } // default runtime globals must be registered before packages are built addGlobalString(conf, "runtime.defaultGOROOT="+runtime.GOROOT(), nil) addGlobalString(conf, "runtime.buildVersion="+runtime.Version(), nil) - pkgs, err := buildAllPkgs(ctx, initial, verbose) + pkgs, err := buildSSAPkgs(ctx, initial, verbose) check(err) + depPkgs, err := buildSSAPkgs(ctx, altPkgs, verbose) + check(err) + + allPkgs := append([]*aPackage{}, pkgs...) + allPkgs = append(allPkgs, depPkgs...) + allPkgs, err = buildAllPkgs(ctx, allPkgs, verbose) + check(err) + if mode == ModeGen { - for _, pkg := range pkgs { + for _, pkg := range allPkgs { if pkg.Package == initial[0] { return []*aPackage{pkg}, nil } } return nil, fmt.Errorf("initial package not found") } - dpkg, err := buildAllPkgs(ctx, altPkgs[noRt:], verbose) - check(err) - allPkgs := append([]*aPackage{}, pkgs...) - allPkgs = append(allPkgs, dpkg...) for _, pkg := range initial { if needLink(pkg, mode) { @@ -468,7 +472,7 @@ func Do(args []string, conf *Config) ([]Package, error) { mockable.Exit(1) } - return dpkg, nil + return allPkgs, nil } func needLink(pkg *packages.Package, mode Mode) bool { @@ -478,14 +482,14 @@ func needLink(pkg *packages.Package, mode Mode) bool { return pkg.Name == "main" } -func setNeedRuntimeOrPyInit(ctx *context, pkg *packages.Package, needRuntime, needPyInit bool) { - ctx.needRt[pkg] = needRuntime - ctx.needPyInit[pkg] = needPyInit +func (p Package) setNeedRuntimeOrPyInit(needRuntime, needPyInit bool) { + p.NeedRt = needRuntime + p.NeedPyInit = needPyInit } -func isNeedRuntimeOrPyInit(ctx *context, pkg *packages.Package) (needRuntime, needPyInit bool) { - needRuntime = ctx.needRt[pkg] - needPyInit = ctx.needPyInit[pkg] +func (p Package) isNeedRuntimeOrPyInit() (needRuntime, needPyInit bool) { + needRuntime = p.NeedRt + needPyInit = p.NeedPyInit return } @@ -494,20 +498,20 @@ const ( ) type context struct { - env *llvm.Env - conf *packages.Config - progSSA *ssa.Program - prog llssa.Program - dedup packages.Deduper - patches cl.Patches - built map[string]none - initial []*packages.Package - mode Mode - nLibdir int - output bool - - needRt map[*packages.Package]bool - needPyInit map[*packages.Package]bool + env *llvm.Env + conf *packages.Config + progSSA *ssa.Program + prog llssa.Program + dedup packages.Deduper + patches cl.Patches + built map[string]none + fingerprinting map[string]bool + initial []*packages.Package + pkgs map[*packages.Package]Package // cache for lookup + pkgByID map[string]Package // cache for lookup by pkg.ID + mode Mode + nLibdir int32 + output bool buildConf *Config crossCompile crosscompile.Export @@ -515,6 +519,10 @@ type context struct { cTransformer *cabi.Transformer testFail bool + + // Cache related fields + cacheManager *cacheManager + llvmVersion string } func (c *context) compiler() *clang.Cmd { @@ -543,92 +551,136 @@ func (c *context) linker() *clang.Cmd { return cmd } -func buildAllPkgs(ctx *context, initial []*packages.Package, verbose bool) (pkgs []*aPackage, err error) { - pkgs, errPkgs := allPkgs(ctx, initial, verbose) - for _, errPkg := range errPkgs { - for _, err := range errPkg.Errors { - fmt.Fprintln(os.Stderr, err) +func buildAllPkgs(ctx *context, pkgs []*aPackage, verbose bool) ([]*aPackage, error) { + built := ctx.built + + // Split packages into runtime tree vs others so we can defer runtime build. + var runtimePkgs []*aPackage + var normalPkgs []*aPackage + for _, p := range pkgs { + if isRuntimePkg(p.PkgPath) { + runtimePkgs = append(runtimePkgs, p) + } else { + normalPkgs = append(normalPkgs, p) } - fmt.Fprintln(os.Stderr, "cannot build SSA for package", errPkg) } - if len(errPkgs) > 0 { - return nil, fmt.Errorf("cannot build SSA for packages") - } - built := ctx.built - for _, aPkg := range pkgs { + + var needRuntime, needPyInit bool + + buildOne := func(aPkg *aPackage) error { pkg := aPkg.Package if _, ok := built[pkg.ID]; ok { - pkg.ExportFile = "" - continue + // Already built, skip but keep ExportFile for linking + return nil } built[pkg.ID] = none{} + switch kind, param := cl.PkgKindOf(pkg.Types); kind { case cl.PkgDeclOnly: - // skip packages that only contain declarations - // and set no export file pkg.ExportFile = "" case cl.PkgLinkIR, cl.PkgLinkExtern, cl.PkgPyModule: if len(pkg.GoFiles) > 0 { - err := buildPkg(ctx, aPkg, verbose) - if err != nil { - return nil, err + if err := ctx.collectFingerprint(aPkg); err != nil { + return err } - } else { - // panic("todo") - // TODO(xsw): support packages out of llgo - pkg.ExportFile = "" - } - if kind == cl.PkgLinkExtern { - // need to be linked with external library - // format: ';' separated alternative link methods. e.g. - // link: $LLGO_LIB_PYTHON; $(pkg-config --libs python3-embed); -lpython3 - altParts := strings.Split(param, ";") - expdArgs := make([]string, 0, len(altParts)) - for _, param := range altParts { - param = strings.TrimSpace(param) - if strings.ContainsRune(param, '$') { - expdArgs = append(expdArgs, xenv.ExpandEnvToArgs(param)...) - ctx.nLibdir++ - } else { - fields := strings.Fields(param) - expdArgs = append(expdArgs, fields...) - } - if len(expdArgs) > 0 { - break - } - } - if len(expdArgs) == 0 { - panic(fmt.Sprintf("'%s' cannot locate the external library", param)) + ctx.tryLoadFromCache(aPkg) + if err := buildPkg(ctx, aPkg, verbose); err != nil { + return err } - - pkgLinkArgs := make([]string, 0, 3) - if expdArgs[0][0] == '-' { - pkgLinkArgs = append(pkgLinkArgs, expdArgs...) - } else { - linkFile := expdArgs[0] - dir, lib := filepath.Split(linkFile) - pkgLinkArgs = append(pkgLinkArgs, "-l"+lib) - if dir != "" { - pkgLinkArgs = append(pkgLinkArgs, "-L"+dir) - ctx.nLibdir++ + if !aPkg.CacheHit { + if kind == cl.PkgLinkExtern { + appendExternalLinkArgs(ctx, aPkg, param) } - } - if ctx.buildConf.CheckLinkArgs { - if err := ctx.compiler().CheckLinkArgs(pkgLinkArgs, isWasmTarget(ctx.buildConf.Goos)); err != nil { - panic(fmt.Sprintf("test link args '%s' failed\n\texpanded to: %v\n\tresolved to: %v\n\terror: %v", param, expdArgs, pkgLinkArgs, err)) + if err := ctx.saveToCache(aPkg); err != nil && verbose { + fmt.Fprintf(os.Stderr, "warning: failed to save cache for %s: %v\n", pkg.PkgPath, err) } } - aPkg.LinkArgs = append(aPkg.LinkArgs, pkgLinkArgs...) + } else { + pkg.ExportFile = "" + if kind == cl.PkgLinkExtern { + appendExternalLinkArgs(ctx, aPkg, param) + } } default: - err := buildPkg(ctx, aPkg, verbose) - if err != nil { + if err := ctx.collectFingerprint(aPkg); err != nil { + return err + } + ctx.tryLoadFromCache(aPkg) + if err := buildPkg(ctx, aPkg, verbose); err != nil { + return err + } + aPkg.setNeedRuntimeOrPyInit(aPkg.LPkg.NeedRuntime, aPkg.LPkg.NeedPyInit) + needRuntime = needRuntime || aPkg.NeedRt + needPyInit = needPyInit || aPkg.NeedPyInit + if !aPkg.CacheHit { + if err := ctx.saveToCache(aPkg); err != nil && verbose { + fmt.Fprintf(os.Stderr, "warning: failed to save cache for %s: %v\n", pkg.PkgPath, err) + } + } + } + return nil + } + + // Build non-runtime packages first, so we know whether runtime is actually needed. + for _, p := range normalPkgs { + if err := buildOne(p); err != nil { + return nil, err + } + } + + // Only build runtime packages when required (or host build with empty Target). + if needRuntime || needPyInit || ctx.buildConf.Target == "" { + for _, p := range runtimePkgs { + if err := buildOne(p); err != nil { return nil, err } - setNeedRuntimeOrPyInit(ctx, pkg, aPkg.LPkg.NeedRuntime, aPkg.LPkg.NeedPyInit) } } - return + + return pkgs, nil +} + +func appendExternalLinkArgs(ctx *context, aPkg *aPackage, spec string) { + // need to be linked with external library + // format: ';' separated alternative link methods. e.g. + // link: $LLGO_LIB_PYTHON; $(pkg-config --libs python3-embed); -lpython3 + altParts := strings.Split(spec, ";") + expdArgs := make([]string, 0, len(altParts)) + for _, alt := range altParts { + alt = strings.TrimSpace(alt) + if strings.ContainsRune(alt, '$') { + expdArgs = append(expdArgs, xenv.ExpandEnvToArgs(alt)...) + atomic.AddInt32(&ctx.nLibdir, 1) + } else { + fields := strings.Fields(alt) + expdArgs = append(expdArgs, fields...) + } + if len(expdArgs) > 0 { + break + } + } + if len(expdArgs) == 0 { + panic(fmt.Sprintf("'%s' cannot locate the external library", spec)) + } + + pkgLinkArgs := make([]string, 0, 3) + if expdArgs[0][0] == '-' { + pkgLinkArgs = append(pkgLinkArgs, expdArgs...) + } else { + linkFile := expdArgs[0] + dir, lib := filepath.Split(linkFile) + pkgLinkArgs = append(pkgLinkArgs, "-l"+lib) + if dir != "" { + pkgLinkArgs = append(pkgLinkArgs, "-L"+dir) + atomic.AddInt32(&ctx.nLibdir, 1) + } + } + if ctx.buildConf.CheckLinkArgs { + if err := ctx.compiler().CheckLinkArgs(pkgLinkArgs, isWasmTarget(ctx.buildConf.Goos)); err != nil { + panic(fmt.Sprintf("test link args '%s' failed\n\texpanded to: %v\n\tresolved to: %v\n\terror: %v", spec, expdArgs, pkgLinkArgs, err)) + } + } + aPkg.LinkArgs = append(aPkg.LinkArgs, pkgLinkArgs...) } var ( @@ -760,31 +812,53 @@ func compileExtraFiles(ctx *context, verbose bool) ([]string, error) { } func linkMainPkg(ctx *context, pkg *packages.Package, pkgs []*aPackage, outputPath string, verbose bool) error { - needRuntime := false needPyInit := false - pkgsMap := make(map[*packages.Package]*aPackage, len(pkgs)) allPkgs := []*packages.Package{pkg} for _, v := range pkgs { - pkgsMap[v.Package] = v allPkgs = append(allPkgs, v.Package) } var objFiles []string var linkArgs []string + var rtObjFiles []string + var rtLinkArgs []string + linkedPkgs := make(map[string]bool) // Track linked packages by ID to avoid duplicates packages.Visit(allPkgs, nil, func(p *packages.Package) { - aPkg := pkgsMap[p] + // Skip if already linked this package (by ID) + if linkedPkgs[p.ID] { + return + } + aPkg := ctx.pkgs[p] + if aPkg == nil { + // Fallback: lookup by pkg.ID for packages that may be different instances + aPkg = ctx.pkgByID[p.ID] + } if p.ExportFile != "" && aPkg != nil { // skip packages that only contain declarations + linkedPkgs[p.ID] = true + + // Defer linking runtime packages unless we actually need the runtime. + if isRuntimePkg(p.PkgPath) { + rtLinkArgs = append(rtLinkArgs, aPkg.LinkArgs...) + rtObjFiles = append(rtObjFiles, aPkg.LLFiles...) + return + } else { + // Only let non-runtime packages influence whether runtime is needed. + need1, need2 := aPkg.isNeedRuntimeOrPyInit() + needRuntime = needRuntime || need1 + needPyInit = needPyInit || need2 + } + linkArgs = append(linkArgs, aPkg.LinkArgs...) objFiles = append(objFiles, aPkg.LLFiles...) - need1, need2 := isNeedRuntimeOrPyInit(ctx, p) - if !needRuntime { - needRuntime = need1 - } - if !needPyInit { - needPyInit = need2 - } } }) + + // Only link runtime objects when needed (or for host builds where runtime is always required). + if needRuntime || needPyInit || ctx.buildConf.Target == "" { + linkArgs = append(linkArgs, rtLinkArgs...) + objFiles = append(objFiles, rtObjFiles...) + } + // Generate main module file (needed for global variables even in library modes) entryPkg := genMainModule(ctx, llssa.PkgRuntime, pkg, needRuntime, needPyInit) entryObjFile, err := exportObject(ctx, entryPkg.PkgPath, entryPkg.ExportFile, []byte(entryPkg.LPkg.String())) @@ -825,10 +899,16 @@ func linkMainPkg(ctx *context, pkg *packages.Package, pkgs []*aPackage, outputPa return nil } +// isRuntimePkg reports whether the package path belongs to the llgo runtime tree. +func isRuntimePkg(pkgPath string) bool { + rtRoot := env.LLGoRuntimePkg + return pkgPath == rtRoot || strings.HasPrefix(pkgPath, rtRoot+"/") +} + func linkObjFiles(ctx *context, app string, objFiles, linkArgs []string, verbose bool) error { // Handle c-archive mode differently - use ar tool instead of linker if ctx.buildConf.BuildMode == BuildModeCArchive { - return createStaticArchive(ctx, app, objFiles, verbose) + return createArchiveFile(app, objFiles, verbose) } buildArgs := []string{"-o", app} @@ -874,24 +954,40 @@ func linkObjFiles(ctx *context, app string, objFiles, linkArgs []string, verbose return cmd.Link(buildArgs...) } -func createStaticArchive(ctx *context, archivePath string, objFiles []string, verbose bool) error { - // Use ar tool to create static archive - args := []string{"rcs", archivePath} - args = append(args, objFiles...) +// createArchiveFile builds an archive at archivePath atomically to avoid races when +// multiple builds target the same output concurrently. +func createArchiveFile(archivePath string, objFiles []string, verbose ...bool) error { + if len(objFiles) == 0 { + return fmt.Errorf("no object files provided for archive %s", archivePath) + } - if verbose { - fmt.Fprintf(os.Stderr, "ar %s\n", strings.Join(args, " ")) + if err := os.MkdirAll(filepath.Dir(archivePath), 0o755); err != nil { + return err } - cmd := exec.Command("ar", args...) - output, err := cmd.CombinedOutput() + tmp, err := os.CreateTemp(filepath.Dir(archivePath), filepath.Base(archivePath)+".tmp-*") if err != nil { - if verbose && len(output) > 0 { - fmt.Fprintf(os.Stderr, "ar output: %s\n", output) - } - return fmt.Errorf("ar command failed: %w", err) + return err } + tmp.Close() + tmpName := tmp.Name() + // Remove the placeholder so ar can create the archive fresh. + _ = os.Remove(tmpName) + args := append([]string{"rcs", tmpName}, objFiles...) + cmd := exec.Command("ar", args...) + if len(verbose) > 0 && verbose[0] { + fmt.Fprintf(os.Stderr, "ar %s\n", strings.Join(args, " ")) + } + if output, err := cmd.CombinedOutput(); err != nil { + os.Remove(tmpName) + return fmt.Errorf("create archive %s: %w\n%s", archivePath, err, output) + } + + if err := os.Rename(tmpName, archivePath); err != nil { + os.Remove(tmpName) + return fmt.Errorf("publish archive %s: %w", archivePath, err) + } return nil } @@ -943,9 +1039,15 @@ func buildPkg(ctx *context, aPkg *aPackage, verbose bool) error { } check(err) + aPkg.LPkg = ret + + // If cache hit, we only needed to register types - skip compilation + if aPkg.CacheHit { + return nil + } + ctx.cTransformer.TransformModule(ret.Path(), ret.Module()) - aPkg.LPkg = ret cgoLLFiles, cgoLdflags, err := buildCgo(ctx, aPkg, aPkg.Package.Syntax, externs, verbose) if err != nil { return fmt.Errorf("build cgo of %v failed: %v", pkgPath, err) @@ -1066,20 +1168,30 @@ type aPackage struct { AltPkg *packages.Cached LPkg llssa.Package + NeedRt bool + NeedPyInit bool + LinkArgs []string LLFiles []string rewriteVars map[string]string + + // Cache related fields + Fingerprint string // fingerprint digest + Manifest string // manifest text content + CacheHit bool // whether cache was hit } type Package = *aPackage -func allPkgs(ctx *context, initial []*packages.Package, verbose bool) (all []*aPackage, errs []*packages.Package) { +func buildSSAPkgs(ctx *context, initial []*packages.Package, verbose bool) ([]*aPackage, error) { prog := ctx.progSSA - built := ctx.built + var all []*aPackage + var errs []*packages.Package packages.Visit(initial, nil, func(p *packages.Package) { if p.Types != nil && !p.IllTyped { pkgPath := p.PkgPath - if _, ok := built[pkgPath]; ok || strings.HasPrefix(pkgPath, altPkgPathPrefix) { + // Use p.ID to check duplicates since same pkgPath may have different IDs + if _, ok := ctx.pkgByID[p.ID]; ok || strings.HasPrefix(pkgPath, altPkgPathPrefix) { return } var altPkg *packages.Cached @@ -1090,12 +1202,34 @@ func allPkgs(ctx *context, initial []*packages.Package, verbose bool) (all []*aP } } rewrites := collectRewriteVars(ctx, pkgPath) - all = append(all, &aPackage{p, ssaPkg, altPkg, nil, nil, nil, rewrites}) + aPkg := &aPackage{ + Package: p, + SSA: ssaPkg, + AltPkg: altPkg, + LPkg: nil, + NeedRt: false, + NeedPyInit: false, + LinkArgs: nil, + LLFiles: nil, + rewriteVars: rewrites, + } + ctx.pkgs[p] = aPkg + ctx.pkgByID[p.ID] = aPkg + all = append(all, aPkg) } else { errs = append(errs, p) } }) - return + if len(errs) > 0 { + for _, errPkg := range errs { + for _, err := range errPkg.Errors { + fmt.Fprintln(os.Stderr, err) + } + fmt.Fprintln(os.Stderr, "cannot build SSA for package", errPkg) + } + return nil, fmt.Errorf("cannot build SSA for packages") + } + return all, nil } func collectRewriteVars(ctx *context, pkgPath string) map[string]string { @@ -1201,6 +1335,7 @@ const llgoWasmRuntime = "LLGO_WASM_RUNTIME" const llgoWasiThreads = "LLGO_WASI_THREADS" const llgoStdioNobuf = "LLGO_STDIO_NOBUF" const llgoFullRpath = "LLGO_FULL_RPATH" +const llgoBuildCache = "LLGO_BUILD_CACHE" const defaultWasmRuntime = "wasmtime" @@ -1220,6 +1355,12 @@ func isEnvOn(env string, defVal bool) bool { return envVal == "1" || envVal == "true" || envVal == "on" } +// cacheEnabled checks if build cache is enabled. +// Cache can be disabled by setting LLGO_BUILD_CACHE=off|0 +func cacheEnabled() bool { + return isEnvOn(llgoBuildCache, true) +} + func IsTraceEnabled() bool { return isEnvOn(llgoTrace, false) } diff --git a/internal/build/build_test.go b/internal/build/build_test.go index 65aba54145..7e2659c580 100644 --- a/internal/build/build_test.go +++ b/internal/build/build_test.go @@ -16,46 +16,42 @@ import ( "github.com/goplus/llgo/internal/mockable" ) +func TestMain(m *testing.M) { + old := cacheRootFunc + td, _ := os.MkdirTemp("", "llgo-cache-*") + cacheRootFunc = func() string { return td } + code := m.Run() + cacheRootFunc = old + _ = os.RemoveAll(td) + os.Exit(code) +} + func mockRun(args []string, cfg *Config) { - const maxAttempts = 3 - var lastErr error - var lastPanic interface{} - for attempt := 0; attempt < maxAttempts; attempt++ { - mockable.EnableMock() - func() { - defer func() { - if r := recover(); r != nil { - if r != "exit" { - lastPanic = r - } else { - exitCode := mockable.ExitCode() - if (exitCode != 0) != false { - lastPanic = fmt.Errorf("got exit code %d", exitCode) - } - } - } - }() - file, _ := os.CreateTemp("", "llgo-*") - cfg.OutFile = file.Name() - file.Close() - defer os.Remove(cfg.OutFile) - _, err := Do(args, cfg) - if err == nil { - return // Success, return immediately from the inner function + defer mockable.DisableMock() + mockable.EnableMock() + + var panicVal interface{} + defer func() { + if r := recover(); r != nil { + // Ignore mocked os.Exit + if s, ok := r.(string); ok && s == "exit" { + return } - lastErr = err - }() - - if lastPanic == nil && lastErr == nil { - return // Success, return from mockRun + panicVal = r } - // Continue to next attempt if this one failed - } - // If we get here, all attempts failed - if lastPanic != nil { - panic(lastPanic) + if panicVal != nil { + panic(panicVal) + } + }() + + file, _ := os.CreateTemp("", "llgo-*") + cfg.OutFile = file.Name() + file.Close() + defer os.Remove(cfg.OutFile) + + if _, err := Do(args, cfg); err != nil { + panic(err) } - panic(fmt.Errorf("all %d attempts failed, last error: %v", maxAttempts, lastErr)) } func TestRun(t *testing.T) { diff --git a/internal/build/cache.go b/internal/build/cache.go new file mode 100644 index 0000000000..9583f1d4e8 --- /dev/null +++ b/internal/build/cache.go @@ -0,0 +1,253 @@ +/* + * Copyright (c) 2024 The GoPlus Authors (goplus.org). All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package build + +import ( + "crypto/sha256" + "encoding/hex" + "fmt" + "os" + "path/filepath" + "strings" + + "github.com/goplus/llgo/internal/env" +) + +const ( + cacheBuildDirName = "build" + cacheArchiveExt = ".a" + cacheManifestExt = ".manifest" +) + +// cacheRootFunc can be overridden for testing +var cacheRootFunc = env.LLGoCacheDir + +// buildCacheRootDir returns the root directory for build cache +func buildCacheRootDir() string { + return filepath.Join(cacheRootFunc(), cacheBuildDirName) +} + +// cacheManager manages the build cache directory structure +type cacheManager struct { + root string +} + +// newCacheManager creates a new cache manager +func newCacheManager() *cacheManager { + return &cacheManager{root: buildCacheRootDir()} +} + +// cachePaths holds the paths for a cached package +type cachePaths struct { + Dir string // Directory containing cache files + Archive string // Path to .a file + Manifest string // Path to .manifest file +} + +// PackagePaths returns the cache paths for a package +func (cm *cacheManager) PackagePaths(targetTriple, pkgPath, fingerprint string) cachePaths { + dir := cm.packageDir(targetTriple, pkgPath) + fingerprint = sanitizeComponent(fingerprint) + return cachePaths{ + Dir: dir, + Archive: filepath.Join(dir, fingerprint+cacheArchiveExt), + Manifest: filepath.Join(dir, fingerprint+cacheManifestExt), + } +} + +func (cm *cacheManager) packageDir(targetTriple, pkgPath string) string { + root := filepath.Clean(cm.root) + dir := filepath.Join(root, sanitizeComponent(targetTriple), sanitizePkgPath(pkgPath)) + dir = filepath.Clean(dir) + if dir != root && !strings.HasPrefix(dir, root+string(os.PathSeparator)) { + dir = filepath.Join(root, "_") + } + return dir +} + +// sanitizeComponent ensures a single path component is safe. +func sanitizeComponent(s string) string { + if s == "" || s == "." || s == ".." { + return "_" + } + if isSafeComponent(s) { + return s + } + sum := sha256.Sum256([]byte(s)) + return hex.EncodeToString(sum[:8]) +} + +func isSafeComponent(s string) bool { + for _, r := range s { + if r >= 'a' && r <= 'z' { + continue + } + if r >= 'A' && r <= 'Z' { + continue + } + if r >= '0' && r <= '9' { + continue + } + switch r { + case '-', '_', '.': + continue + default: + return false + } + } + return true +} + +// sanitizePkgPath converts a package path to a safe directory path +func sanitizePkgPath(pkgPath string) string { + if pkgPath == "" { + return "_" + } + segments := strings.Split(pkgPath, "/") + for i, segment := range segments { + if segment == "" || segment == "." || segment == ".." { + segments[i] = "_" + continue + } + segments[i] = sanitizeComponent(segment) + } + return filepath.Join(segments...) +} + +// EnsureDir creates the cache directory if it doesn't exist +func (cm *cacheManager) EnsureDir(paths cachePaths) error { + return os.MkdirAll(paths.Dir, 0o755) +} + +// writeManifest writes manifest content to a file atomically +func writeManifest(path string, content string) error { + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + return fmt.Errorf("create manifest dir: %w", err) + } + + // Write to temp file first + tmp, err := os.CreateTemp(filepath.Dir(path), "manifest-*.tmp") + if err != nil { + return fmt.Errorf("create temp manifest: %w", err) + } + + if _, err := tmp.WriteString(content); err != nil { + tmp.Close() + os.Remove(tmp.Name()) + return fmt.Errorf("write manifest: %w", err) + } + + if err := tmp.Close(); err != nil { + os.Remove(tmp.Name()) + return fmt.Errorf("close manifest: %w", err) + } + + // Atomic rename + if err := os.Rename(tmp.Name(), path); err != nil { + os.Remove(tmp.Name()) + return fmt.Errorf("publish manifest: %w", err) + } + + return nil +} + +// readManifest reads manifest content from a file +func readManifest(path string) (string, error) { + content, err := os.ReadFile(path) + if err != nil { + return "", err + } + return string(content), nil +} + +// cacheExists checks if a valid cache entry exists +func (cm *cacheManager) cacheExists(paths cachePaths) bool { + // Both archive and manifest must exist + if _, err := os.Stat(paths.Archive); err != nil { + return false + } + if _, err := os.Stat(paths.Manifest); err != nil { + return false + } + return true +} + +// cleanPackageCache removes all cache entries for a package +func (cm *cacheManager) cleanPackageCache(targetTriple, pkgPath string) error { + return os.RemoveAll(cm.packageDir(targetTriple, pkgPath)) +} + +// cleanAllCache removes the entire build cache +func (cm *cacheManager) cleanAllCache() error { + return os.RemoveAll(cm.root) +} + +// listCachedPackages returns all cached fingerprints for a package +func (cm *cacheManager) listCachedPackages(targetTriple, pkgPath string) ([]string, error) { + dir := cm.packageDir(targetTriple, pkgPath) + + entries, err := os.ReadDir(dir) + if err != nil { + if os.IsNotExist(err) { + return nil, nil + } + return nil, err + } + + var fingerprints []string + for _, entry := range entries { + name := entry.Name() + if strings.HasSuffix(name, cacheArchiveExt) { + fp := strings.TrimSuffix(name, cacheArchiveExt) + fingerprints = append(fingerprints, fp) + } + } + + return fingerprints, nil +} + +// cacheStats holds statistics about the cache +type cacheStats struct { + TotalPackages int + TotalSize int64 +} + +// stats returns statistics about the cache +func (cm *cacheManager) stats() (cacheStats, error) { + var stats cacheStats + + err := filepath.Walk(cm.root, func(path string, info os.FileInfo, err error) error { + if err != nil { + if os.IsNotExist(err) { + return nil + } + return err + } + if !info.IsDir() { + stats.TotalSize += info.Size() + if strings.HasSuffix(path, cacheArchiveExt) { + stats.TotalPackages++ + } + } + return nil + }) + + if os.IsNotExist(err) { + return stats, nil + } + return stats, err +} diff --git a/internal/build/cache_test.go b/internal/build/cache_test.go new file mode 100644 index 0000000000..66860253e1 --- /dev/null +++ b/internal/build/cache_test.go @@ -0,0 +1,347 @@ +//go:build !llgo + +/* + * Copyright (c) 2024 The GoPlus Authors (goplus.org). All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package build + +import ( + "os" + "path/filepath" + "strings" + "testing" +) + +func TestSanitizePkgPath(t *testing.T) { + tests := []struct { + input string + want string + }{ + {"github.com/user/repo", filepath.Join("github.com", "user", "repo")}, + {"example.com/pkg", filepath.Join("example.com", "pkg")}, + {"simple", "simple"}, + {"", "_"}, + {"a//b", filepath.Join("a", "_", "b")}, + } + + for _, tt := range tests { + got := sanitizePkgPath(tt.input) + if got != tt.want { + t.Errorf("sanitizePkgPath(%q) = %q, want %q", tt.input, got, tt.want) + } + } +} + +func TestCacheManager_PackagePaths(t *testing.T) { + // Override cache root for testing + td := t.TempDir() + oldFunc := cacheRootFunc + cacheRootFunc = func() string { return td } + defer func() { cacheRootFunc = oldFunc }() + + cm := newCacheManager() + paths := cm.PackagePaths("arm64-darwin", "github.com/user/pkg", "abc123") + + expectedDir := filepath.Join(td, "build", "arm64-darwin", "github.com", "user", "pkg") + if paths.Dir != expectedDir { + t.Errorf("Dir = %q, want %q", paths.Dir, expectedDir) + } + + expectedArchive := filepath.Join(expectedDir, "abc123.a") + if paths.Archive != expectedArchive { + t.Errorf("Archive = %q, want %q", paths.Archive, expectedArchive) + } + + expectedManifest := filepath.Join(expectedDir, "abc123.manifest") + if paths.Manifest != expectedManifest { + t.Errorf("Manifest = %q, want %q", paths.Manifest, expectedManifest) + } +} + +func TestCacheManager_EnsureDir(t *testing.T) { + td := t.TempDir() + oldFunc := cacheRootFunc + cacheRootFunc = func() string { return td } + defer func() { cacheRootFunc = oldFunc }() + + cm := newCacheManager() + paths := cm.PackagePaths("x86_64-linux", "test/pkg", "fp123") + + if err := cm.EnsureDir(paths); err != nil { + t.Fatalf("EnsureDir: %v", err) + } + + if _, err := os.Stat(paths.Dir); err != nil { + t.Errorf("directory not created: %v", err) + } + + // Should be idempotent + if err := cm.EnsureDir(paths); err != nil { + t.Errorf("EnsureDir second call: %v", err) + } +} + +func TestWriteManifest(t *testing.T) { + td := t.TempDir() + path := filepath.Join(td, "subdir", "test.manifest") + content := "Env:\nGOOS=linux\n" + + if err := writeManifest(path, content); err != nil { + t.Fatalf("WriteManifest: %v", err) + } + + // Read back + got, err := readManifest(path) + if err != nil { + t.Fatalf("ReadManifest: %v", err) + } + + if got != content { + t.Errorf("content = %q, want %q", got, content) + } +} + +func TestWriteManifest_Atomic(t *testing.T) { + td := t.TempDir() + path := filepath.Join(td, "test.manifest") + + // Write first version + if err := writeManifest(path, "version1"); err != nil { + t.Fatalf("WriteManifest v1: %v", err) + } + + // Write second version (should replace atomically) + if err := writeManifest(path, "version2"); err != nil { + t.Fatalf("WriteManifest v2: %v", err) + } + + got, _ := readManifest(path) + if got != "version2" { + t.Errorf("content = %q, want version2", got) + } + + // No temp files should remain + entries, _ := os.ReadDir(td) + for _, e := range entries { + if strings.Contains(e.Name(), ".tmp") { + t.Errorf("temp file left behind: %s", e.Name()) + } + } +} + +func TestReadManifest_NotExist(t *testing.T) { + _, err := readManifest("/nonexistent/file.manifest") + if err == nil { + t.Error("expected error for nonexistent file") + } +} + +func TestCacheManager_CacheExists(t *testing.T) { + td := t.TempDir() + oldFunc := cacheRootFunc + cacheRootFunc = func() string { return td } + defer func() { cacheRootFunc = oldFunc }() + + cm := newCacheManager() + paths := cm.PackagePaths("arm64-darwin", "test/pkg", "fp123") + + // Initially should not exist + if cm.cacheExists(paths) { + t.Error("cache should not exist initially") + } + + // Create directory and files + if err := cm.EnsureDir(paths); err != nil { + t.Fatal(err) + } + os.WriteFile(paths.Archive, []byte("archive"), 0644) + + // Still should not exist (manifest missing) + if cm.cacheExists(paths) { + t.Error("cache should not exist without manifest") + } + + // Create manifest + os.WriteFile(paths.Manifest, []byte("manifest"), 0644) + + // Now should exist + if !cm.cacheExists(paths) { + t.Error("cache should exist with both files") + } +} + +func TestTargetTriple(t *testing.T) { + tests := []struct { + goos, goarch, llvmTarget, abi string + want string + }{ + {"darwin", "arm64", "arm64-apple-darwin", "", "arm64-apple-darwin"}, + {"linux", "amd64", "x86_64-unknown-linux-gnu", "", "x86_64-unknown-linux-gnu"}, + {"darwin", "arm64", "", "", "arm64-darwin"}, + {"linux", "amd64", "", "gnu", "amd64-linux-gnu"}, + {"windows", "amd64", "x86_64-pc-windows-msvc", "msvc", "x86_64-pc-windows-msvc-msvc"}, + } + + for _, tt := range tests { + got := targetTriple(tt.goos, tt.goarch, tt.llvmTarget, tt.abi) + if got != tt.want { + t.Errorf("TargetTriple(%q, %q, %q, %q) = %q, want %q", + tt.goos, tt.goarch, tt.llvmTarget, tt.abi, got, tt.want) + } + } +} + +func TestCacheManager_CleanPackageCache(t *testing.T) { + td := t.TempDir() + oldFunc := cacheRootFunc + cacheRootFunc = func() string { return td } + defer func() { cacheRootFunc = oldFunc }() + + cm := newCacheManager() + paths := cm.PackagePaths("arm64-darwin", "test/pkg", "fp123") + + // Create cache + cm.EnsureDir(paths) + os.WriteFile(paths.Archive, []byte("archive"), 0644) + os.WriteFile(paths.Manifest, []byte("manifest"), 0644) + + // Clean + if err := cm.cleanPackageCache("arm64-darwin", "test/pkg"); err != nil { + t.Fatalf("CleanPackageCache: %v", err) + } + + // Should not exist + if cm.cacheExists(paths) { + t.Error("cache should be cleaned") + } +} + +func TestCacheManager_CleanAllCache(t *testing.T) { + td := t.TempDir() + oldFunc := cacheRootFunc + cacheRootFunc = func() string { return td } + defer func() { cacheRootFunc = oldFunc }() + + cm := newCacheManager() + + // Create multiple caches + paths1 := cm.PackagePaths("arm64-darwin", "pkg1", "fp1") + paths2 := cm.PackagePaths("x86_64-linux", "pkg2", "fp2") + + cm.EnsureDir(paths1) + cm.EnsureDir(paths2) + os.WriteFile(paths1.Archive, []byte("1"), 0644) + os.WriteFile(paths2.Archive, []byte("2"), 0644) + + // Clean all + if err := cm.cleanAllCache(); err != nil { + t.Fatalf("CleanAllCache: %v", err) + } + + // Check root is removed + if _, err := os.Stat(filepath.Join(td, "build")); !os.IsNotExist(err) { + t.Error("cache root should be removed") + } +} + +func TestCacheManager_ListCachedPackages(t *testing.T) { + td := t.TempDir() + oldFunc := cacheRootFunc + cacheRootFunc = func() string { return td } + defer func() { cacheRootFunc = oldFunc }() + + cm := newCacheManager() + + // Initially empty + fps, err := cm.listCachedPackages("arm64-darwin", "test/pkg") + if err != nil { + t.Fatalf("ListCachedPackages: %v", err) + } + if len(fps) != 0 { + t.Errorf("expected empty list, got %v", fps) + } + + // Create some caches + paths1 := cm.PackagePaths("arm64-darwin", "test/pkg", "fp1") + paths2 := cm.PackagePaths("arm64-darwin", "test/pkg", "fp2") + cm.EnsureDir(paths1) + os.WriteFile(paths1.Archive, []byte("1"), 0644) + os.WriteFile(paths2.Archive, []byte("2"), 0644) + + fps, err = cm.listCachedPackages("arm64-darwin", "test/pkg") + if err != nil { + t.Fatalf("ListCachedPackages: %v", err) + } + if len(fps) != 2 { + t.Errorf("expected 2 fingerprints, got %d", len(fps)) + } +} + +func TestCacheManager_Stats(t *testing.T) { + td := t.TempDir() + oldFunc := cacheRootFunc + cacheRootFunc = func() string { return td } + defer func() { cacheRootFunc = oldFunc }() + + cm := newCacheManager() + + // Create some caches + paths1 := cm.PackagePaths("arm64-darwin", "pkg1", "fp1") + paths2 := cm.PackagePaths("arm64-darwin", "pkg2", "fp2") + cm.EnsureDir(paths1) + cm.EnsureDir(paths2) + + content1 := []byte("archive content 1") + content2 := []byte("archive content 2 longer") + os.WriteFile(paths1.Archive, content1, 0644) + os.WriteFile(paths2.Archive, content2, 0644) + os.WriteFile(paths1.Manifest, []byte("m1"), 0644) + os.WriteFile(paths2.Manifest, []byte("m2"), 0644) + + stats, err := cm.stats() + if err != nil { + t.Fatalf("Stats: %v", err) + } + + if stats.TotalPackages != 2 { + t.Errorf("TotalPackages = %d, want 2", stats.TotalPackages) + } + + expectedSize := int64(len(content1) + len(content2) + 4) // +4 for manifests + if stats.TotalSize != expectedSize { + t.Errorf("TotalSize = %d, want %d", stats.TotalSize, expectedSize) + } +} + +func TestCacheManager_Stats_Empty(t *testing.T) { + td := t.TempDir() + oldFunc := cacheRootFunc + cacheRootFunc = func() string { return td } + defer func() { cacheRootFunc = oldFunc }() + + cm := newCacheManager() + + // Stats on empty cache should not error + stats, err := cm.stats() + if err != nil { + t.Fatalf("Stats on empty: %v", err) + } + + if stats.TotalPackages != 0 || stats.TotalSize != 0 { + t.Errorf("expected empty stats, got packages=%d size=%d", + stats.TotalPackages, stats.TotalSize) + } +} diff --git a/internal/build/collect.go b/internal/build/collect.go new file mode 100644 index 0000000000..e38d90602d --- /dev/null +++ b/internal/build/collect.go @@ -0,0 +1,489 @@ +/* + * Copyright (c) 2024 The GoPlus Authors (goplus.org). All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package build + +import ( + "fmt" + "os" + "os/exec" + "path/filepath" + "runtime" + "sort" + "strings" + + "github.com/goplus/llgo/internal/env" + "github.com/goplus/llgo/internal/packages" + gopackages "golang.org/x/tools/go/packages" +) + +// collectFingerprint collects all inputs and generates fingerprint for a package. +func (c *context) collectFingerprint(pkg *aPackage) error { + if pkg.Manifest != "" && pkg.Fingerprint != "" { + return nil + } + if c.fingerprinting == nil { + c.fingerprinting = make(map[string]bool) + } + if c.fingerprinting[pkg.ID] { + return fmt.Errorf("fingerprint cycle detected for %s", pkg.ID) + } + c.fingerprinting[pkg.ID] = true + defer delete(c.fingerprinting, pkg.ID) + + m := newManifestBuilder() + + // Env section + c.collectEnvInputs(m) + + // Common section + c.collectCommonInputs(m) + + // Package section + if err := c.collectPackageInputs(m, pkg); err != nil { + return err + } + + // Dependency section + if err := c.collectDependencyInputs(m, pkg); err != nil { + return err + } + + pkg.Manifest = m.Build() + pkg.Fingerprint = m.Fingerprint() + return nil +} + +// collectEnvInputs collects environment-related inputs. +func (c *context) collectEnvInputs(m *manifestBuilder) { + m.env.Goos = c.buildConf.Goos + m.env.Goarch = c.buildConf.Goarch + m.env.LlvmTriple = c.crossCompile.LLVMTarget + m.env.LlgoVersion = env.Version() + m.env.GoVersion = runtime.Version() + m.env.LlvmVersion = c.getLLVMVersion() + + // Environment variables that affect build + envVars := []string{ + llgoDebug, + llgoDbgSyms, + llgoTrace, + llgoOptimize, + llgoWasmRuntime, + llgoWasiThreads, + llgoStdioNobuf, + llgoFullRpath, + } + for _, envVar := range envVars { + if v := os.Getenv(envVar); v != "" { + m.env.Vars = m.env.Vars.Add(envVar, v) + } + } +} + +// collectCommonInputs collects common build configuration inputs. +func (c *context) collectCommonInputs(m *manifestBuilder) { + m.common.AbiMode = fmt.Sprintf("%d", c.buildConf.AbiMode) + if c.buildConf.Tags != "" { + m.common.BuildTags = strings.Split(c.buildConf.Tags, ",") + } + m.common.Target = c.buildConf.Target + m.common.TargetABI = c.crossCompile.TargetABI + + // Compiler configuration + if c.crossCompile.CC != "" { + m.common.CC = c.crossCompile.CC + } + if len(c.crossCompile.CCFLAGS) > 0 { + m.common.CCFlags = append([]string(nil), c.crossCompile.CCFLAGS...) + } + if len(c.crossCompile.CFLAGS) > 0 { + m.common.CFlags = append([]string(nil), c.crossCompile.CFLAGS...) + } + if len(c.crossCompile.LDFLAGS) > 0 { + m.common.LDFlags = append([]string(nil), c.crossCompile.LDFLAGS...) + } + if c.crossCompile.Linker != "" { + m.common.Linker = c.crossCompile.Linker + } + + // Extra files from target configuration + if len(c.crossCompile.ExtraFiles) > 0 { + extraList, err := digestFiles(c.crossCompile.ExtraFiles) + if err == nil && len(extraList) > 0 { + m.common.ExtraFiles = extraList + } + } +} + +// collectPackageInputs collects package-specific inputs. +func (c *context) collectPackageInputs(m *manifestBuilder, pkg *aPackage) error { + p := pkg.Package + + m.pkg.PkgPath = p.PkgPath + m.pkg.PkgID = p.ID + + // Go source files + goFilesList, err := digestFilesWithOverlay(p.GoFiles, c.conf.Overlay) + if err != nil { + return fmt.Errorf("digest go files: %w", err) + } + m.pkg.GoFiles = goFilesList + + // Alt package files (if any) + if pkg.AltPkg != nil { + altList, err := digestFilesWithOverlay(pkg.AltPkg.GoFiles, c.conf.Overlay) + if err != nil { + return fmt.Errorf("digest alt go files: %w", err) + } + m.pkg.AltGoFiles = altList + } + + // Other files (C, assembly, etc.) + if len(p.OtherFiles) > 0 { + otherList, err := digestFiles(p.OtherFiles) + if err != nil { + return fmt.Errorf("digest other files: %w", err) + } + m.pkg.OtherFiles = otherList + } + + // Rewrite vars + if len(pkg.rewriteVars) > 0 { + rewrites := make(map[string]string, len(pkg.rewriteVars)) + for k, v := range pkg.rewriteVars { + rewrites[k] = v + } + m.pkg.RewriteVars = m.pkg.RewriteVars.AddMap(rewrites) + } + + // Add metadata fields if available (for cache saving) + // (LINK_ARGS/NEED_RT/NEED_PY_INIT are appended later in saveToCache) + + return nil +} + +// collectDependencyInputs adds dependency fingerprints/versions into manifest. +func (c *context) collectDependencyInputs(m *manifestBuilder, pkg *aPackage) error { + if len(pkg.Imports) == 0 { + return nil + } + + deps := make([]*packages.Package, 0, len(pkg.Imports)) + for _, dep := range pkg.Imports { + if dep == nil || dep.ID == pkg.ID { + continue + } + deps = append(deps, dep) + } + + sort.Slice(deps, func(i, j int) bool { return deps[i].ID < deps[j].ID }) + + for _, dep := range deps { + depEntry, err := c.dependencyFingerprint(dep) + if err != nil { + return err + } + m.deps = append(m.deps, depEntry) + } + + return nil +} + +func (c *context) dependencyFingerprint(dep *packages.Package) (depEntry, error) { + entry := depEntry{ID: dep.ID} + if v := moduleVersion(dep.Module); v != "" { + entry.Version = v + return entry, nil + } + + if c.pkgByID != nil { + if aDep, ok := c.pkgByID[dep.ID]; ok { + if aDep.Fingerprint == "" { + if err := c.collectFingerprint(aDep); err != nil { + return entry, fmt.Errorf("collect fingerprint for %s: %w", dep.ID, err) + } + } + entry.Fingerprint = aDep.Fingerprint + return entry, nil + } + } + + temp := &aPackage{Package: dep} + if err := c.collectFingerprint(temp); err != nil { + return entry, fmt.Errorf("collect fingerprint for %s: %w", dep.ID, err) + } + entry.Fingerprint = temp.Fingerprint + return entry, nil +} + +func moduleVersion(mod *gopackages.Module) string { + if mod == nil { + return "" + } + if mod.Replace != nil { + // replace to local path (Version empty) should not use version for fingerprint + if mod.Replace.Version != "" { + return mod.Replace.Version + } + return "" + } + return mod.Version +} + +// getLLVMVersion returns the cached LLVM version or detects it. +func (c *context) getLLVMVersion() string { + if c.llvmVersion != "" { + return c.llvmVersion + } + c.llvmVersion = detectLLVMVersion(c) + return c.llvmVersion +} + +// detectLLVMVersion detects LLVM version from clang --version. +func detectLLVMVersion(ctx *context) string { + // Get compiler path from cross compile config + cc := ctx.crossCompile.CC + if cc == "" { + cc = "clang" + } + versionCmd := exec.Command(cc, "--version") + output, err := versionCmd.Output() + if err != nil { + return "" + } + line := string(output) + if idx := strings.IndexByte(line, '\n'); idx >= 0 { + line = line[:idx] + } + return strings.TrimSpace(line) +} + +// targetTriple returns the target triple for cache directory. +func (c *context) targetTriple() string { + return targetTriple( + c.buildConf.Goos, + c.buildConf.Goarch, + c.crossCompile.LLVMTarget, + c.crossCompile.TargetABI, + ) +} + +// targetTriple returns the target triple string for cache directory +func targetTriple(goos, goarch, llvmTarget, targetABI string) string { + triple := llvmTarget + if triple == "" { + triple = fmt.Sprintf("%s-%s", goarch, goos) + } + if targetABI != "" { + triple = triple + "-" + targetABI + } + return triple +} + +// ensureCacheManager creates cacheManager if not exists. +func (c *context) ensureCacheManager() *cacheManager { + if c.cacheManager == nil { + c.cacheManager = newCacheManager() + } + return c.cacheManager +} + +// tryLoadFromCache attempts to load a package from cache. +// Returns true if cache hit, false otherwise. +func (c *context) tryLoadFromCache(pkg *aPackage) bool { + if !cacheEnabled() { + return false + } + + if pkg.Fingerprint == "" { + return false + } + + cm := c.ensureCacheManager() + paths := cm.PackagePaths(c.targetTriple(), pkg.PkgPath, pkg.Fingerprint) + + // Check if archive file exists + if _, err := os.Stat(paths.Archive); err != nil { + return false + } + + // Read metadata from manifest + content, err := readManifest(paths.Manifest) + if err != nil { + return false + } + + // Parse metadata from manifest [Package] section (INI format) + meta, err := parseManifestMetadata(content) + if err != nil { + return false + } + + // Use the .a archive directly for linking (no extraction needed) + pkg.LLFiles = []string{paths.Archive} + pkg.LinkArgs = meta.LinkArgs + pkg.NeedRt = meta.NeedRt + pkg.NeedPyInit = meta.NeedPyInit + pkg.CacheHit = true + + return true +} + +// parseManifestMetadata extracts metadata from manifest content. +// It supports the new YAML format and falls back to the legacy INI layout for +// backward compatibility with existing cache entries. +func parseManifestMetadata(content string) (*cacheArchiveMetadata, error) { + meta := &cacheArchiveMetadata{} + if data, err := decodeManifest(content); err == nil { + if data.Metadata != nil { + meta.LinkArgs = append([]string(nil), data.Metadata.LinkArgs...) + meta.NeedRt = data.Metadata.NeedRt + meta.NeedPyInit = data.Metadata.NeedPyInit + } + return meta, nil + } + + return parseManifestMetadataLegacy(content, meta) +} + +func parseManifestMetadataLegacy(content string, meta *cacheArchiveMetadata) (*cacheArchiveMetadata, error) { + // Find Package section + idx := strings.Index(content, "[Package]\n") + if idx == -1 { + return meta, nil + } + + // Extract Package section content (until next section or end) + pkgSection := content[idx+len("[Package]\n"):] + if nextIdx := strings.Index(pkgSection, "\n["); nextIdx != -1 { + pkgSection = pkgSection[:nextIdx] + } + + // Parse key-value pairs in Package section + lines := strings.Split(pkgSection, "\n") + for _, line := range lines { + line = strings.TrimSpace(line) + if line == "" { + continue + } + parts := strings.SplitN(line, " = ", 2) + if len(parts) != 2 { + continue + } + key, value := strings.TrimSpace(parts[0]), strings.TrimSpace(parts[1]) + + switch key { + case "LINK_ARGS": + if value != "" { + meta.LinkArgs = strings.Fields(value) + } + case "NEED_RT": + meta.NeedRt = value == "true" + case "NEED_PY_INIT": + meta.NeedPyInit = value == "true" + } + } + + return meta, nil +} + +// cacheArchiveMetadata holds metadata about a cached archive. +type cacheArchiveMetadata struct { + LinkArgs []string + NeedRt bool + NeedPyInit bool +} + +// saveToCache saves a built package to cache. +func (c *context) saveToCache(pkg *aPackage) error { + if !cacheEnabled() { + return nil + } + + if pkg.Fingerprint == "" || pkg.Manifest == "" { + return nil + } + + // Don't cache main packages + if pkg.Name == "main" { + return nil + } + + cm := c.ensureCacheManager() + paths := cm.PackagePaths(c.targetTriple(), pkg.PkgPath, pkg.Fingerprint) + + // Ensure directory exists + if err := cm.EnsureDir(paths); err != nil { + return err + } + + // Collect object files to cache + // Deduplicate by full path first + var objectFiles []string + seenPath := make(map[string]bool) + for _, f := range pkg.LLFiles { + if filepath.Ext(f) == ".o" || filepath.Ext(f) == ".ll" { + if !seenPath[f] { + seenPath[f] = true + objectFiles = append(objectFiles, f) + } + } + } + + if len(objectFiles) == 0 { + return nil + } + + // Create .a archive from object files (atomic write to avoid races) + if err := createArchiveFile(paths.Archive, objectFiles); err != nil { + return err + } + + // Append metadata to existing manifest (pkg.Manifest was built in collectFingerprint). + manifestContent := pkg.Manifest + if manifestContent == "" { + return fmt.Errorf("package %s missing manifest for fingerprint %s", pkg.PkgPath, pkg.Fingerprint) + } + + data, err := decodeManifest(manifestContent) + if err != nil { + return fmt.Errorf("decode manifest: %w", err) + } + + meta := &manifestMetadata{ + LinkArgs: append([]string(nil), pkg.LinkArgs...), + NeedRt: pkg.NeedRt, + NeedPyInit: pkg.NeedPyInit, + } + if len(meta.LinkArgs) == 0 && !meta.NeedRt && !meta.NeedPyInit { + data.Metadata = nil + } else { + data.Metadata = meta + } + + manifestWithMeta, err := buildManifestYAML(data) + if err != nil { + return err + } + + // Write manifest with metadata + if err := writeManifest(paths.Manifest, manifestWithMeta); err != nil { + return err + } + + return nil +} diff --git a/internal/build/collect_test.go b/internal/build/collect_test.go new file mode 100644 index 0000000000..c35418b1f0 --- /dev/null +++ b/internal/build/collect_test.go @@ -0,0 +1,527 @@ +//go:build !llgo + +/* + * Copyright (c) 2024 The GoPlus Authors (goplus.org). All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package build + +import ( + "os" + "os/exec" + "path/filepath" + "runtime" + "strings" + "testing" + + "github.com/goplus/llgo/internal/crosscompile" + "github.com/goplus/llgo/internal/packages" + gopackages "golang.org/x/tools/go/packages" +) + +func TestCollectFingerprint(t *testing.T) { + td := t.TempDir() + + // Create a test file + goFile := filepath.Join(td, "main.go") + if err := os.WriteFile(goFile, []byte("package main\nfunc main() {}"), 0644); err != nil { + t.Fatal(err) + } + + ctx := &context{ + conf: &packages.Config{}, + buildConf: &Config{ + Goos: "darwin", + Goarch: "arm64", + BuildMode: BuildModeExe, + Tags: "test", + }, + crossCompile: crosscompile.Export{ + LLVMTarget: "arm64-apple-darwin", + }, + } + + pkg := &aPackage{ + Package: &packages.Package{ + PkgPath: "example.com/test", + GoFiles: []string{goFile}, + }, + } + + if err := ctx.collectFingerprint(pkg); err != nil { + t.Fatalf("collectFingerprint: %v", err) + } + + // Check fingerprint is generated + if pkg.Fingerprint == "" { + t.Error("fingerprint should not be empty") + } + if len(pkg.Fingerprint) != 64 { + t.Errorf("fingerprint length = %d, want 64", len(pkg.Fingerprint)) + } + + data, err := decodeManifest(pkg.Manifest) + if err != nil { + t.Fatalf("decodeManifest: %v", err) + } + if data.Env == nil || data.Common == nil || data.Package == nil { + t.Fatal("manifest sections should not be empty") + } + if data.Env.Goos != "darwin" { + t.Error("manifest should contain GOOS = darwin") + } + if data.Package.PkgPath != "example.com/test" { + t.Error("manifest should contain PKG_PATH") + } +} + +func TestCollectFingerprintDeterminism(t *testing.T) { + td := t.TempDir() + + goFile := filepath.Join(td, "main.go") + if err := os.WriteFile(goFile, []byte("package main"), 0644); err != nil { + t.Fatal(err) + } + + ctx := &context{ + conf: &packages.Config{}, + buildConf: &Config{ + Goos: "linux", + Goarch: "amd64", + BuildMode: BuildModeExe, + }, + crossCompile: crosscompile.Export{}, + } + + pkg1 := &aPackage{ + Package: &packages.Package{ + PkgPath: "test/pkg", + GoFiles: []string{goFile}, + }, + } + + pkg2 := &aPackage{ + Package: &packages.Package{ + PkgPath: "test/pkg", + GoFiles: []string{goFile}, + }, + } + + if err := ctx.collectFingerprint(pkg1); err != nil { + t.Fatal(err) + } + if err := ctx.collectFingerprint(pkg2); err != nil { + t.Fatal(err) + } + + if pkg1.Fingerprint != pkg2.Fingerprint { + t.Error("same inputs should produce same fingerprint") + } +} + +func TestCollectFingerprintDependencies(t *testing.T) { + td := t.TempDir() + + depFile := filepath.Join(td, "dep.go") + if err := os.WriteFile(depFile, []byte("package dep"), 0644); err != nil { + t.Fatal(err) + } + mainFile := filepath.Join(td, "main.go") + if err := os.WriteFile(mainFile, []byte("package main"), 0644); err != nil { + t.Fatal(err) + } + + ctx := &context{ + conf: &packages.Config{}, + buildConf: &Config{Goos: "linux", Goarch: "amd64"}, + crossCompile: crosscompile.Export{}, + pkgs: map[*packages.Package]Package{}, + pkgByID: map[string]Package{}, + } + + depPkg := &aPackage{Package: &packages.Package{ + ID: "example.com/dep", + PkgPath: "example.com/dep", + GoFiles: []string{depFile}, + }} + depWithVersion := &aPackage{Package: &packages.Package{ + ID: "example.com/depver", + PkgPath: "example.com/depver", + GoFiles: []string{depFile}, + Module: &gopackages.Module{Path: "example.com/depver", Version: "v1.0.0"}, + }} + ctx.pkgByID[depPkg.ID] = depPkg + ctx.pkgByID[depWithVersion.ID] = depWithVersion + + mainPkg := &aPackage{Package: &packages.Package{ + ID: "example.com/main", + PkgPath: "example.com/main", + GoFiles: []string{mainFile}, + Imports: map[string]*packages.Package{ + "example.com/dep": depPkg.Package, + "example.com/depver": depWithVersion.Package, + }, + }} + + if err := ctx.collectFingerprint(mainPkg); err != nil { + t.Fatalf("collectFingerprint: %v", err) + } + + data, err := decodeManifest(mainPkg.Manifest) + if err != nil { + t.Fatalf("decodeManifest: %v", err) + } + if len(data.Deps) != 2 { + t.Fatalf("expected 2 deps, got %d", len(data.Deps)) + } + var seenFingerprint, seenVersion bool + for _, dep := range data.Deps { + switch dep.ID { + case "example.com/depver": + seenVersion = dep.Version == "v1.0.0" && dep.Fingerprint == "" + case "example.com/dep": + seenFingerprint = dep.Fingerprint == depPkg.Fingerprint && dep.Version == "" + } + } + if !seenVersion { + t.Fatalf("versioned dependency not recorded with version: %+v", data.Deps) + } + if !seenFingerprint { + t.Fatalf("workspace dependency not recorded with fingerprint: %+v", data.Deps) + } +} + +func TestBuildDo_DepFingerprintAndVersion(t *testing.T) { + root := t.TempDir() + mainDir := filepath.Join(root, "main") + depPathDir := filepath.Join(root, "depPath") + depWorkDir := filepath.Join(root, "depWork") + + must(os.MkdirAll(mainDir, 0o755)) + must(os.MkdirAll(depPathDir, 0o755)) + must(os.MkdirAll(depWorkDir, 0o755)) + + writeFile(t, filepath.Join(depPathDir, "go.mod"), "module github.com/matryer/is\n\ngo 1.23\n") + writeFile(t, filepath.Join(depPathDir, "is.go"), "package is\nfunc OK(v bool) bool { return v }\n") + + writeFile(t, filepath.Join(depWorkDir, "go.mod"), "module github.com/pmezard/go-difflib\n\ngo 1.23\n") + must(os.MkdirAll(filepath.Join(depWorkDir, "difflib"), 0o755)) + writeFile(t, filepath.Join(depWorkDir, "difflib", "difflib.go"), "package difflib\nconst Name = \"work\"\n") + + mainMod := `module example.com/main + +go 1.23 + +require ( + github.com/davecgh/go-spew v1.1.1 + github.com/matryer/is v1.4.1 + github.com/pmezard/go-difflib v1.0.0 +) + +replace github.com/davecgh/go-spew v1.1.1 => github.com/davecgh/go-spew v1.1.0 +replace github.com/matryer/is v1.4.1 => ../depPath +replace github.com/pmezard/go-difflib v1.0.0 => ../depWork +` + writeFile(t, filepath.Join(mainDir, "go.mod"), mainMod) + writeFile(t, filepath.Join(mainDir, "main.go"), "package main\nimport (\"github.com/davecgh/go-spew/spew\"\n\"github.com/matryer/is\"\n\"github.com/pmezard/go-difflib/difflib\"\n)\nvar _ = spew.Sdump(is.OK(true)) + difflib.Name\nfunc main() {}\n") + + oldWD, _ := os.Getwd() + goWork := "go 1.24\nuse ./main\nuse ./depWork\nuse ./depPath\n" + writeFile(t, filepath.Join(root, "go.work"), goWork) + cmd := exec.Command("go", "work", "sync") + cmd.Dir = root + if out, err := cmd.CombinedOutput(); err != nil { + t.Fatalf("go work sync: %v\n%s", err, out) + } + + must(os.Chdir(mainDir)) + t.Cleanup(func() { _ = os.Chdir(oldWD) }) + + conf := &Config{Goos: runtime.GOOS, Goarch: runtime.GOARCH, Mode: ModeBuild, BinPath: filepath.Join(root, "bin")} + _ = os.MkdirAll(conf.BinPath, 0o755) + + // Let Go discover workspace via parent go.work (no extra env needed) + + pkgs, err := Do(nil, conf) + if err != nil { + t.Fatalf("Do: %v", err) + } + + mainPkg := findPkg(pkgs, "example.com/main") + if mainPkg == nil { + t.Fatalf("main package not built") + } + data, err := decodeManifest(mainPkg.Manifest) + if err != nil { + t.Fatalf("decodeManifest: %v", err) + } + + get := func(prefix string) *depEntry { + for i := range data.Deps { + if strings.HasPrefix(data.Deps[i].ID, prefix) { + return &data.Deps[i] + } + } + return nil + } + + if dep := get("github.com/davecgh/go-spew"); dep == nil || dep.Version != "v1.1.0" || dep.Fingerprint != "" { + t.Fatalf("version replace expected version only: %+v", dep) + } + if dep := get("github.com/matryer/is"); dep == nil || dep.Version != "" || dep.Fingerprint == "" { + t.Fatalf("relative replace should fingerprint: %+v", dep) + } + if dep := get("github.com/pmezard/go-difflib"); dep == nil || dep.Version != "" || dep.Fingerprint == "" { + t.Fatalf("workspace dep should fingerprint: %+v", dep) + } +} + +func TestTargetTripleMethod(t *testing.T) { + tests := []struct { + name string + ctx *context + expect string + }{ + { + name: "with llvm target", + ctx: &context{ + buildConf: &Config{ + Goos: "darwin", + Goarch: "arm64", + }, + crossCompile: crosscompile.Export{ + LLVMTarget: "arm64-apple-darwin", + }, + }, + expect: "arm64-apple-darwin", + }, + { + name: "without llvm target", + ctx: &context{ + buildConf: &Config{ + Goos: "linux", + Goarch: "amd64", + }, + crossCompile: crosscompile.Export{}, + }, + expect: "amd64-linux", + }, + { + name: "with abi", + ctx: &context{ + buildConf: &Config{ + Goos: "linux", + Goarch: "arm", + }, + crossCompile: crosscompile.Export{ + TargetABI: "gnueabihf", + }, + }, + expect: "arm-linux-gnueabihf", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := tt.ctx.targetTriple() + if got != tt.expect { + t.Errorf("targetTriple() = %q, want %q", got, tt.expect) + } + }) + } +} + +func TestEnsureCacheManager(t *testing.T) { + ctx := &context{ + buildConf: &Config{}, + } + + // First call should create manager + cm1 := ctx.ensureCacheManager() + if cm1 == nil { + t.Fatal("ensureCacheManager returned nil") + } + + // Second call should return same instance + cm2 := ctx.ensureCacheManager() + if cm1 != cm2 { + t.Error("ensureCacheManager should return same instance") + } +} + +func TestTryLoadFromCache_NoFingerprint(t *testing.T) { + ctx := &context{ + buildConf: &Config{}, + } + + pkg := &aPackage{ + Package: &packages.Package{ + PkgPath: "test/pkg", + }, + Fingerprint: "", // No fingerprint + } + + if ctx.tryLoadFromCache(pkg) { + t.Error("should return false when no fingerprint") + } +} + +func TestSaveToCache_MainPackage(t *testing.T) { + td := t.TempDir() + oldFunc := cacheRootFunc + cacheRootFunc = func() string { return td } + defer func() { cacheRootFunc = oldFunc }() + + ctx := &context{ + conf: &packages.Config{}, + buildConf: &Config{ + Goos: "darwin", + Goarch: "arm64", + }, + crossCompile: crosscompile.Export{ + LLVMTarget: "arm64-apple-darwin", + }, + } + + pkg := &aPackage{ + Package: &packages.Package{ + PkgPath: "main", + Name: "main", // Main package + }, + Fingerprint: "abc123", + Manifest: "test manifest", + } + + // Should not error but also should not create cache + if err := ctx.saveToCache(pkg); err != nil { + t.Fatalf("saveToCache: %v", err) + } + + // Check no cache was created + cm := ctx.ensureCacheManager() + paths := cm.PackagePaths("arm64-apple-darwin", "main", "abc123") + if _, err := os.Stat(paths.Manifest); !os.IsNotExist(err) { + t.Error("main package should not be cached") + } +} + +func TestSaveToCache_Success(t *testing.T) { + td := t.TempDir() + oldFunc := cacheRootFunc + cacheRootFunc = func() string { return td } + defer func() { cacheRootFunc = oldFunc }() + + ctx := &context{ + conf: &packages.Config{}, + buildConf: &Config{ + Goos: "darwin", + Goarch: "arm64", + }, + crossCompile: crosscompile.Export{ + LLVMTarget: "arm64-apple-darwin", + }, + } + + // Create a temporary .o file + objFile, err := os.CreateTemp(td, "test-*.o") + if err != nil { + t.Fatalf("CreateTemp: %v", err) + } + objFile.WriteString("fake object file") + objFile.Close() + + pkg := &aPackage{ + Package: &packages.Package{ + PkgPath: "example.com/lib", + Name: "lib", + GoFiles: []string{objFile.Name()}, // Add GoFiles for manifest generation + }, + Fingerprint: "def456", + Manifest: func() string { + m := newManifestBuilder() + m.env.Goos = "darwin" + m.pkg.PkgPath = "example.com/lib" + return m.Build() + }(), + LLFiles: []string{objFile.Name()}, + } + + if err := ctx.saveToCache(pkg); err != nil { + t.Fatalf("saveToCache: %v", err) + } + + // Check cache was created + cm := ctx.ensureCacheManager() + paths := cm.PackagePaths("arm64-apple-darwin", "example.com/lib", "def456") + + // Check manifest contains original content and metadata in Package section + content, err := readManifest(paths.Manifest) + if err != nil { + t.Fatalf("ReadManifest: %v", err) + } + data, err := decodeManifest(content) + if err != nil { + t.Fatalf("decodeManifest: %v", err) + } + if data.Env.Goos != "darwin" { + t.Errorf("manifest should contain original env content") + } + if data.Metadata != nil { + t.Errorf("metadata should be empty when no link args/runtime flags") + } + + // Check archive exists + if _, err := os.Stat(paths.Archive); err != nil { + t.Errorf("archive should exist: %v", err) + } +} + +func TestGetLLVMVersion(t *testing.T) { + ctx := &context{ + crossCompile: crosscompile.Export{}, + } + + // First call should detect version + v1 := ctx.getLLVMVersion() + // May be empty if clang is not installed, but should not panic + + // Second call should return cached version + v2 := ctx.getLLVMVersion() + if v1 != v2 { + t.Error("getLLVMVersion should return cached value") + } +} + +func writeFile(t *testing.T, path, content string) { + t.Helper() + if err := os.WriteFile(path, []byte(content), 0o644); err != nil { + t.Fatalf("write %s: %v", path, err) + } +} + +func must(err error) { + if err != nil { + panic(err) + } +} + +func findPkg(pkgs []Package, pkgPath string) Package { + for _, p := range pkgs { + if p.PkgPath == pkgPath { + return p + } + } + return nil +} diff --git a/internal/build/fingerprint.go b/internal/build/fingerprint.go new file mode 100644 index 0000000000..9134bbbcc0 --- /dev/null +++ b/internal/build/fingerprint.go @@ -0,0 +1,301 @@ +/* + * Copyright (c) 2024 The GoPlus Authors (goplus.org). All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package build + +import ( + "crypto/sha256" + "encoding/hex" + "fmt" + "io" + "os" + "sort" + "strings" + + "gopkg.in/yaml.v3" +) + +// depEntry captures dependency identity plus either version or fingerprint. +type depEntry struct { + ID string `yaml:"id"` + Version string `yaml:"version,omitempty"` + Fingerprint string `yaml:"fingerprint,omitempty"` +} + +// manifestMetadata stores metadata produced during build but not part of the fingerprint. +type manifestMetadata struct { + LinkArgs []string `yaml:"link_args,omitempty"` + NeedRt bool `yaml:"need_rt,omitempty"` + NeedPyInit bool `yaml:"need_py_init,omitempty"` +} + +// manifestData is the structured representation of manifest content. +type manifestData struct { + Env *envSection `yaml:"env,omitempty"` + Common *commonSection `yaml:"common,omitempty"` + Package *packageSection `yaml:"package,omitempty"` + Metadata *manifestMetadata `yaml:"metadata,omitempty"` + Deps []depEntry `yaml:"deps,omitempty"` +} + +// orderedStringMap keeps deterministic order for map[string]string when marshaling. +type orderedStringMap map[string]string + +func (m orderedStringMap) Add(key, val string) orderedStringMap { + if m == nil { + m = make(map[string]string) + } + m[key] = val + return m +} + +func (m orderedStringMap) AddMap(src map[string]string) orderedStringMap { + if len(src) == 0 { + return m + } + if m == nil { + m = make(map[string]string, len(src)) + } + for k, v := range src { + m[k] = v + } + return m +} + +func (m orderedStringMap) MarshalYAML() (interface{}, error) { + if len(m) == 0 { + return nil, nil + } + type kv struct{ K, V string } + list := make([]kv, 0, len(m)) + for k, v := range m { + list = append(list, kv{k, v}) + } + sort.Slice(list, func(i, j int) bool { return list[i].K < list[j].K }) + out := &yaml.Node{Kind: yaml.MappingNode} + for _, item := range list { + out.Content = append(out.Content, + &yaml.Node{Kind: yaml.ScalarNode, Tag: "!!str", Value: item.K}, + &yaml.Node{Kind: yaml.ScalarNode, Tag: "!!str", Value: item.V}, + ) + } + return out, nil +} + +// envSection holds fixed environment fields and optional vars. +type envSection struct { + Goos string `yaml:"GOOS,omitempty"` + Goarch string `yaml:"GOARCH,omitempty"` + GoVersion string `yaml:"GO_VERSION,omitempty"` + LlgoVersion string `yaml:"LLGO_VERSION,omitempty"` + LlvmTriple string `yaml:"LLVM_TRIPLE,omitempty"` + LlvmVersion string `yaml:"LLVM_VERSION,omitempty"` + Vars orderedStringMap `yaml:"VARS,omitempty"` +} + +func (s *envSection) empty() bool { + return s.Goos == "" && s.Goarch == "" && s.LlvmTriple == "" && s.LlgoVersion == "" && s.GoVersion == "" && s.LlvmVersion == "" && len(s.Vars) == 0 +} + +type commonSection struct { + AbiMode string `yaml:"ABI_MODE,omitempty"` + BuildTags []string `yaml:"BUILD_TAGS,omitempty"` + Target string `yaml:"TARGET,omitempty"` + TargetABI string `yaml:"TARGET_ABI,omitempty"` + CC string `yaml:"CC,omitempty"` + CCFlags []string `yaml:"CCFLAGS,omitempty"` + CFlags []string `yaml:"CFLAGS,omitempty"` + LDFlags []string `yaml:"LDFLAGS,omitempty"` + Linker string `yaml:"LINKER,omitempty"` + ExtraFiles []fileDigest `yaml:"EXTRA_FILES,omitempty"` +} + +func (s *commonSection) empty() bool { + return s.AbiMode == "" && len(s.BuildTags) == 0 && s.Target == "" && s.TargetABI == "" && + s.CC == "" && len(s.CCFlags) == 0 && len(s.CFlags) == 0 && len(s.LDFlags) == 0 && s.Linker == "" && len(s.ExtraFiles) == 0 +} + +type packageSection struct { + PkgPath string `yaml:"pkg_path,omitempty"` + PkgID string `yaml:"pkg_id,omitempty"` + GoFiles []fileDigest `yaml:"go_files,omitempty"` + AltGoFiles []fileDigest `yaml:"alt_go_files,omitempty"` + OtherFiles []fileDigest `yaml:"other_files,omitempty"` + RewriteVars orderedStringMap `yaml:"rewrite_vars,omitempty"` +} + +func (s *packageSection) empty() bool { + return s.PkgPath == "" && s.PkgID == "" && len(s.GoFiles) == 0 && len(s.AltGoFiles) == 0 && len(s.OtherFiles) == 0 && len(s.RewriteVars) == 0 +} + +// manifestBuilder builds manifest text with sorted sections. +type manifestBuilder struct { + env envSection + common commonSection + pkg packageSection + deps []depEntry + meta *manifestMetadata +} + +// newManifestBuilder creates a new manifestBuilder. +func newManifestBuilder() *manifestBuilder { + return &manifestBuilder{} +} + +// Build generates the sorted manifest text in INI format. +func (m *manifestBuilder) Build() string { + env := m.env + common := m.common + pkg := m.pkg + + sort.Strings(common.BuildTags) + + data := manifestData{ + Env: &env, + Common: &common, + Package: &pkg, + Deps: sortDeps(m.deps), + Metadata: m.meta, + } + content, _ := buildManifestYAML(data) + return content +} + +// Fingerprint returns the sha256 hash of the manifest content. +func (m *manifestBuilder) Fingerprint() string { + content := m.Build() + hash := sha256.Sum256([]byte(content)) + return hex.EncodeToString(hash[:]) +} + +func sortDeps(deps []depEntry) []depEntry { + if len(deps) == 0 { + return nil + } + sorted := append([]depEntry(nil), deps...) + sort.Slice(sorted, func(i, j int) bool { + if sorted[i].ID == sorted[j].ID { + if sorted[i].Version == sorted[j].Version { + return sorted[i].Fingerprint < sorted[j].Fingerprint + } + return sorted[i].Version < sorted[j].Version + } + return sorted[i].ID < sorted[j].ID + }) + return sorted +} + +func (d manifestData) isEmpty() bool { + return (d.Env == nil || d.Env.empty()) && + (d.Common == nil || d.Common.empty()) && + (d.Package == nil || d.Package.empty()) && + len(d.Deps) == 0 && d.Metadata == nil +} + +func buildManifestYAML(data manifestData) (string, error) { + if data.isEmpty() { + return "", nil + } + out, err := yaml.Marshal(data) + return string(out), err +} + +const maxManifestSize = 10 * 1024 * 1024 // 10MB safety bound + +func decodeManifest(content string) (manifestData, error) { + var data manifestData + trimmed := strings.TrimSpace(content) + if trimmed == "" { + return data, nil + } + if len(trimmed) > maxManifestSize { + return manifestData{}, fmt.Errorf("manifest too large: %d bytes", len(trimmed)) + } + if err := yaml.Unmarshal([]byte(trimmed), &data); err != nil { + return manifestData{}, err + } + return data, nil +} + +// digestFile calculates the sha256 hash of a file. +func digestFile(path string) (string, error) { + f, err := os.Open(path) + if err != nil { + return "", err + } + defer f.Close() + + h := sha256.New() + buf := make([]byte, 32*1024) + if _, err := io.CopyBuffer(h, f, buf); err != nil { + return "", err + } + + return hex.EncodeToString(h.Sum(nil)), nil +} + +// digestBytes calculates the sha256 hash of bytes. +func digestBytes(data []byte) string { + hash := sha256.Sum256(data) + return hex.EncodeToString(hash[:]) +} + +// fileDigest represents a file with its path and metadata. +type fileDigest struct { + Path string `yaml:"path"` + Size int64 `yaml:"size"` + ModTime int64 `yaml:"mtime"` + OverlayHash string `yaml:"overlay_hash,omitempty"` +} + +// digestFiles calculates digests for multiple files. +func digestFiles(paths []string) ([]fileDigest, error) { + return digestFilesWithOverlay(paths, nil) +} + +// digestFilesWithOverlay calculates digests for files, using overlay content when available. +func digestFilesWithOverlay(paths []string, overlay map[string][]byte) ([]fileDigest, error) { + if len(paths) == 0 { + return nil, nil + } + + digests := make([]fileDigest, 0, len(paths)) + for _, path := range paths { + if content, ok := overlay[path]; ok { + fd := fileDigest{ + Path: path, + Size: int64(len(content)), + ModTime: 0, + } + fd.OverlayHash = digestBytes(content) + digests = append(digests, fd) + continue + } + info, err := os.Stat(path) + if err != nil { + return nil, fmt.Errorf("stat file %q: %w", path, err) + } + digests = append(digests, fileDigest{ + Path: path, + Size: info.Size(), + ModTime: info.ModTime().UnixNano(), + }) + } + + sort.Slice(digests, func(i, j int) bool { return digests[i].Path < digests[j].Path }) + + return digests, nil +} diff --git a/internal/build/fingerprint_test.go b/internal/build/fingerprint_test.go new file mode 100644 index 0000000000..93b76c38ad --- /dev/null +++ b/internal/build/fingerprint_test.go @@ -0,0 +1,323 @@ +//go:build !llgo + +/* + * Copyright (c) 2024 The GoPlus Authors (goplus.org). All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package build + +import ( + "os" + "path/filepath" + "reflect" + "sort" + "strings" + "testing" +) + +func TestManifestBuilder_Build(t *testing.T) { + m := newManifestBuilder() + m.env.Goarch = "arm64" + m.env.Goos = "darwin" + m.common.AbiMode = "2" + m.pkg.PkgPath = "example.com/foo" + + content := m.Build() + data, err := decodeManifest(content) + if err != nil { + t.Fatalf("decodeManifest: %v", err) + } + + if data.Env.Goarch != "arm64" || data.Env.Goos != "darwin" { + t.Fatalf("env section mismatch: %+v", data.Env) + } + if data.Common.AbiMode != "2" { + t.Fatalf("common section mismatch: %+v", data.Common) + } + if data.Package.PkgPath != "example.com/foo" { + t.Fatalf("package section mismatch: %+v", data.Package) + } +} + +func TestManifestBuilder_BuildSorting(t *testing.T) { + m := newManifestBuilder() + // Add in reverse order + m.env.Vars = orderedStringMap{"Z_KEY": "z", "A_KEY": "a", "M_KEY": "m"} + m.pkg.RewriteVars = orderedStringMap{"Z": "1", "A": "2"} + + content := m.Build() + data, err := decodeManifest(content) + if err != nil { + t.Fatalf("decodeManifest: %v", err) + } + // env vars are sorted by key + keys := []string{} + for k := range data.Env.Vars { + keys = append(keys, k) + } + sort.Strings(keys) + if strings.Join(keys, ",") != "A_KEY,M_KEY,Z_KEY" { + t.Fatalf("env vars not sorted: %v", keys) + } + rvKeys := []string{} + for k := range data.Package.RewriteVars { + rvKeys = append(rvKeys, k) + } + sort.Strings(rvKeys) + if strings.Join(rvKeys, ",") != "A,Z" { + t.Fatalf("rewrite vars not sorted: %v", rvKeys) + } +} + +func TestManifestBuilder_Fingerprint(t *testing.T) { + m := newManifestBuilder() + m.env.Goos = "linux" + m.env.Goarch = "amd64" + m.pkg.PkgPath = "test/pkg" + + fp1 := m.Fingerprint() + fp2 := m.Fingerprint() + + if fp1 != fp2 { + t.Error("fingerprint not stable") + } + + // Fingerprint should be 64 hex characters (sha256) + if len(fp1) != 64 { + t.Errorf("fingerprint length = %d, want 64", len(fp1)) + } +} + +func TestManifestBuilder_FingerprintDeterminism(t *testing.T) { + // Different order of adding, same fingerprint + m1 := newManifestBuilder() + m1.env.Vars = orderedStringMap{"A": "1", "B": "2"} + m1.common.BuildTags = []string{"10", "20"} + + m2 := newManifestBuilder() + m2.env.Vars = orderedStringMap{"B": "2", "A": "1"} + m2.common.BuildTags = []string{"20", "10"} + + if m1.Fingerprint() != m2.Fingerprint() { + t.Error("order should not affect fingerprint") + } + + if m1.Build() != m2.Build() { + t.Error("order should not affect build output") + } +} + +func TestManifestBuilder_FingerprintDifferentValues(t *testing.T) { + m1 := newManifestBuilder() + m1.env.Vars = orderedStringMap{"KEY": "value1"} + + m2 := newManifestBuilder() + m2.env.Vars = orderedStringMap{"KEY": "value2"} + + if m1.Fingerprint() == m2.Fingerprint() { + t.Error("different values should produce different fingerprints") + } +} + +func TestManifestBuilder_EmptySections(t *testing.T) { + m := newManifestBuilder() + content := m.Build() + + // Empty sections should not be written + expected := `` + if content != expected { + t.Errorf("unexpected empty manifest:\ngot:\n%s\nwant:\n%s", content, expected) + } + + // Should still produce a valid fingerprint + fp := m.Fingerprint() + if len(fp) != 64 { + t.Errorf("fingerprint length = %d, want 64", len(fp)) + } +} + +func TestDigestBytes(t *testing.T) { + data := []byte("hello world") + hash := digestBytes(data) + + // Known sha256 of "hello world" + expected := "b94d27b9934d3e08a52e52d7da7dabfac484efe37a5380ee9088f7ace2efcde9" + if hash != expected { + t.Errorf("digestBytes = %s, want %s", hash, expected) + } + + // Empty data + emptyHash := digestBytes([]byte{}) + if len(emptyHash) != 64 { + t.Errorf("empty hash length = %d, want 64", len(emptyHash)) + } +} + +func TestDigestFiles(t *testing.T) { + td := t.TempDir() + + // Create test files + file1 := filepath.Join(td, "a.go") + file2 := filepath.Join(td, "b.go") + content1 := []byte("package a") + content2 := []byte("package b") + + if err := os.WriteFile(file1, content1, 0644); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(file2, content2, 0644); err != nil { + t.Fatal(err) + } + + // Test with files in reverse order (should be sorted) + list, err := digestFiles([]string{file2, file1}) + if err != nil { + t.Fatalf("digestFiles: %v", err) + } + + // Should be sorted by path + if len(list) != 2 || list[0].Path != file1 || list[1].Path != file2 { + t.Errorf("structured list not sorted: %+v", list) + } + info1, err := os.Stat(file1) + if err != nil { + t.Fatal(err) + } + info2, err := os.Stat(file2) + if err != nil { + t.Fatal(err) + } + if list[0].Size != info1.Size() || list[0].ModTime != info1.ModTime().UnixNano() { + t.Errorf("metadata mismatch for %s: %+v vs %v", file1, list[0], info1) + } + if list[1].Size != info2.Size() || list[1].ModTime != info2.ModTime().UnixNano() { + t.Errorf("metadata mismatch for %s: %+v vs %v", file2, list[1], info2) + } +} + +func TestDigestFiles_Empty(t *testing.T) { + list, err := digestFiles([]string{}) + if err != nil { + t.Fatalf("digestFiles: %v", err) + } + if list != nil { + t.Errorf("digestFiles empty list = %#v, want nil", list) + } +} + +func TestDigestFiles_Determinism(t *testing.T) { + td := t.TempDir() + + file1 := filepath.Join(td, "x.go") + file2 := filepath.Join(td, "y.go") + if err := os.WriteFile(file1, []byte("x"), 0644); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(file2, []byte("y"), 0644); err != nil { + t.Fatal(err) + } + + // Different order should produce same result + result1, _ := digestFiles([]string{file1, file2}) + result2, _ := digestFiles([]string{file2, file1}) + + if !reflect.DeepEqual(result1, result2) { + t.Errorf("order should not affect digestFiles result: %+v vs %+v", result1, result2) + } +} + +func TestDigestFilesWithOverlay(t *testing.T) { + td := t.TempDir() + + // Create a real file + realFile := filepath.Join(td, "real.go") + realContent := []byte("real content") + if err := os.WriteFile(realFile, realContent, 0644); err != nil { + t.Fatal(err) + } + + // Create overlay + overlayFile := filepath.Join(td, "overlay.go") + overlayContent := []byte("overlay content") + overlay := map[string][]byte{ + overlayFile: overlayContent, + } + + list, err := digestFilesWithOverlay([]string{overlayFile, realFile}, overlay) + if err != nil { + t.Fatalf("digestFilesWithOverlay: %v", err) + } + + hashOverlay := digestBytes(overlayContent) + + // Should be sorted by path + if len(list) != 2 { + t.Fatalf("expected 2 digests, got %d", len(list)) + } + + if list[0].Path != overlayFile || list[0].OverlayHash != hashOverlay || list[0].Size != int64(len(overlayContent)) || list[0].ModTime != 0 { + t.Errorf("overlay digest mismatch: %+v", list[0]) + } + infoReal, err := os.Stat(realFile) + if err != nil { + t.Fatal(err) + } + if list[1].Path != realFile || list[1].Size != infoReal.Size() || list[1].ModTime != infoReal.ModTime().UnixNano() { + t.Errorf("real file digest mismatch: %+v vs %v", list[1], infoReal) + } +} + +func TestDigestFilesWithOverlay_Empty(t *testing.T) { + list, err := digestFilesWithOverlay([]string{}, nil) + if err != nil { + t.Fatalf("digestFilesWithOverlay: %v", err) + } + if list != nil { + t.Errorf("digestFilesWithOverlay empty list = %#v, want nil", list) + } +} + +func TestManifestBuilder_SpecialCharacters(t *testing.T) { + m := newManifestBuilder() + m.env.Vars = orderedStringMap{"PATH": "/usr/bin:/usr/local/bin"} + m.pkg.RewriteVars = orderedStringMap{"FLAGS": "-X main.version=1.0.0 -ldflags=-s"} + + content := m.Build() + data, err := decodeManifest(content) + if err != nil { + t.Fatalf("decodeManifest: %v", err) + } + if data.Env == nil || data.Env.Vars["PATH"] != "/usr/bin:/usr/local/bin" { + t.Errorf("PATH not preserved: %+v", data.Env) + } + if data.Package == nil || data.Package.RewriteVars["FLAGS"] != "-X main.version=1.0.0 -ldflags=-s" { + t.Errorf("FLAGS not preserved: %+v", data.Package) + } +} + +func TestManifestBuilder_MultipleValues(t *testing.T) { + m := newManifestBuilder() + m.env.Vars = orderedStringMap{"KEY": "value1"} + m.env.Vars = m.env.Vars.Add("KEY", "value2") + + content := m.Build() + data, err := decodeManifest(content) + if err != nil { + t.Fatalf("decodeManifest: %v", err) + } + if data.Env == nil || data.Env.Vars["KEY"] != "value2" { + t.Fatalf("duplicate env should keep last value, got %+v", data.Env.Vars) + } +} diff --git a/internal/mockable/mockable.go b/internal/mockable/mockable.go index e9ca7fe304..e1089ef199 100644 --- a/internal/mockable/mockable.go +++ b/internal/mockable/mockable.go @@ -18,6 +18,11 @@ func EnableMock() { } } +// DisableMock restores the default os.Exit behavior +func DisableMock() { + exitFunc = os.Exit +} + // Exit calls the current exit function func Exit(code int) { exitFunc(code)