Include esbuild files besides the handler

[ewsbr]

Issue description

I'm currently migrating from Serverless V3 to V4, and one of the blockers today seems to be the inability to include files besides the handler in the esbuild package. This StackOverflow user has a similar issue.

I have the following configuration:

const fs = require('fs');
const { copy } = require('esbuild-plugin-copy');

// inspired by https://github.com/evanw/esbuild/issues/1685
const excludeVendorFromSourceMap = (includes = []) => ({
  name: 'excludeVendorFromSourceMap',
  setup(build) {
    const emptySourceMap = '\n//# sourceMappingURL=data:application/json;base64,' + Buffer.from(JSON.stringify({
      version  : 3,
      sources  : [''],
      mappings : 'A'
    })).toString('base64');

    build.onLoad({ filter: /node_modules.+\.(js|ts)$/ }, (args) => {
      return {
        contents: `${fs.readFileSync(args.path, 'utf8')}\n//# sourceMappingURL=data:application/json;base64,${emptySourceMap}`,
        loader: 'default'
      };
    });
  }
});

/** @type {() => import('esbuild').BuildOptions} */
module.exports = () => ({
  bundle    : true,
  minify    : false,
  sourcemap : true,
  format    : 'cjs',
  exclude   : [
    'better-sqlite3',
    'mysql2',
    'mysql',
    'tedious',
    'sqlite3',
    'pg-query-stream',
    'oracledb',
    'canvas'
  ],
  loader: {
    '.json': 'copy',
  },
  plugins: [
    excludeVendorFromSourceMap(),
    copy({
      resolveFrom: 'cwd',
      assets: [
        {
          from: './src/api/locales/*.json',
          to: './.serverless/build/src/locales/',
        },
        {
          from: './src/api/locales/*.json',
          to: './.serverless/build/src/workers/locales/',
        },
        {
          from: './src/assets/**',
          to: './.serverless/build/src/assets/',
        },
        {
          from: './src/assets/**',
          to: './.serverless/build/src/workers/assets/',
        }
      ],
    }),
  ]
});

Output inside .serverless:

build
  - locales
    - X.json
  - api-XXXXXXX.json
  - db-XXXXXXX.json
  - handler.js
  - handler.js.map

Some JSON and other asset files are manually copied via esbuild (locales, for example). This used to work fine with serverless-esbuild, however when I switch to the native esbuild plugin, they're no longer copied.

Is there a way to ensure everything from the build output is copied? I've attempted to add include patterns to serverless.yml to no avail:

package:
  individually: true
  excludeDevDependencies: true
  exclude:
    - .vscode/**
    - .git/**
    - .gitignore
  patterns:
    - 'src/api/locales/**'
    - '.serverless/build/src/**' # The folder gets included, but instead of being inside /src/* it keeps the .serverless path

What approach should I take from here?
From what I've seen, the _package and _packageAll functions in the native plugin don't support this. I was thinking about just monkey-patching them with another plugin, but it doesn't seem optimal.

Context

No response

[ewsbr]

I solved this by monkey-patching the _package and _packageAll functions with another plugin. They now include everything in the src directory.

    const injectCustomAssets = (zip, serviceDir, log) => {
      const buildSrcPath = path.join(serviceDir, '.serverless', 'build', 'src');
      if (existsSync(buildSrcPath)) {
        zip.directory(buildSrcPath, 'src');
      }
    };

I then imported it in serverless.yml:

plugins:
  - serverless-offline
  - serverless-prune-plugin
  - serverless-plugin-ifelse
  - ./v4.js # this

What I don't understand is why all files from the esbuild output aren't included by default in the zip files.
If anyone has a better solution, please let me know.

Full plugin

'use strict';

const path = require('path');
const { createWriteStream, existsSync } = require('fs');
const { stat } = require('fs').promises;
const archiver = require('archiver');
const globby = require('globby');
const _ = require('lodash');
const pLimit = require('p-limit');

class EsbuildMonkeyPatchPlugin {
  constructor(serverless, options) {
    this.serverless = serverless;
    this.options = options ?? {};
    this.hooks = {
      'before:package:createDeploymentArtifacts': this.ensurePatchApplied.bind(this),
    };

    this.patchApplied = false;
  }

  async asyncInit() {
    await this.patchEsbuildPlugin();
  }

  async ensurePatchApplied() {
    if (!this.patchApplied) {
      await this.patchEsbuildPlugin();
    }
  }

  async patchEsbuildPlugin() {
    if (this.patchApplied) return;

    const esbuildPlugin = this.serverless.pluginManager.getPlugins().find((p) => {
      return p.constructor && p.constructor.name === 'Esbuild';
    });

    if (!esbuildPlugin) {
      return;
    }

    this.serverless.cli.log('[EsbuildPatch] Found native Esbuild plugin. Applying monkey patch...');

    const injectCustomAssets = (zip, serviceDir, log) => {
      const buildSrcPath = path.join(serviceDir, '.serverless', 'build', 'src');
      if (existsSync(buildSrcPath)) {
        zip.directory(buildSrcPath, 'src');
      }
    };

    esbuildPlugin._package = async function (handlerPropertyName = 'handler') {
      const functions = await this.functions(handlerPropertyName);
      const buildProperties = await this._buildProperties();

      if (Object.keys(functions).length === 0) {
        this.serverless.cli.log('[EsbuildPatch] No functions to package.');
        return;
      }

      if (!this.serverless?.service?.package?.individually) {
        await this._packageAll(functions, handlerPropertyName);
        return;
      }

      const concurrency = buildProperties.buildConcurrency ?? Object.keys(functions).length;
      const limit = pLimit(concurrency);

      await this.serverless.pluginManager.spawn('esbuild-package');

      const packageIncludes = await globby(
        this.serverless.service.package?.patterns ?? [],
        { cwd: this.serverless.serviceDir }
      );

      const zipPromises = Object.entries(functions).map(([functionAlias, functionObject]) => {
        return limit(async () => {
          const zipName = `${this.serverless.service.service}-${functionAlias}.zip`;
          const zipPath = path.join(
            this.serverless.config.serviceDir,
            '.serverless',
            'build',
            zipName
          );

          const zip = archiver.create('zip');
          const output = createWriteStream(zipPath);

          const zipPromise = new Promise(async (resolve, reject) => {
            output.on('close', () => resolve(zipPath));
            output.on('error', (err) => reject(err));

            output.on('open', async () => {
              const functionIncludes = await globby(
                functionObject.package?.patterns ?? [],
                { cwd: this.serverless.serviceDir }
              );

              const includesToPackage = _.union(packageIncludes, functionIncludes);

              zip.pipe(output);

              const functionName = path.extname(functionObject[handlerPropertyName]).slice(1);
              const handlerPath = functionObject[handlerPropertyName].replace(`.${functionName}`, '');
              const packageJsonPath = path.join(this.serverless.config.serviceDir, '.serverless', 'build', 'package.json');

              if (existsSync(packageJsonPath)) {
                zip.file(packageJsonPath, { name: `package.json` });
              }

              const handlerZipPath = path.join(this.serverless.config.serviceDir, '.serverless', 'build', handlerPath + '.js');
              zip.file(handlerZipPath, { name: `${handlerPath}.js` });

              if (existsSync(`${handlerZipPath}.map`)) {
                zip.file(`${handlerZipPath}.map`, { name: `${handlerPath}.js.map` });
              }

              zip.directory(
                path.join(this.serverless.config.serviceDir, '.serverless', 'build', 'node_modules'),
                'node_modules'
              );

              // --- PATCH START ---
              injectCustomAssets(zip, this.serverless.config.serviceDir, this.serverless.cli.log.bind(this.serverless.cli));
              // --- PATCH END ---

              await Promise.all(
                includesToPackage.map(async (filePath) => {
                  const absolutePath = path.join(this.serverless.config.serviceDir, filePath);
                  const stats = await stat(absolutePath);
                  if (stats.isDirectory()) {
                    zip.directory(absolutePath, filePath);
                  } else {
                    zip.file(absolutePath, { name: filePath });
                  }
                })
              );

              await zip.finalize();
              functionObject.package = { artifact: zipPath };
            });
          });
          await zipPromise;
        });
      });

      try {
        await Promise.all(zipPromises);
      } catch (err) {
        throw new Error(`[EsbuildPatch] Package Error: ${err.message}`);
      }
    };

    esbuildPlugin._packageAll = async function (functions, handlerPropertyName = 'handler') {
      const zipName = `${this.serverless.service.service}.zip`;
      const zipPath = path.join(
        this.serverless.config.serviceDir,
        '.serverless',
        'build',
        zipName
      );

      await this.serverless.pluginManager.spawn('esbuild-package');

      const packageIncludes = await globby(
        this.serverless.service.package.patterns ?? [],
        { cwd: this.serverless.serviceDir }
      );

      const zip = archiver.create('zip');
      const output = createWriteStream(zipPath);
      const addedFiles = new Set();

      const zipPromise = new Promise(async (resolve, reject) => {
        output.on('close', () => resolve(zipPath));
        output.on('error', (err) => reject(err));

        output.on('open', async () => {
          zip.pipe(output);

          for (const [, functionObject] of Object.entries(functions)) {
            const functionName = path.extname(functionObject[handlerPropertyName]).slice(1);
            const handlerPath = functionObject[handlerPropertyName].replace(`.${functionName}`, '');
            const packageJsonPath = path.join(this.serverless.config.serviceDir, '.serverless', 'build', 'package.json');

            if (existsSync(packageJsonPath) && !addedFiles.has(packageJsonPath)) {
              zip.file(packageJsonPath, { name: `package.json` });
              addedFiles.add(packageJsonPath);
            }

            const handlerZipPath = path.join(this.serverless.config.serviceDir, '.serverless', 'build', handlerPath + '.js');

            if (!addedFiles.has(handlerZipPath)) {
              zip.file(handlerZipPath, { name: `${handlerPath}.js` });
              addedFiles.add(handlerZipPath);
            }

            if (existsSync(`${handlerZipPath}.map`) && !addedFiles.has(`${handlerZipPath}.map`)) {
              zip.file(`${handlerZipPath}.map`, { name: `${handlerPath}.js.map` });
              addedFiles.add(`${handlerZipPath}.map`);
            }
          }

          zip.directory(
            path.join(this.serverless.config.serviceDir, '.serverless', 'build', 'node_modules'),
            'node_modules'
          );

          // --- PATCH START ---
          injectCustomAssets(zip, this.serverless.config.serviceDir, this.serverless.cli.log.bind(this.serverless.cli));
          // --- PATCH END ---

          await Promise.all(
            packageIncludes.map(async (filePath) => {
              const absolutePath = path.join(this.serverless.serviceDir, filePath);
              const stats = await stat(absolutePath);
              if (stats.isDirectory()) {
                zip.directory(absolutePath, filePath);
              } else if (!addedFiles.has(absolutePath)) {
                zip.file(absolutePath, { name: filePath });
                addedFiles.add(absolutePath);
              }
            })
          );

          await zip.finalize();
          this.serverless.service.package.artifact = zipPath;
        });
      });

      try {
        await zipPromise;
      } catch (err) {
        throw new Error(`[EsbuildPatch] PackageAll Error: ${err.message}`);
      }
    };

    this.patchApplied = true;
    this.serverless.cli.log('[EsbuildPatch] Patch successfully applied.', 'green');
  }
}

module.exports = EsbuildMonkeyPatchPlugin;