perf(NODE-5875): use builtin native crypto

.github/scripts/libmongocrypt.mjs

@@ -1,3 +1,4 @@
+// @ts-check
 import util from 'node:util';
 import process from 'node:process';
 import fs from 'node:fs/promises';
@@ -23,6 +24,8 @@ async function parseArguments() {
  libVersion: { short: 'l', type: 'string', default: pkg['mongodb:libmongocrypt'] },
  clean: { short: 'c', type: 'boolean', default: false },
  build: { short: 'b', type: 'boolean', default: false },
+ 'no-crypto': { type: 'boolean', default: false }, // Use Node.js builtin crypto
+ fastDownload: { type: 'boolean', default: false }, // Potentially incorrect download, only for the brave and impatient
  help: { short: 'h', type: 'boolean', default: false }
  };
 
@@ -39,7 +42,10 @@ async function parseArguments() {
  }
 
  return {
- libmongocrypt: { url: args.values.gitURL, ref: args.values.libVersion },
+ url: args.values.gitURL,
+ ref: args.values.libVersion,
+ crypto: !args.values['no-crypto'],
+ fastDownload: args.values.fastDownload,
  clean: args.values.clean,
  build: args.values.build,
  pkg
@@ -77,7 +83,7 @@ export async function cloneLibMongoCrypt(libmongocryptRoot, { url, ref }) {
  }
 }
 
-export async function buildLibMongoCrypt(libmongocryptRoot, nodeDepsRoot) {
+export async function buildLibMongoCrypt(libmongocryptRoot, nodeDepsRoot, options) {
  console.error('building libmongocrypt...');
 
  const nodeBuildRoot = resolveRoot(nodeDepsRoot, 'tmp', 'libmongocrypt-build');
@@ -88,7 +94,6 @@ export async function buildLibMongoCrypt(libmongocryptRoot, nodeDepsRoot) {
  const CMAKE_FLAGS = toFlags({
  /**
  * We provide crypto hooks from Node.js binding to openssl (so disable system crypto)
- * TODO: NODE-5455
  *
  * One thing that is not obvious from the build instructions for libmongocrypt
  * and the Node.js bindings is that the Node.js driver uses libmongocrypt in
@@ -101,7 +106,7 @@ export async function buildLibMongoCrypt(libmongocryptRoot, nodeDepsRoot) {
  * have a copy of OpenSSL available directly, but for now it seems to make sense
  * to stick with what the Node.js addon does here.
  */
- DDISABLE_NATIVE_CRYPTO: '1',
+ DDISABLE_NATIVE_CRYPTO: options.crypto ? '0' : '1',
  /** A consistent name for the output "library" directory */
  DCMAKE_INSTALL_LIBDIR: 'lib',
  /** No warnings allowed */
@@ -131,12 +136,13 @@ export async function buildLibMongoCrypt(libmongocryptRoot, nodeDepsRoot) {
  [...CMAKE_FLAGS, ...WINDOWS_CMAKE_FLAGS, ...MACOS_CMAKE_FLAGS, libmongocryptRoot],
  { cwd: nodeBuildRoot }
  );
+
  await run('cmake', ['--build', '.', '--target', 'install', '--config', 'RelWithDebInfo'], {
  cwd: nodeBuildRoot
  });
 }
 
-export async function downloadLibMongoCrypt(nodeDepsRoot, { ref }) {
+export async function downloadLibMongoCrypt(nodeDepsRoot, { ref, crypto, fastDownload }) {
  const downloadURL =
  ref === 'latest'
  ? 'https://mciuploads.s3.amazonaws.com/libmongocrypt/all/master/latest/libmongocrypt-all.tar.gz'
@@ -164,61 +170,96 @@ export async function downloadLibMongoCrypt(nodeDepsRoot, { ref }) {
 
  console.error(`Platform: ${detectedPlatform} Prebuild: ${prebuild}`);
 
- const unzipArgs = ['-xzv', '-C', `_libmongocrypt-${ref}`, `${prebuild}/nocrypto`];
+ const downloadDestination = crypto ? `${prebuild}` : `${prebuild}/nocrypto`;
+ const unzipArgs = ['-xzv', '-C', `_libmongocrypt-${ref}`, downloadDestination];
  console.error(`+ tar ${unzipArgs.join(' ')}`);
  const unzip = child_process.spawn('tar', unzipArgs, {
- stdio: ['pipe', 'inherit'],
+ stdio: ['pipe', 'inherit', 'pipe'],
  cwd: resolveRoot('.')
  });
 
  const [response] = await events.once(https.get(downloadURL), 'response');
 
  const start = performance.now();
- await stream.pipeline(response, unzip.stdin);
+
+ let signal;
+ if (fastDownload) {
+ /**
+ * Tar will print out each file it finds inside MEMBER (ex. macos/nocrypto)
+ * For each file it prints, we give it a deadline of 10seconds to print the next one.
+ * If nothing prints after 10 seconds we exit early.
+ * This depends on the tar file being in order and un-tar-able in under 10sec.
+ *
+ * download time went from 230s to 80s
+ */
+ const controller = new AbortController();
+ signal = controller.signal;
+ let isFirstStdoutDataEv = true;
+ let timeout;
+ unzip.stderr.on('data', chunk => {
+ process.stderr.write(chunk, () => {
+ if (isFirstStdoutDataEv) {
+ isFirstStdoutDataEv = false;
+ timeout = setTimeout(() => controller.abort(), 5_000);
+ }
+ timeout?.refresh();
+ });
+ });
+ }
+
+ try {
+ await stream.pipeline(response, unzip.stdin, { signal });
+ } catch {
+ await fs.access(path.join(`_libmongocrypt-${ref}`, downloadDestination));
+ }
  const end = performance.now();
 
  console.error(`downloaded libmongocrypt in ${(end - start) / 1000} secs...`);
 
  await fs.rm(nodeDepsRoot, { recursive: true, force: true });
- await fs.cp(resolveRoot(destination, prebuild, 'nocrypto'), nodeDepsRoot, { recursive: true });
- const currentPath = path.join(nodeDepsRoot, 'lib64');
+ const source = crypto
+ ? resolveRoot(destination, prebuild)
+ : resolveRoot(destination, prebuild, 'nocrypto');
+ await fs.cp(source, nodeDepsRoot, { recursive: true });
+ const potentialLib64Path = path.join(nodeDepsRoot, 'lib64');
  try {
- await fs.rename(currentPath, path.join(nodeDepsRoot, 'lib'));
+ await fs.rename(potentialLib64Path, path.join(nodeDepsRoot, 'lib'));
  } catch (error) {
- console.error(`error renaming ${currentPath}: ${error.message}`);
+ await fs.access(path.join(nodeDepsRoot, 'lib')); // Ensure there is a "lib" directory
  }
 }
 
 async function main() {
- const { libmongocrypt, build, clean, pkg } = await parseArguments();
+ const args = await parseArguments();
+ console.log(args);
 
  const nodeDepsDir = resolveRoot('deps');
 
- if (build) {
+ if (args.build) {
  const libmongocryptCloneDir = resolveRoot('_libmongocrypt');
 
  const currentLibMongoCryptBranch = await fs
  .readFile(path.join(libmongocryptCloneDir, '.git', 'HEAD'), 'utf8')
  .catch(() => '');
  const isClonedAndCheckedOut = currentLibMongoCryptBranch
  .trim()
- .endsWith(`r-${libmongocrypt.ref}`);
+ .endsWith(`r-${args.ref}`);
 
- if (clean || !isClonedAndCheckedOut) {
- await cloneLibMongoCrypt(libmongocryptCloneDir, libmongocrypt);
+ if (args.clean || !isClonedAndCheckedOut) {
+ await cloneLibMongoCrypt(libmongocryptCloneDir, args);
  }
 
  const libmongocryptBuiltVersion = await fs
  .readFile(path.join(libmongocryptCloneDir, 'VERSION_CURRENT'), 'utf8')
  .catch(() => '');
- const isBuilt = libmongocryptBuiltVersion.trim() === libmongocrypt.ref;
+ const isBuilt = libmongocryptBuiltVersion.trim() === args.ref;
 
- if (clean || !isBuilt) {
- await buildLibMongoCrypt(libmongocryptCloneDir, nodeDepsDir);
+ if (args.clean || !isBuilt) {
+ await buildLibMongoCrypt(libmongocryptCloneDir, nodeDepsDir, args);
  }
  } else {
  // Download
- await downloadLibMongoCrypt(nodeDepsDir, libmongocrypt);
+ await downloadLibMongoCrypt(nodeDepsDir, args);
  }
 
  await fs.rm(resolveRoot('build'), { force: true, recursive: true });
@@ -233,8 +274,8 @@ async function main() {
  if (process.platform === 'darwin') {
  // The "arm64" build is actually a universal binary
  await fs.copyFile(
- resolveRoot('prebuilds', `mongodb-client-encryption-v${pkg.version}-napi-v4-darwin-arm64.tar.gz`),
- resolveRoot('prebuilds', `mongodb-client-encryption-v${pkg.version}-napi-v4-darwin-x64.tar.gz`)
+ resolveRoot('prebuilds', `mongodb-client-encryption-v${args.pkg.version}-napi-v4-darwin-arm64.tar.gz`),
+ resolveRoot('prebuilds', `mongodb-client-encryption-v${args.pkg.version}-napi-v4-darwin-x64.tar.gz`)
  );
  }
 }

.github/workflows/build.yml

@@ -22,7 +22,7 @@ jobs:
  - uses: actions/checkout@v4
 
  - name: Build ${{ matrix.os }} Prebuild
- run: node .github/scripts/libmongocrypt.mjs ${{ runner.os == 'Windows' && '--build' || '' }}
+ run: npm run install:libmongocrypt ${{ runner.os == 'Windows' && '-- --build' || '' }}
  shell: bash
 
  - name: Test ${{ matrix.os }}

.github/workflows/codeql.yml

@@ -46,7 +46,7 @@ jobs:
 
  - if: matrix.build-mode == 'manual'
  shell: bash
- run: node .github/scripts/libmongocrypt.mjs
+ run: npm run install:libmongocrypt
 
  - name: Perform CodeQL Analysis
  uses: github/codeql-action/analyze@v3

README.md

@@ -27,10 +27,32 @@ npm install mongodb-client-encryption
 
 #### Setup
 
-Run the following command to build libmongocrypt and setup the node bindings for development:
+Run the following command to build libmongocrypt and you are setup to develop the node bindings:
 
 ```shell
-bash ./etc/build-static.sh
+npm run install:libmongocrypt
+```
+
+#### `libmongocrypt.mjs`
+
+```
+node libmongocrypt.mjs [--gitURL=string] [--libVersion=string] [--clean] [--build] [--no-crypto] [--fastDownload]
+
+By default attempts to download and compile the bindings with the crypto prebuilds of libmongocrypt.
+Can be configured to clone and build without crypto.
+
+--gitURL=string A custom remote git repository to clone libmongocrypt from. You must also set --build to use this.
+--libVersion=string A custom version reference to either download or checkout after cloning.
+ You may use "latest" to get current libmongocrypt `HEAD`.
+--clean Combined with --build, the script will not skip cloning and rebuilding libmongocrypt.
+--build Instead of downloading, clone and build libmongocrypt along with the bindings.
+--no-crypto Use libmongocrypt prebuild or build libmongocrypt without OpenSSL symbols.
+ Requires the bindings to provide cryptoCallbacks.
+
+Only suitable for local development:
+
+--fastDownload If you are improving this script or otherwise repeatedly downloading libmongocrypt,
+ this flag will interrupt the un-tar operation as early as possible. It should work, most of the time.
 ```
 
 #### Prebuild Platforms

addon/mongocrypt.cc

@@ -472,7 +472,7 @@ MongoCrypt::MongoCrypt(const CallbackInfo& info)
  }
  }
 
- if (options.Has("cryptoCallbacks")) {
+ if (!mongocrypt_is_crypto_available() && options.Has("cryptoCallbacks")) {
  Object cryptoCallbacks = options.Get("cryptoCallbacks").ToObject();
 
  SetCallback("aes256CbcEncryptHook", cryptoCallbacks["aes256CbcEncryptHook"]);

test/benchmarks/bench.mjs

@@ -15,8 +15,7 @@ const ERROR = 0;
 
 const __dirname = path.dirname(url.fileURLToPath(import.meta.url));
 
-const { CRYPT_SHARED_LIB_PATH: cryptSharedLibPath = '', BENCH_WITH_NATIVE_CRYPTO = '' } =
- process.env;
+const { CRYPT_SHARED_LIB_PATH: cryptSharedLibPath = '' } = process.env;
 
 const warmupSecs = 2;
 const testInSecs = 57;
@@ -121,8 +120,7 @@ function main() {
  `testInSecs=${testInSecs}`
  );
 
- const mongoCryptOptions = { kmsProviders: BSON.serialize(kmsProviders) };
- if (!BENCH_WITH_NATIVE_CRYPTO) mongoCryptOptions.cryptoCallbacks = cryptoCallbacks;
+ const mongoCryptOptions = { kmsProviders: BSON.serialize(kmsProviders), cryptoCallbacks };
  if (cryptSharedLibPath) mongoCryptOptions.cryptSharedLibPath = cryptSharedLibPath;
 
  const mongoCrypt = new MongoCrypt(mongoCryptOptions);