mknichel/dedupe strings in module layers

lib/ModuleLayerCache.js

@@ -0,0 +1,81 @@
+/*
+ MIT License http://www.opensource.org/licenses/mit-license.php
+ Author @mknichel
+*/
+
+"use strict";
+
+/**
+ * A class used to represent a key in a {@link WeakMap}. {@link Symbol} can not
+ * be used due to https://github.com/nodejs/node/issues/49135.
+ */
+class WeakMapKey {
+ /**
+ * @param {string} key The key to use for caching.
+ */
+ constructor(key) {
+ this.key = key;
+ }
+}
+
+/**
+ * A cache for storing information that could be shared by modules across
+ * different layers. This can be useful for memory management since the same
+ * file that appears in multiple layers may duplicate large strings, such as
+ * the contents of the source file. The values in this cache can be garbage
+ * collected if there are no more retaining references.
+ */
+class ModuleLayerCache {
+ constructor() {
+ /** @type {Map<string, WeakMapKey>} */
+ this.weakMapKeys = new Map();
+ /** @type {WeakMap<WeakMapKey, string | Buffer>} */
+ this.cachedValues = new WeakMap();
+ }
+
+ /**
+ * Checks to see if there is an entry for the key in this cache. The key is
+ * typically {@link NormalModule#request}.
+ *
+ * @param {string} key The key to use for caching.
+ * @returns {boolean} Whether there is an entry for the key in this cache.
+ */
+ has(key) {
+ return (
+ this.weakMapKeys.has(key) &&
+ this.cachedValues.has(this.weakMapKeys.get(key))
+ );
+ }
+
+ /**
+ * Returns the cached value value for the key in this cache if the value has
+ * not already been garbage collected.
+ *
+ * @param {string} key The key to use for caching.
+ * @returns {string | Buffer} The cached value for this key.
+ */
+ get(key) {
+ if (!this.has(key)) {
+ return null;
+ }
+ const weakMapKey = this.weakMapKeys.get(key);
+ return this.cachedValues.get(weakMapKey);
+ }
+
+ /**
+ * Stores the value in this cache as a weakly held value (i.e. it can be
+ * garbage collected).
+ *
+ * @param {string} key The key to use for caching.
+ * @param {string | Buffer} value The value to store in the cache.
+ */
+ set(key, value) {
+ const weakMapKey = this.weakMapKeys.has(key)
+ ? this.weakMapKeys.get(key)
+ : new WeakMapKey(key);
+ this.weakMapKeys.set(key, weakMapKey);
+ this.cachedValues.set(weakMapKey, value);
+ }
+}
+
+module.exports = ModuleLayerCache;

lib/NormalModule.js

@@ -21,6 +21,7 @@ const Module = require("./Module");
 const ModuleBuildError = require("./ModuleBuildError");
 const ModuleError = require("./ModuleError");
 const ModuleGraphConnection = require("./ModuleGraphConnection");
+const ModuleLayerCache = require("./ModuleLayerCache");
 const ModuleParseError = require("./ModuleParseError");
 const { JAVASCRIPT_MODULE_TYPE_AUTO } = require("./ModuleTypeConstants");
 const ModuleWarning = require("./ModuleWarning");
@@ -231,6 +232,9 @@ makeSerializable(
 /** @type {WeakMap<Compilation, NormalModuleCompilationHooks>} */
 const compilationHooksMap = new WeakMap();
 
+/** @type {WeakMap<Object, ModuleLayerCache>} */
+const cachedModuleLayerCaches = new WeakMap();
+
 class NormalModule extends Module {
  /**
  * @param {Compilation} compilation the compilation
@@ -780,6 +784,31 @@ class NormalModule extends Module {
  * @returns {Source} the created source
  */
  createSource(context, content, sourceMap, associatedObjectForCache) {
+ // If this module exists in a layer, try to reuse the value for the
+ // source string so it is not duplicated in multiple modules.
+ if (this.layer && this.userRequest) {
+ let moduleLayerCache = cachedModuleLayerCaches.get(
+ associatedObjectForCache
+ );
+ if (moduleLayerCache) {
+ moduleLayerCache = new ModuleLayerCache();
+ cachedModuleLayerCaches.set(associatedObjectForCache, moduleLayerCache);
+ }
+ if (moduleLayerCache.has(this.userRequest)) {
+ const cachedSource = moduleLayerCache.get(this.userRequest);
+ if (
+ (Buffer.isBuffer(content) &&
+ Buffer.isBuffer(cachedSource) &&
+ content.equals(cachedSource)) ||
+ content === cachedSource
+ ) {
+ content = cachedSource;
+ }
+ } else {
+ moduleLayerCache.set(this.userRequest, content);
+ }
+ }
+
  if (Buffer.isBuffer(content)) {
  return new RawSource(content);
  }

lib/json/JsonModulesPlugin.js

@@ -44,7 +44,7 @@ class JsonModulesPlugin {
  .tap(PLUGIN_NAME, parserOptions => {
  validate(parserOptions);
 
- return new JsonParser(parserOptions);
+ return new JsonParser(parserOptions, compilation);
  });
  normalModuleFactory.hooks.createGenerator
  .for(JSON_MODULE_TYPE)

lib/json/JsonParser.js

@@ -5,6 +5,7 @@
 
 "use strict";
 
+const ModuleLayerCache = require("../ModuleLayerCache");
 const Parser = require("../Parser");
 const JsonExportsDependency = require("../dependencies/JsonExportsDependency");
 const memoize = require("../util/memoize");
@@ -19,13 +20,18 @@ const JsonData = require("./JsonData");
 
 const getParseJson = memoize(() => require("json-parse-even-better-errors"));
 
+/** @type {WeakMap<Object, ModuleLayerCache>} */
+const cachedModuleLayerCaches = new WeakMap();
+
 class JsonParser extends Parser {
  /**
  * @param {JsonModulesPluginParserOptions} options parser options
+ * @param {Object} associatedObjectForCache An object to associate cached data with.
  */
- constructor(options) {
+ constructor(options, associatedObjectForCache) {
  super();
  this.options = options || {};
+ this.associatedObjectForCache = associatedObjectForCache;
  }
 
  /**
@@ -53,6 +59,31 @@ class JsonParser extends Parser {
  } catch (e) {
  throw new Error(`Cannot parse JSON: ${/** @type {Error} */ (e).message}`);
  }
+
+ // If the module is associated with a layer, try to reuse cached data instead
+ // of duplicating the data multiple times.
+ const module = state.module;
+ if (module && module.userRequest && module.layer && Buffer.isBuffer(data)) {
+ let moduleLayerCache = cachedModuleLayerCaches.get(
+ this.associatedObjectForCache
+ );
+ if (moduleLayerCache) {
+ moduleLayerCache = new ModuleLayerCache();
+ cachedModuleLayerCaches.set(
+ this.associatedObjectForCache,
+ moduleLayerCache
+ );
+ }
+ if (moduleLayerCache.has(module.userRequest)) {
+ const cachedData = moduleLayerCache.get(module.userRequest);
+ if (Buffer.isBuffer(cachedData) && data.equals(cachedData)) {
+ data = cachedData;
+ }
+ } else {
+ moduleLayerCache.set(module.userRequest, data);
+ }
+ }
+
  const jsonData = new JsonData(/** @type {Buffer | RawJsonData} */ (data));
  const buildInfo = /** @type {BuildInfo} */ (state.module.buildInfo);
  buildInfo.jsonData = jsonData;