Current File : //home/tradevaly/www/node_modules/webpack/lib/optimize/ModuleConcatenationPlugin.js |
/*
MIT License http://www.opensource.org/licenses/mit-license.php
Author Tobias Koppers @sokra
*/
"use strict";
const asyncLib = require("neo-async");
const ChunkGraph = require("../ChunkGraph");
const ModuleGraph = require("../ModuleGraph");
const { STAGE_DEFAULT } = require("../OptimizationStages");
const HarmonyImportDependency = require("../dependencies/HarmonyImportDependency");
const { compareModulesByIdentifier } = require("../util/comparators");
const {
intersectRuntime,
mergeRuntimeOwned,
filterRuntime,
runtimeToString,
mergeRuntime
} = require("../util/runtime");
const ConcatenatedModule = require("./ConcatenatedModule");
/** @typedef {import("../Compilation")} Compilation */
/** @typedef {import("../Compiler")} Compiler */
/** @typedef {import("../Module")} Module */
/** @typedef {import("../RequestShortener")} RequestShortener */
/** @typedef {import("../util/runtime").RuntimeSpec} RuntimeSpec */
/**
* @typedef {Object} Statistics
* @property {number} cached
* @property {number} alreadyInConfig
* @property {number} invalidModule
* @property {number} incorrectChunks
* @property {number} incorrectDependency
* @property {number} incorrectModuleDependency
* @property {number} incorrectChunksOfImporter
* @property {number} incorrectRuntimeCondition
* @property {number} importerFailed
* @property {number} added
*/
const formatBailoutReason = msg => {
return "ModuleConcatenation bailout: " + msg;
};
class ModuleConcatenationPlugin {
constructor(options) {
if (typeof options !== "object") options = {};
this.options = options;
}
/**
* Apply the plugin
* @param {Compiler} compiler the compiler instance
* @returns {void}
*/
apply(compiler) {
compiler.hooks.compilation.tap("ModuleConcatenationPlugin", compilation => {
const moduleGraph = compilation.moduleGraph;
const bailoutReasonMap = new Map();
const setBailoutReason = (module, reason) => {
setInnerBailoutReason(module, reason);
moduleGraph
.getOptimizationBailout(module)
.push(
typeof reason === "function"
? rs => formatBailoutReason(reason(rs))
: formatBailoutReason(reason)
);
};
const setInnerBailoutReason = (module, reason) => {
bailoutReasonMap.set(module, reason);
};
const getInnerBailoutReason = (module, requestShortener) => {
const reason = bailoutReasonMap.get(module);
if (typeof reason === "function") return reason(requestShortener);
return reason;
};
const formatBailoutWarning = (module, problem) => requestShortener => {
if (typeof problem === "function") {
return formatBailoutReason(
`Cannot concat with ${module.readableIdentifier(
requestShortener
)}: ${problem(requestShortener)}`
);
}
const reason = getInnerBailoutReason(module, requestShortener);
const reasonWithPrefix = reason ? `: ${reason}` : "";
if (module === problem) {
return formatBailoutReason(
`Cannot concat with ${module.readableIdentifier(
requestShortener
)}${reasonWithPrefix}`
);
} else {
return formatBailoutReason(
`Cannot concat with ${module.readableIdentifier(
requestShortener
)} because of ${problem.readableIdentifier(
requestShortener
)}${reasonWithPrefix}`
);
}
};
compilation.hooks.optimizeChunkModules.tapAsync(
{
name: "ModuleConcatenationPlugin",
stage: STAGE_DEFAULT
},
(allChunks, modules, callback) => {
const logger = compilation.getLogger(
"webpack.ModuleConcatenationPlugin"
);
const { chunkGraph, moduleGraph } = compilation;
const relevantModules = [];
const possibleInners = new Set();
const context = {
chunkGraph,
moduleGraph
};
logger.time("select relevant modules");
for (const module of modules) {
let canBeRoot = true;
let canBeInner = true;
const bailoutReason = module.getConcatenationBailoutReason(context);
if (bailoutReason) {
setBailoutReason(module, bailoutReason);
continue;
}
// Must not be an async module
if (moduleGraph.isAsync(module)) {
setBailoutReason(module, `Module is async`);
continue;
}
// Must be in strict mode
if (!module.buildInfo.strict) {
setBailoutReason(module, `Module is not in strict mode`);
continue;
}
// Module must be in any chunk (we don't want to do useless work)
if (chunkGraph.getNumberOfModuleChunks(module) === 0) {
setBailoutReason(module, "Module is not in any chunk");
continue;
}
// Exports must be known (and not dynamic)
const exportsInfo = moduleGraph.getExportsInfo(module);
const relevantExports = exportsInfo.getRelevantExports(undefined);
const unknownReexports = relevantExports.filter(exportInfo => {
return (
exportInfo.isReexport() && !exportInfo.getTarget(moduleGraph)
);
});
if (unknownReexports.length > 0) {
setBailoutReason(
module,
`Reexports in this module do not have a static target (${Array.from(
unknownReexports,
exportInfo =>
`${
exportInfo.name || "other exports"
}: ${exportInfo.getUsedInfo()}`
).join(", ")})`
);
continue;
}
// Root modules must have a static list of exports
const unknownProvidedExports = relevantExports.filter(
exportInfo => {
return exportInfo.provided !== true;
}
);
if (unknownProvidedExports.length > 0) {
setBailoutReason(
module,
`List of module exports is dynamic (${Array.from(
unknownProvidedExports,
exportInfo =>
`${
exportInfo.name || "other exports"
}: ${exportInfo.getProvidedInfo()} and ${exportInfo.getUsedInfo()}`
).join(", ")})`
);
canBeRoot = false;
}
// Module must not be an entry point
if (chunkGraph.isEntryModule(module)) {
setInnerBailoutReason(module, "Module is an entry point");
canBeInner = false;
}
if (canBeRoot) relevantModules.push(module);
if (canBeInner) possibleInners.add(module);
}
logger.timeEnd("select relevant modules");
logger.debug(
`${relevantModules.length} potential root modules, ${possibleInners.size} potential inner modules`
);
// sort by depth
// modules with lower depth are more likely suited as roots
// this improves performance, because modules already selected as inner are skipped
logger.time("sort relevant modules");
relevantModules.sort((a, b) => {
return moduleGraph.getDepth(a) - moduleGraph.getDepth(b);
});
logger.timeEnd("sort relevant modules");
/** @type {Statistics} */
const stats = {
cached: 0,
alreadyInConfig: 0,
invalidModule: 0,
incorrectChunks: 0,
incorrectDependency: 0,
incorrectModuleDependency: 0,
incorrectChunksOfImporter: 0,
incorrectRuntimeCondition: 0,
importerFailed: 0,
added: 0
};
let statsCandidates = 0;
let statsSizeSum = 0;
let statsEmptyConfigurations = 0;
logger.time("find modules to concatenate");
const concatConfigurations = [];
const usedAsInner = new Set();
for (const currentRoot of relevantModules) {
// when used by another configuration as inner:
// the other configuration is better and we can skip this one
// TODO reconsider that when it's only used in a different runtime
if (usedAsInner.has(currentRoot)) continue;
let chunkRuntime = undefined;
for (const r of chunkGraph.getModuleRuntimes(currentRoot)) {
chunkRuntime = mergeRuntimeOwned(chunkRuntime, r);
}
const exportsInfo = moduleGraph.getExportsInfo(currentRoot);
const filteredRuntime = filterRuntime(chunkRuntime, r =>
exportsInfo.isModuleUsed(r)
);
const activeRuntime =
filteredRuntime === true
? chunkRuntime
: filteredRuntime === false
? undefined
: filteredRuntime;
// create a configuration with the root
const currentConfiguration = new ConcatConfiguration(
currentRoot,
activeRuntime
);
// cache failures to add modules
const failureCache = new Map();
// potential optional import candidates
/** @type {Set<Module>} */
const candidates = new Set();
// try to add all imports
for (const imp of this._getImports(
compilation,
currentRoot,
activeRuntime
)) {
candidates.add(imp);
}
for (const imp of candidates) {
const impCandidates = new Set();
const problem = this._tryToAdd(
compilation,
currentConfiguration,
imp,
chunkRuntime,
activeRuntime,
possibleInners,
impCandidates,
failureCache,
chunkGraph,
true,
stats
);
if (problem) {
failureCache.set(imp, problem);
currentConfiguration.addWarning(imp, problem);
} else {
for (const c of impCandidates) {
candidates.add(c);
}
}
}
statsCandidates += candidates.size;
if (!currentConfiguration.isEmpty()) {
const modules = currentConfiguration.getModules();
statsSizeSum += modules.size;
concatConfigurations.push(currentConfiguration);
for (const module of modules) {
if (module !== currentConfiguration.rootModule) {
usedAsInner.add(module);
}
}
} else {
statsEmptyConfigurations++;
const optimizationBailouts =
moduleGraph.getOptimizationBailout(currentRoot);
for (const warning of currentConfiguration.getWarningsSorted()) {
optimizationBailouts.push(
formatBailoutWarning(warning[0], warning[1])
);
}
}
}
logger.timeEnd("find modules to concatenate");
logger.debug(
`${
concatConfigurations.length
} successful concat configurations (avg size: ${
statsSizeSum / concatConfigurations.length
}), ${statsEmptyConfigurations} bailed out completely`
);
logger.debug(
`${statsCandidates} candidates were considered for adding (${stats.cached} cached failure, ${stats.alreadyInConfig} already in config, ${stats.invalidModule} invalid module, ${stats.incorrectChunks} incorrect chunks, ${stats.incorrectDependency} incorrect dependency, ${stats.incorrectChunksOfImporter} incorrect chunks of importer, ${stats.incorrectModuleDependency} incorrect module dependency, ${stats.incorrectRuntimeCondition} incorrect runtime condition, ${stats.importerFailed} importer failed, ${stats.added} added)`
);
// HACK: Sort configurations by length and start with the longest one
// to get the biggest groups possible. Used modules are marked with usedModules
// TODO: Allow to reuse existing configuration while trying to add dependencies.
// This would improve performance. O(n^2) -> O(n)
logger.time(`sort concat configurations`);
concatConfigurations.sort((a, b) => {
return b.modules.size - a.modules.size;
});
logger.timeEnd(`sort concat configurations`);
const usedModules = new Set();
logger.time("create concatenated modules");
asyncLib.each(
concatConfigurations,
(concatConfiguration, callback) => {
const rootModule = concatConfiguration.rootModule;
// Avoid overlapping configurations
// TODO: remove this when todo above is fixed
if (usedModules.has(rootModule)) return callback();
const modules = concatConfiguration.getModules();
for (const m of modules) {
usedModules.add(m);
}
// Create a new ConcatenatedModule
let newModule = ConcatenatedModule.create(
rootModule,
modules,
concatConfiguration.runtime,
compiler.root
);
const build = () => {
newModule.build(
compiler.options,
compilation,
null,
null,
err => {
if (err) {
if (!err.module) {
err.module = newModule;
}
return callback(err);
}
integrate();
}
);
};
const integrate = () => {
ChunkGraph.setChunkGraphForModule(newModule, chunkGraph);
ModuleGraph.setModuleGraphForModule(newModule, moduleGraph);
for (const warning of concatConfiguration.getWarningsSorted()) {
moduleGraph
.getOptimizationBailout(newModule)
.push(formatBailoutWarning(warning[0], warning[1]));
}
moduleGraph.cloneModuleAttributes(rootModule, newModule);
for (const m of modules) {
// add to builtModules when one of the included modules was built
if (compilation.builtModules.has(m)) {
compilation.builtModules.add(newModule);
}
if (m !== rootModule) {
// attach external references to the concatenated module too
moduleGraph.copyOutgoingModuleConnections(
m,
newModule,
c => {
return (
c.originModule === m &&
!(
c.dependency instanceof HarmonyImportDependency &&
modules.has(c.module)
)
);
}
);
// remove module from chunk
for (const chunk of chunkGraph.getModuleChunksIterable(
rootModule
)) {
chunkGraph.disconnectChunkAndModule(chunk, m);
}
}
}
compilation.modules.delete(rootModule);
ChunkGraph.clearChunkGraphForModule(rootModule);
ModuleGraph.clearModuleGraphForModule(rootModule);
// remove module from chunk
chunkGraph.replaceModule(rootModule, newModule);
// replace module references with the concatenated module
moduleGraph.moveModuleConnections(rootModule, newModule, c => {
const otherModule =
c.module === rootModule ? c.originModule : c.module;
const innerConnection =
c.dependency instanceof HarmonyImportDependency &&
modules.has(otherModule);
return !innerConnection;
});
// add concatenated module to the compilation
compilation.modules.add(newModule);
callback();
};
build();
},
err => {
logger.timeEnd("create concatenated modules");
process.nextTick(callback.bind(null, err));
}
);
}
);
});
}
/**
* @param {Compilation} compilation the compilation
* @param {Module} module the module to be added
* @param {RuntimeSpec} runtime the runtime scope
* @returns {Set<Module>} the imported modules
*/
_getImports(compilation, module, runtime) {
const moduleGraph = compilation.moduleGraph;
const set = new Set();
for (const dep of module.dependencies) {
// Get reference info only for harmony Dependencies
if (!(dep instanceof HarmonyImportDependency)) continue;
const connection = moduleGraph.getConnection(dep);
// Reference is valid and has a module
if (
!connection ||
!connection.module ||
!connection.isTargetActive(runtime)
) {
continue;
}
const importedNames = compilation.getDependencyReferencedExports(
dep,
undefined
);
if (
importedNames.every(i =>
Array.isArray(i) ? i.length > 0 : i.name.length > 0
) ||
Array.isArray(moduleGraph.getProvidedExports(module))
) {
set.add(connection.module);
}
}
return set;
}
/**
* @param {Compilation} compilation webpack compilation
* @param {ConcatConfiguration} config concat configuration (will be modified when added)
* @param {Module} module the module to be added
* @param {RuntimeSpec} runtime the runtime scope of the generated code
* @param {RuntimeSpec} activeRuntime the runtime scope of the root module
* @param {Set<Module>} possibleModules modules that are candidates
* @param {Set<Module>} candidates list of potential candidates (will be added to)
* @param {Map<Module, Module | function(RequestShortener): string>} failureCache cache for problematic modules to be more performant
* @param {ChunkGraph} chunkGraph the chunk graph
* @param {boolean} avoidMutateOnFailure avoid mutating the config when adding fails
* @param {Statistics} statistics gathering metrics
* @returns {Module | function(RequestShortener): string} the problematic module
*/
_tryToAdd(
compilation,
config,
module,
runtime,
activeRuntime,
possibleModules,
candidates,
failureCache,
chunkGraph,
avoidMutateOnFailure,
statistics
) {
const cacheEntry = failureCache.get(module);
if (cacheEntry) {
statistics.cached++;
return cacheEntry;
}
// Already added?
if (config.has(module)) {
statistics.alreadyInConfig++;
return null;
}
// Not possible to add?
if (!possibleModules.has(module)) {
statistics.invalidModule++;
failureCache.set(module, module); // cache failures for performance
return module;
}
// Module must be in the correct chunks
const missingChunks = Array.from(
chunkGraph.getModuleChunksIterable(config.rootModule)
).filter(chunk => !chunkGraph.isModuleInChunk(module, chunk));
if (missingChunks.length > 0) {
const problem = requestShortener => {
const missingChunksList = Array.from(
new Set(missingChunks.map(chunk => chunk.name || "unnamed chunk(s)"))
).sort();
const chunks = Array.from(
new Set(
Array.from(chunkGraph.getModuleChunksIterable(module)).map(
chunk => chunk.name || "unnamed chunk(s)"
)
)
).sort();
return `Module ${module.readableIdentifier(
requestShortener
)} is not in the same chunk(s) (expected in chunk(s) ${missingChunksList.join(
", "
)}, module is in chunk(s) ${chunks.join(", ")})`;
};
statistics.incorrectChunks++;
failureCache.set(module, problem); // cache failures for performance
return problem;
}
const moduleGraph = compilation.moduleGraph;
const incomingConnections =
moduleGraph.getIncomingConnectionsByOriginModule(module);
const incomingConnectionsFromNonModules =
incomingConnections.get(null) || incomingConnections.get(undefined);
if (incomingConnectionsFromNonModules) {
const activeNonModulesConnections =
incomingConnectionsFromNonModules.filter(connection => {
// We are not interested in inactive connections
// or connections without dependency
return connection.isActive(runtime) || connection.dependency;
});
if (activeNonModulesConnections.length > 0) {
const problem = requestShortener => {
const importingExplanations = new Set(
activeNonModulesConnections.map(c => c.explanation).filter(Boolean)
);
const explanations = Array.from(importingExplanations).sort();
return `Module ${module.readableIdentifier(
requestShortener
)} is referenced ${
explanations.length > 0
? `by: ${explanations.join(", ")}`
: "in an unsupported way"
}`;
};
statistics.incorrectDependency++;
failureCache.set(module, problem); // cache failures for performance
return problem;
}
}
/** @type {Map<Module, readonly ModuleGraph.ModuleGraphConnection[]>} */
const incomingConnectionsFromModules = new Map();
for (const [originModule, connections] of incomingConnections) {
if (originModule) {
// Ignore connection from orphan modules
if (chunkGraph.getNumberOfModuleChunks(originModule) === 0) continue;
// We don't care for connections from other runtimes
let originRuntime = undefined;
for (const r of chunkGraph.getModuleRuntimes(originModule)) {
originRuntime = mergeRuntimeOwned(originRuntime, r);
}
if (!intersectRuntime(runtime, originRuntime)) continue;
// We are not interested in inactive connections
const activeConnections = connections.filter(connection =>
connection.isActive(runtime)
);
if (activeConnections.length > 0)
incomingConnectionsFromModules.set(originModule, activeConnections);
}
}
const incomingModules = Array.from(incomingConnectionsFromModules.keys());
// Module must be in the same chunks like the referencing module
const otherChunkModules = incomingModules.filter(originModule => {
for (const chunk of chunkGraph.getModuleChunksIterable(
config.rootModule
)) {
if (!chunkGraph.isModuleInChunk(originModule, chunk)) {
return true;
}
}
return false;
});
if (otherChunkModules.length > 0) {
const problem = requestShortener => {
const names = otherChunkModules
.map(m => m.readableIdentifier(requestShortener))
.sort();
return `Module ${module.readableIdentifier(
requestShortener
)} is referenced from different chunks by these modules: ${names.join(
", "
)}`;
};
statistics.incorrectChunksOfImporter++;
failureCache.set(module, problem); // cache failures for performance
return problem;
}
/** @type {Map<Module, readonly ModuleGraph.ModuleGraphConnection[]>} */
const nonHarmonyConnections = new Map();
for (const [originModule, connections] of incomingConnectionsFromModules) {
const selected = connections.filter(
connection =>
!connection.dependency ||
!(connection.dependency instanceof HarmonyImportDependency)
);
if (selected.length > 0)
nonHarmonyConnections.set(originModule, connections);
}
if (nonHarmonyConnections.size > 0) {
const problem = requestShortener => {
const names = Array.from(nonHarmonyConnections)
.map(([originModule, connections]) => {
return `${originModule.readableIdentifier(
requestShortener
)} (referenced with ${Array.from(
new Set(
connections
.map(c => c.dependency && c.dependency.type)
.filter(Boolean)
)
)
.sort()
.join(", ")})`;
})
.sort();
return `Module ${module.readableIdentifier(
requestShortener
)} is referenced from these modules with unsupported syntax: ${names.join(
", "
)}`;
};
statistics.incorrectModuleDependency++;
failureCache.set(module, problem); // cache failures for performance
return problem;
}
if (runtime !== undefined && typeof runtime !== "string") {
// Module must be consistently referenced in the same runtimes
/** @type {{ originModule: Module, runtimeCondition: RuntimeSpec }[]} */
const otherRuntimeConnections = [];
outer: for (const [
originModule,
connections
] of incomingConnectionsFromModules) {
/** @type {false | RuntimeSpec} */
let currentRuntimeCondition = false;
for (const connection of connections) {
const runtimeCondition = filterRuntime(runtime, runtime => {
return connection.isTargetActive(runtime);
});
if (runtimeCondition === false) continue;
if (runtimeCondition === true) continue outer;
if (currentRuntimeCondition !== false) {
currentRuntimeCondition = mergeRuntime(
currentRuntimeCondition,
runtimeCondition
);
} else {
currentRuntimeCondition = runtimeCondition;
}
}
if (currentRuntimeCondition !== false) {
otherRuntimeConnections.push({
originModule,
runtimeCondition: currentRuntimeCondition
});
}
}
if (otherRuntimeConnections.length > 0) {
const problem = requestShortener => {
return `Module ${module.readableIdentifier(
requestShortener
)} is runtime-dependent referenced by these modules: ${Array.from(
otherRuntimeConnections,
({ originModule, runtimeCondition }) =>
`${originModule.readableIdentifier(
requestShortener
)} (expected runtime ${runtimeToString(
runtime
)}, module is only referenced in ${runtimeToString(
/** @type {RuntimeSpec} */ (runtimeCondition)
)})`
).join(", ")}`;
};
statistics.incorrectRuntimeCondition++;
failureCache.set(module, problem); // cache failures for performance
return problem;
}
}
let backup;
if (avoidMutateOnFailure) {
backup = config.snapshot();
}
// Add the module
config.add(module);
incomingModules.sort(compareModulesByIdentifier);
// Every module which depends on the added module must be in the configuration too.
for (const originModule of incomingModules) {
const problem = this._tryToAdd(
compilation,
config,
originModule,
runtime,
activeRuntime,
possibleModules,
candidates,
failureCache,
chunkGraph,
false,
statistics
);
if (problem) {
if (backup !== undefined) config.rollback(backup);
statistics.importerFailed++;
failureCache.set(module, problem); // cache failures for performance
return problem;
}
}
// Add imports to possible candidates list
for (const imp of this._getImports(compilation, module, runtime)) {
candidates.add(imp);
}
statistics.added++;
return null;
}
}
class ConcatConfiguration {
/**
* @param {Module} rootModule the root module
* @param {RuntimeSpec} runtime the runtime
*/
constructor(rootModule, runtime) {
this.rootModule = rootModule;
this.runtime = runtime;
/** @type {Set<Module>} */
this.modules = new Set();
this.modules.add(rootModule);
/** @type {Map<Module, Module | function(RequestShortener): string>} */
this.warnings = new Map();
}
add(module) {
this.modules.add(module);
}
has(module) {
return this.modules.has(module);
}
isEmpty() {
return this.modules.size === 1;
}
addWarning(module, problem) {
this.warnings.set(module, problem);
}
getWarningsSorted() {
return new Map(
Array.from(this.warnings).sort((a, b) => {
const ai = a[0].identifier();
const bi = b[0].identifier();
if (ai < bi) return -1;
if (ai > bi) return 1;
return 0;
})
);
}
/**
* @returns {Set<Module>} modules as set
*/
getModules() {
return this.modules;
}
snapshot() {
return this.modules.size;
}
rollback(snapshot) {
const modules = this.modules;
for (const m of modules) {
if (snapshot === 0) {
modules.delete(m);
} else {
snapshot--;
}
}
}
}
module.exports = ModuleConcatenationPlugin;