Compiler.js 36 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247
  1. /*
  2. MIT License http://www.opensource.org/licenses/mit-license.php
  3. Author Tobias Koppers @sokra
  4. */
  5. "use strict";
  6. const parseJson = require("json-parse-even-better-errors");
  7. const asyncLib = require("neo-async");
  8. const {
  9. SyncHook,
  10. SyncBailHook,
  11. AsyncParallelHook,
  12. AsyncSeriesHook
  13. } = require("tapable");
  14. const { SizeOnlySource } = require("webpack-sources");
  15. const webpack = require("./");
  16. const Cache = require("./Cache");
  17. const CacheFacade = require("./CacheFacade");
  18. const ChunkGraph = require("./ChunkGraph");
  19. const Compilation = require("./Compilation");
  20. const ConcurrentCompilationError = require("./ConcurrentCompilationError");
  21. const ContextModuleFactory = require("./ContextModuleFactory");
  22. const ModuleGraph = require("./ModuleGraph");
  23. const NormalModuleFactory = require("./NormalModuleFactory");
  24. const RequestShortener = require("./RequestShortener");
  25. const ResolverFactory = require("./ResolverFactory");
  26. const Stats = require("./Stats");
  27. const Watching = require("./Watching");
  28. const WebpackError = require("./WebpackError");
  29. const { Logger } = require("./logging/Logger");
  30. const { join, dirname, mkdirp } = require("./util/fs");
  31. const { makePathsRelative } = require("./util/identifier");
  32. const { isSourceEqual } = require("./util/source");
  33. /** @typedef {import("webpack-sources").Source} Source */
  34. /** @typedef {import("../declarations/WebpackOptions").EntryNormalized} Entry */
  35. /** @typedef {import("../declarations/WebpackOptions").OutputNormalized} OutputOptions */
  36. /** @typedef {import("../declarations/WebpackOptions").WatchOptions} WatchOptions */
  37. /** @typedef {import("../declarations/WebpackOptions").WebpackOptionsNormalized} WebpackOptions */
  38. /** @typedef {import("../declarations/WebpackOptions").WebpackPluginInstance} WebpackPluginInstance */
  39. /** @typedef {import("./Chunk")} Chunk */
  40. /** @typedef {import("./Dependency")} Dependency */
  41. /** @typedef {import("./FileSystemInfo").FileSystemInfoEntry} FileSystemInfoEntry */
  42. /** @typedef {import("./Module")} Module */
  43. /** @typedef {import("./util/WeakTupleMap")} WeakTupleMap */
  44. /** @typedef {import("./util/fs").InputFileSystem} InputFileSystem */
  45. /** @typedef {import("./util/fs").IntermediateFileSystem} IntermediateFileSystem */
  46. /** @typedef {import("./util/fs").OutputFileSystem} OutputFileSystem */
  47. /** @typedef {import("./util/fs").WatchFileSystem} WatchFileSystem */
  48. /**
  49. * @typedef {Object} CompilationParams
  50. * @property {NormalModuleFactory} normalModuleFactory
  51. * @property {ContextModuleFactory} contextModuleFactory
  52. */
  53. /**
  54. * @template T
  55. * @callback Callback
  56. * @param {(Error | null)=} err
  57. * @param {T=} result
  58. */
  59. /**
  60. * @callback RunAsChildCallback
  61. * @param {(Error | null)=} err
  62. * @param {Chunk[]=} entries
  63. * @param {Compilation=} compilation
  64. */
  65. /**
  66. * @typedef {Object} AssetEmittedInfo
  67. * @property {Buffer} content
  68. * @property {Source} source
  69. * @property {Compilation} compilation
  70. * @property {string} outputPath
  71. * @property {string} targetPath
  72. */
  73. /**
  74. * @param {string[]} array an array
  75. * @returns {boolean} true, if the array is sorted
  76. */
  77. const isSorted = array => {
  78. for (let i = 1; i < array.length; i++) {
  79. if (array[i - 1] > array[i]) return false;
  80. }
  81. return true;
  82. };
  83. /**
  84. * @param {Object} obj an object
  85. * @param {string[]} keys the keys of the object
  86. * @returns {Object} the object with properties sorted by property name
  87. */
  88. const sortObject = (obj, keys) => {
  89. const o = {};
  90. for (const k of keys.sort()) {
  91. o[k] = obj[k];
  92. }
  93. return o;
  94. };
  95. /**
  96. * @param {string} filename filename
  97. * @param {string | string[] | undefined} hashes list of hashes
  98. * @returns {boolean} true, if the filename contains any hash
  99. */
  100. const includesHash = (filename, hashes) => {
  101. if (!hashes) return false;
  102. if (Array.isArray(hashes)) {
  103. return hashes.some(hash => filename.includes(hash));
  104. } else {
  105. return filename.includes(hashes);
  106. }
  107. };
  108. class Compiler {
  109. /**
  110. * @param {string} context the compilation path
  111. * @param {WebpackOptions} options options
  112. */
  113. constructor(context, options = /** @type {WebpackOptions} */ ({})) {
  114. this.hooks = Object.freeze({
  115. /** @type {SyncHook<[]>} */
  116. initialize: new SyncHook([]),
  117. /** @type {SyncBailHook<[Compilation], boolean | undefined>} */
  118. shouldEmit: new SyncBailHook(["compilation"]),
  119. /** @type {AsyncSeriesHook<[Stats]>} */
  120. done: new AsyncSeriesHook(["stats"]),
  121. /** @type {SyncHook<[Stats]>} */
  122. afterDone: new SyncHook(["stats"]),
  123. /** @type {AsyncSeriesHook<[]>} */
  124. additionalPass: new AsyncSeriesHook([]),
  125. /** @type {AsyncSeriesHook<[Compiler]>} */
  126. beforeRun: new AsyncSeriesHook(["compiler"]),
  127. /** @type {AsyncSeriesHook<[Compiler]>} */
  128. run: new AsyncSeriesHook(["compiler"]),
  129. /** @type {AsyncSeriesHook<[Compilation]>} */
  130. emit: new AsyncSeriesHook(["compilation"]),
  131. /** @type {AsyncSeriesHook<[string, AssetEmittedInfo]>} */
  132. assetEmitted: new AsyncSeriesHook(["file", "info"]),
  133. /** @type {AsyncSeriesHook<[Compilation]>} */
  134. afterEmit: new AsyncSeriesHook(["compilation"]),
  135. /** @type {SyncHook<[Compilation, CompilationParams]>} */
  136. thisCompilation: new SyncHook(["compilation", "params"]),
  137. /** @type {SyncHook<[Compilation, CompilationParams]>} */
  138. compilation: new SyncHook(["compilation", "params"]),
  139. /** @type {SyncHook<[NormalModuleFactory]>} */
  140. normalModuleFactory: new SyncHook(["normalModuleFactory"]),
  141. /** @type {SyncHook<[ContextModuleFactory]>} */
  142. contextModuleFactory: new SyncHook(["contextModuleFactory"]),
  143. /** @type {AsyncSeriesHook<[CompilationParams]>} */
  144. beforeCompile: new AsyncSeriesHook(["params"]),
  145. /** @type {SyncHook<[CompilationParams]>} */
  146. compile: new SyncHook(["params"]),
  147. /** @type {AsyncParallelHook<[Compilation]>} */
  148. make: new AsyncParallelHook(["compilation"]),
  149. /** @type {AsyncParallelHook<[Compilation]>} */
  150. finishMake: new AsyncSeriesHook(["compilation"]),
  151. /** @type {AsyncSeriesHook<[Compilation]>} */
  152. afterCompile: new AsyncSeriesHook(["compilation"]),
  153. /** @type {AsyncSeriesHook<[]>} */
  154. readRecords: new AsyncSeriesHook([]),
  155. /** @type {AsyncSeriesHook<[]>} */
  156. emitRecords: new AsyncSeriesHook([]),
  157. /** @type {AsyncSeriesHook<[Compiler]>} */
  158. watchRun: new AsyncSeriesHook(["compiler"]),
  159. /** @type {SyncHook<[Error]>} */
  160. failed: new SyncHook(["error"]),
  161. /** @type {SyncHook<[string | null, number]>} */
  162. invalid: new SyncHook(["filename", "changeTime"]),
  163. /** @type {SyncHook<[]>} */
  164. watchClose: new SyncHook([]),
  165. /** @type {AsyncSeriesHook<[]>} */
  166. shutdown: new AsyncSeriesHook([]),
  167. /** @type {SyncBailHook<[string, string, any[]], true>} */
  168. infrastructureLog: new SyncBailHook(["origin", "type", "args"]),
  169. // TODO the following hooks are weirdly located here
  170. // TODO move them for webpack 5
  171. /** @type {SyncHook<[]>} */
  172. environment: new SyncHook([]),
  173. /** @type {SyncHook<[]>} */
  174. afterEnvironment: new SyncHook([]),
  175. /** @type {SyncHook<[Compiler]>} */
  176. afterPlugins: new SyncHook(["compiler"]),
  177. /** @type {SyncHook<[Compiler]>} */
  178. afterResolvers: new SyncHook(["compiler"]),
  179. /** @type {SyncBailHook<[string, Entry], boolean>} */
  180. entryOption: new SyncBailHook(["context", "entry"])
  181. });
  182. this.webpack = webpack;
  183. /** @type {string=} */
  184. this.name = undefined;
  185. /** @type {Compilation=} */
  186. this.parentCompilation = undefined;
  187. /** @type {Compiler} */
  188. this.root = this;
  189. /** @type {string} */
  190. this.outputPath = "";
  191. /** @type {Watching | undefined} */
  192. this.watching = undefined;
  193. /** @type {OutputFileSystem} */
  194. this.outputFileSystem = null;
  195. /** @type {IntermediateFileSystem} */
  196. this.intermediateFileSystem = null;
  197. /** @type {InputFileSystem} */
  198. this.inputFileSystem = null;
  199. /** @type {WatchFileSystem} */
  200. this.watchFileSystem = null;
  201. /** @type {string|null} */
  202. this.recordsInputPath = null;
  203. /** @type {string|null} */
  204. this.recordsOutputPath = null;
  205. this.records = {};
  206. /** @type {Set<string | RegExp>} */
  207. this.managedPaths = new Set();
  208. /** @type {Set<string | RegExp>} */
  209. this.unmanagedPaths = new Set();
  210. /** @type {Set<string | RegExp>} */
  211. this.immutablePaths = new Set();
  212. /** @type {ReadonlySet<string> | undefined} */
  213. this.modifiedFiles = undefined;
  214. /** @type {ReadonlySet<string> | undefined} */
  215. this.removedFiles = undefined;
  216. /** @type {ReadonlyMap<string, FileSystemInfoEntry | "ignore" | null> | undefined} */
  217. this.fileTimestamps = undefined;
  218. /** @type {ReadonlyMap<string, FileSystemInfoEntry | "ignore" | null> | undefined} */
  219. this.contextTimestamps = undefined;
  220. /** @type {number | undefined} */
  221. this.fsStartTime = undefined;
  222. /** @type {ResolverFactory} */
  223. this.resolverFactory = new ResolverFactory();
  224. this.infrastructureLogger = undefined;
  225. this.options = options;
  226. this.context = context;
  227. this.requestShortener = new RequestShortener(context, this.root);
  228. this.cache = new Cache();
  229. /** @type {Map<Module, { buildInfo: object, references: WeakMap<Dependency, Module>, memCache: WeakTupleMap }> | undefined} */
  230. this.moduleMemCaches = undefined;
  231. this.compilerPath = "";
  232. /** @type {boolean} */
  233. this.running = false;
  234. /** @type {boolean} */
  235. this.idle = false;
  236. /** @type {boolean} */
  237. this.watchMode = false;
  238. this._backCompat = this.options.experiments.backCompat !== false;
  239. /** @type {Compilation} */
  240. this._lastCompilation = undefined;
  241. /** @type {NormalModuleFactory} */
  242. this._lastNormalModuleFactory = undefined;
  243. /** @private @type {WeakMap<Source, { sizeOnlySource: SizeOnlySource, writtenTo: Map<string, number> }>} */
  244. this._assetEmittingSourceCache = new WeakMap();
  245. /** @private @type {Map<string, number>} */
  246. this._assetEmittingWrittenFiles = new Map();
  247. /** @private @type {Set<string>} */
  248. this._assetEmittingPreviousFiles = new Set();
  249. }
  250. /**
  251. * @param {string} name cache name
  252. * @returns {CacheFacade} the cache facade instance
  253. */
  254. getCache(name) {
  255. return new CacheFacade(
  256. this.cache,
  257. `${this.compilerPath}${name}`,
  258. this.options.output.hashFunction
  259. );
  260. }
  261. /**
  262. * @param {string | (function(): string)} name name of the logger, or function called once to get the logger name
  263. * @returns {Logger} a logger with that name
  264. */
  265. getInfrastructureLogger(name) {
  266. if (!name) {
  267. throw new TypeError(
  268. "Compiler.getInfrastructureLogger(name) called without a name"
  269. );
  270. }
  271. return new Logger(
  272. (type, args) => {
  273. if (typeof name === "function") {
  274. name = name();
  275. if (!name) {
  276. throw new TypeError(
  277. "Compiler.getInfrastructureLogger(name) called with a function not returning a name"
  278. );
  279. }
  280. }
  281. if (this.hooks.infrastructureLog.call(name, type, args) === undefined) {
  282. if (this.infrastructureLogger !== undefined) {
  283. this.infrastructureLogger(name, type, args);
  284. }
  285. }
  286. },
  287. childName => {
  288. if (typeof name === "function") {
  289. if (typeof childName === "function") {
  290. return this.getInfrastructureLogger(() => {
  291. if (typeof name === "function") {
  292. name = name();
  293. if (!name) {
  294. throw new TypeError(
  295. "Compiler.getInfrastructureLogger(name) called with a function not returning a name"
  296. );
  297. }
  298. }
  299. if (typeof childName === "function") {
  300. childName = childName();
  301. if (!childName) {
  302. throw new TypeError(
  303. "Logger.getChildLogger(name) called with a function not returning a name"
  304. );
  305. }
  306. }
  307. return `${name}/${childName}`;
  308. });
  309. } else {
  310. return this.getInfrastructureLogger(() => {
  311. if (typeof name === "function") {
  312. name = name();
  313. if (!name) {
  314. throw new TypeError(
  315. "Compiler.getInfrastructureLogger(name) called with a function not returning a name"
  316. );
  317. }
  318. }
  319. return `${name}/${childName}`;
  320. });
  321. }
  322. } else {
  323. if (typeof childName === "function") {
  324. return this.getInfrastructureLogger(() => {
  325. if (typeof childName === "function") {
  326. childName = childName();
  327. if (!childName) {
  328. throw new TypeError(
  329. "Logger.getChildLogger(name) called with a function not returning a name"
  330. );
  331. }
  332. }
  333. return `${name}/${childName}`;
  334. });
  335. } else {
  336. return this.getInfrastructureLogger(`${name}/${childName}`);
  337. }
  338. }
  339. }
  340. );
  341. }
  342. // TODO webpack 6: solve this in a better way
  343. // e.g. move compilation specific info from Modules into ModuleGraph
  344. _cleanupLastCompilation() {
  345. if (this._lastCompilation !== undefined) {
  346. for (const childCompilation of this._lastCompilation.children) {
  347. for (const module of childCompilation.modules) {
  348. ChunkGraph.clearChunkGraphForModule(module);
  349. ModuleGraph.clearModuleGraphForModule(module);
  350. module.cleanupForCache();
  351. }
  352. for (const chunk of childCompilation.chunks) {
  353. ChunkGraph.clearChunkGraphForChunk(chunk);
  354. }
  355. }
  356. for (const module of this._lastCompilation.modules) {
  357. ChunkGraph.clearChunkGraphForModule(module);
  358. ModuleGraph.clearModuleGraphForModule(module);
  359. module.cleanupForCache();
  360. }
  361. for (const chunk of this._lastCompilation.chunks) {
  362. ChunkGraph.clearChunkGraphForChunk(chunk);
  363. }
  364. this._lastCompilation = undefined;
  365. }
  366. }
  367. // TODO webpack 6: solve this in a better way
  368. _cleanupLastNormalModuleFactory() {
  369. if (this._lastNormalModuleFactory !== undefined) {
  370. this._lastNormalModuleFactory.cleanupForCache();
  371. this._lastNormalModuleFactory = undefined;
  372. }
  373. }
  374. /**
  375. * @param {WatchOptions} watchOptions the watcher's options
  376. * @param {Callback<Stats>} handler signals when the call finishes
  377. * @returns {Watching} a compiler watcher
  378. */
  379. watch(watchOptions, handler) {
  380. if (this.running) {
  381. return handler(new ConcurrentCompilationError());
  382. }
  383. this.running = true;
  384. this.watchMode = true;
  385. this.watching = new Watching(this, watchOptions, handler);
  386. return this.watching;
  387. }
  388. /**
  389. * @param {Callback<Stats>} callback signals when the call finishes
  390. * @returns {void}
  391. */
  392. run(callback) {
  393. if (this.running) {
  394. return callback(new ConcurrentCompilationError());
  395. }
  396. let logger;
  397. const finalCallback = (err, stats) => {
  398. if (logger) logger.time("beginIdle");
  399. this.idle = true;
  400. this.cache.beginIdle();
  401. this.idle = true;
  402. if (logger) logger.timeEnd("beginIdle");
  403. this.running = false;
  404. if (err) {
  405. this.hooks.failed.call(err);
  406. }
  407. if (callback !== undefined) callback(err, stats);
  408. this.hooks.afterDone.call(stats);
  409. };
  410. const startTime = Date.now();
  411. this.running = true;
  412. const onCompiled = (err, compilation) => {
  413. if (err) return finalCallback(err);
  414. if (this.hooks.shouldEmit.call(compilation) === false) {
  415. compilation.startTime = startTime;
  416. compilation.endTime = Date.now();
  417. const stats = new Stats(compilation);
  418. this.hooks.done.callAsync(stats, err => {
  419. if (err) return finalCallback(err);
  420. return finalCallback(null, stats);
  421. });
  422. return;
  423. }
  424. process.nextTick(() => {
  425. logger = compilation.getLogger("webpack.Compiler");
  426. logger.time("emitAssets");
  427. this.emitAssets(compilation, err => {
  428. logger.timeEnd("emitAssets");
  429. if (err) return finalCallback(err);
  430. if (compilation.hooks.needAdditionalPass.call()) {
  431. compilation.needAdditionalPass = true;
  432. compilation.startTime = startTime;
  433. compilation.endTime = Date.now();
  434. logger.time("done hook");
  435. const stats = new Stats(compilation);
  436. this.hooks.done.callAsync(stats, err => {
  437. logger.timeEnd("done hook");
  438. if (err) return finalCallback(err);
  439. this.hooks.additionalPass.callAsync(err => {
  440. if (err) return finalCallback(err);
  441. this.compile(onCompiled);
  442. });
  443. });
  444. return;
  445. }
  446. logger.time("emitRecords");
  447. this.emitRecords(err => {
  448. logger.timeEnd("emitRecords");
  449. if (err) return finalCallback(err);
  450. compilation.startTime = startTime;
  451. compilation.endTime = Date.now();
  452. logger.time("done hook");
  453. const stats = new Stats(compilation);
  454. this.hooks.done.callAsync(stats, err => {
  455. logger.timeEnd("done hook");
  456. if (err) return finalCallback(err);
  457. this.cache.storeBuildDependencies(
  458. compilation.buildDependencies,
  459. err => {
  460. if (err) return finalCallback(err);
  461. return finalCallback(null, stats);
  462. }
  463. );
  464. });
  465. });
  466. });
  467. });
  468. };
  469. const run = () => {
  470. this.hooks.beforeRun.callAsync(this, err => {
  471. if (err) return finalCallback(err);
  472. this.hooks.run.callAsync(this, err => {
  473. if (err) return finalCallback(err);
  474. this.readRecords(err => {
  475. if (err) return finalCallback(err);
  476. this.compile(onCompiled);
  477. });
  478. });
  479. });
  480. };
  481. if (this.idle) {
  482. this.cache.endIdle(err => {
  483. if (err) return finalCallback(err);
  484. this.idle = false;
  485. run();
  486. });
  487. } else {
  488. run();
  489. }
  490. }
  491. /**
  492. * @param {RunAsChildCallback} callback signals when the call finishes
  493. * @returns {void}
  494. */
  495. runAsChild(callback) {
  496. const startTime = Date.now();
  497. const finalCallback = (err, entries, compilation) => {
  498. try {
  499. callback(err, entries, compilation);
  500. } catch (e) {
  501. const err = new WebpackError(
  502. `compiler.runAsChild callback error: ${e}`
  503. );
  504. err.details = e.stack;
  505. this.parentCompilation.errors.push(err);
  506. }
  507. };
  508. this.compile((err, compilation) => {
  509. if (err) return finalCallback(err);
  510. this.parentCompilation.children.push(compilation);
  511. for (const { name, source, info } of compilation.getAssets()) {
  512. this.parentCompilation.emitAsset(name, source, info);
  513. }
  514. const entries = [];
  515. for (const ep of compilation.entrypoints.values()) {
  516. entries.push(...ep.chunks);
  517. }
  518. compilation.startTime = startTime;
  519. compilation.endTime = Date.now();
  520. return finalCallback(null, entries, compilation);
  521. });
  522. }
  523. purgeInputFileSystem() {
  524. if (this.inputFileSystem && this.inputFileSystem.purge) {
  525. this.inputFileSystem.purge();
  526. }
  527. }
  528. /**
  529. * @param {Compilation} compilation the compilation
  530. * @param {Callback<void>} callback signals when the assets are emitted
  531. * @returns {void}
  532. */
  533. emitAssets(compilation, callback) {
  534. let outputPath;
  535. const emitFiles = err => {
  536. if (err) return callback(err);
  537. const assets = compilation.getAssets();
  538. compilation.assets = { ...compilation.assets };
  539. /** @type {Map<string, { path: string, source: Source, size: number, waiting: { cacheEntry: any, file: string }[] }>} */
  540. const caseInsensitiveMap = new Map();
  541. /** @type {Set<string>} */
  542. const allTargetPaths = new Set();
  543. asyncLib.forEachLimit(
  544. assets,
  545. 15,
  546. ({ name: file, source, info }, callback) => {
  547. let targetFile = file;
  548. let immutable = info.immutable;
  549. const queryStringIdx = targetFile.indexOf("?");
  550. if (queryStringIdx >= 0) {
  551. targetFile = targetFile.slice(0, queryStringIdx);
  552. // We may remove the hash, which is in the query string
  553. // So we recheck if the file is immutable
  554. // This doesn't cover all cases, but immutable is only a performance optimization anyway
  555. immutable =
  556. immutable &&
  557. (includesHash(targetFile, info.contenthash) ||
  558. includesHash(targetFile, info.chunkhash) ||
  559. includesHash(targetFile, info.modulehash) ||
  560. includesHash(targetFile, info.fullhash));
  561. }
  562. const writeOut = err => {
  563. if (err) return callback(err);
  564. const targetPath = join(
  565. this.outputFileSystem,
  566. outputPath,
  567. targetFile
  568. );
  569. allTargetPaths.add(targetPath);
  570. // check if the target file has already been written by this Compiler
  571. const targetFileGeneration =
  572. this._assetEmittingWrittenFiles.get(targetPath);
  573. // create an cache entry for this Source if not already existing
  574. let cacheEntry = this._assetEmittingSourceCache.get(source);
  575. if (cacheEntry === undefined) {
  576. cacheEntry = {
  577. sizeOnlySource: undefined,
  578. writtenTo: new Map()
  579. };
  580. this._assetEmittingSourceCache.set(source, cacheEntry);
  581. }
  582. let similarEntry;
  583. const checkSimilarFile = () => {
  584. const caseInsensitiveTargetPath = targetPath.toLowerCase();
  585. similarEntry = caseInsensitiveMap.get(caseInsensitiveTargetPath);
  586. if (similarEntry !== undefined) {
  587. const { path: other, source: otherSource } = similarEntry;
  588. if (isSourceEqual(otherSource, source)) {
  589. // Size may or may not be available at this point.
  590. // If it's not available add to "waiting" list and it will be updated once available
  591. if (similarEntry.size !== undefined) {
  592. updateWithReplacementSource(similarEntry.size);
  593. } else {
  594. if (!similarEntry.waiting) similarEntry.waiting = [];
  595. similarEntry.waiting.push({ file, cacheEntry });
  596. }
  597. alreadyWritten();
  598. } else {
  599. const err =
  600. new WebpackError(`Prevent writing to file that only differs in casing or query string from already written file.
  601. This will lead to a race-condition and corrupted files on case-insensitive file systems.
  602. ${targetPath}
  603. ${other}`);
  604. err.file = file;
  605. callback(err);
  606. }
  607. return true;
  608. } else {
  609. caseInsensitiveMap.set(
  610. caseInsensitiveTargetPath,
  611. (similarEntry = {
  612. path: targetPath,
  613. source,
  614. size: undefined,
  615. waiting: undefined
  616. })
  617. );
  618. return false;
  619. }
  620. };
  621. /**
  622. * get the binary (Buffer) content from the Source
  623. * @returns {Buffer} content for the source
  624. */
  625. const getContent = () => {
  626. if (typeof source.buffer === "function") {
  627. return source.buffer();
  628. } else {
  629. const bufferOrString = source.source();
  630. if (Buffer.isBuffer(bufferOrString)) {
  631. return bufferOrString;
  632. } else {
  633. return Buffer.from(bufferOrString, "utf8");
  634. }
  635. }
  636. };
  637. const alreadyWritten = () => {
  638. // cache the information that the Source has been already been written to that location
  639. if (targetFileGeneration === undefined) {
  640. const newGeneration = 1;
  641. this._assetEmittingWrittenFiles.set(targetPath, newGeneration);
  642. cacheEntry.writtenTo.set(targetPath, newGeneration);
  643. } else {
  644. cacheEntry.writtenTo.set(targetPath, targetFileGeneration);
  645. }
  646. callback();
  647. };
  648. /**
  649. * Write the file to output file system
  650. * @param {Buffer} content content to be written
  651. * @returns {void}
  652. */
  653. const doWrite = content => {
  654. this.outputFileSystem.writeFile(targetPath, content, err => {
  655. if (err) return callback(err);
  656. // information marker that the asset has been emitted
  657. compilation.emittedAssets.add(file);
  658. // cache the information that the Source has been written to that location
  659. const newGeneration =
  660. targetFileGeneration === undefined
  661. ? 1
  662. : targetFileGeneration + 1;
  663. cacheEntry.writtenTo.set(targetPath, newGeneration);
  664. this._assetEmittingWrittenFiles.set(targetPath, newGeneration);
  665. this.hooks.assetEmitted.callAsync(
  666. file,
  667. {
  668. content,
  669. source,
  670. outputPath,
  671. compilation,
  672. targetPath
  673. },
  674. callback
  675. );
  676. });
  677. };
  678. const updateWithReplacementSource = size => {
  679. updateFileWithReplacementSource(file, cacheEntry, size);
  680. similarEntry.size = size;
  681. if (similarEntry.waiting !== undefined) {
  682. for (const { file, cacheEntry } of similarEntry.waiting) {
  683. updateFileWithReplacementSource(file, cacheEntry, size);
  684. }
  685. }
  686. };
  687. const updateFileWithReplacementSource = (
  688. file,
  689. cacheEntry,
  690. size
  691. ) => {
  692. // Create a replacement resource which only allows to ask for size
  693. // This allows to GC all memory allocated by the Source
  694. // (expect when the Source is stored in any other cache)
  695. if (!cacheEntry.sizeOnlySource) {
  696. cacheEntry.sizeOnlySource = new SizeOnlySource(size);
  697. }
  698. compilation.updateAsset(file, cacheEntry.sizeOnlySource, {
  699. size
  700. });
  701. };
  702. const processExistingFile = stats => {
  703. // skip emitting if it's already there and an immutable file
  704. if (immutable) {
  705. updateWithReplacementSource(stats.size);
  706. return alreadyWritten();
  707. }
  708. const content = getContent();
  709. updateWithReplacementSource(content.length);
  710. // if it exists and content on disk matches content
  711. // skip writing the same content again
  712. // (to keep mtime and don't trigger watchers)
  713. // for a fast negative match file size is compared first
  714. if (content.length === stats.size) {
  715. compilation.comparedForEmitAssets.add(file);
  716. return this.outputFileSystem.readFile(
  717. targetPath,
  718. (err, existingContent) => {
  719. if (
  720. err ||
  721. !content.equals(/** @type {Buffer} */ (existingContent))
  722. ) {
  723. return doWrite(content);
  724. } else {
  725. return alreadyWritten();
  726. }
  727. }
  728. );
  729. }
  730. return doWrite(content);
  731. };
  732. const processMissingFile = () => {
  733. const content = getContent();
  734. updateWithReplacementSource(content.length);
  735. return doWrite(content);
  736. };
  737. // if the target file has already been written
  738. if (targetFileGeneration !== undefined) {
  739. // check if the Source has been written to this target file
  740. const writtenGeneration = cacheEntry.writtenTo.get(targetPath);
  741. if (writtenGeneration === targetFileGeneration) {
  742. // if yes, we may skip writing the file
  743. // if it's already there
  744. // (we assume one doesn't modify files while the Compiler is running, other then removing them)
  745. if (this._assetEmittingPreviousFiles.has(targetPath)) {
  746. // We assume that assets from the last compilation say intact on disk (they are not removed)
  747. compilation.updateAsset(file, cacheEntry.sizeOnlySource, {
  748. size: cacheEntry.sizeOnlySource.size()
  749. });
  750. return callback();
  751. } else {
  752. // Settings immutable will make it accept file content without comparing when file exist
  753. immutable = true;
  754. }
  755. } else if (!immutable) {
  756. if (checkSimilarFile()) return;
  757. // We wrote to this file before which has very likely a different content
  758. // skip comparing and assume content is different for performance
  759. // This case happens often during watch mode.
  760. return processMissingFile();
  761. }
  762. }
  763. if (checkSimilarFile()) return;
  764. if (this.options.output.compareBeforeEmit) {
  765. this.outputFileSystem.stat(targetPath, (err, stats) => {
  766. const exists = !err && stats.isFile();
  767. if (exists) {
  768. processExistingFile(stats);
  769. } else {
  770. processMissingFile();
  771. }
  772. });
  773. } else {
  774. processMissingFile();
  775. }
  776. };
  777. if (targetFile.match(/\/|\\/)) {
  778. const fs = this.outputFileSystem;
  779. const dir = dirname(fs, join(fs, outputPath, targetFile));
  780. mkdirp(fs, dir, writeOut);
  781. } else {
  782. writeOut();
  783. }
  784. },
  785. err => {
  786. // Clear map to free up memory
  787. caseInsensitiveMap.clear();
  788. if (err) {
  789. this._assetEmittingPreviousFiles.clear();
  790. return callback(err);
  791. }
  792. this._assetEmittingPreviousFiles = allTargetPaths;
  793. this.hooks.afterEmit.callAsync(compilation, err => {
  794. if (err) return callback(err);
  795. return callback();
  796. });
  797. }
  798. );
  799. };
  800. this.hooks.emit.callAsync(compilation, err => {
  801. if (err) return callback(err);
  802. outputPath = compilation.getPath(this.outputPath, {});
  803. mkdirp(this.outputFileSystem, outputPath, emitFiles);
  804. });
  805. }
  806. /**
  807. * @param {Callback<void>} callback signals when the call finishes
  808. * @returns {void}
  809. */
  810. emitRecords(callback) {
  811. if (this.hooks.emitRecords.isUsed()) {
  812. if (this.recordsOutputPath) {
  813. asyncLib.parallel(
  814. [
  815. cb => this.hooks.emitRecords.callAsync(cb),
  816. this._emitRecords.bind(this)
  817. ],
  818. err => callback(err)
  819. );
  820. } else {
  821. this.hooks.emitRecords.callAsync(callback);
  822. }
  823. } else {
  824. if (this.recordsOutputPath) {
  825. this._emitRecords(callback);
  826. } else {
  827. callback();
  828. }
  829. }
  830. }
  831. /**
  832. * @param {Callback<void>} callback signals when the call finishes
  833. * @returns {void}
  834. */
  835. _emitRecords(callback) {
  836. const writeFile = () => {
  837. this.outputFileSystem.writeFile(
  838. this.recordsOutputPath,
  839. JSON.stringify(
  840. this.records,
  841. (n, value) => {
  842. if (
  843. typeof value === "object" &&
  844. value !== null &&
  845. !Array.isArray(value)
  846. ) {
  847. const keys = Object.keys(value);
  848. if (!isSorted(keys)) {
  849. return sortObject(value, keys);
  850. }
  851. }
  852. return value;
  853. },
  854. 2
  855. ),
  856. callback
  857. );
  858. };
  859. const recordsOutputPathDirectory = dirname(
  860. this.outputFileSystem,
  861. this.recordsOutputPath
  862. );
  863. if (!recordsOutputPathDirectory) {
  864. return writeFile();
  865. }
  866. mkdirp(this.outputFileSystem, recordsOutputPathDirectory, err => {
  867. if (err) return callback(err);
  868. writeFile();
  869. });
  870. }
  871. /**
  872. * @param {Callback<void>} callback signals when the call finishes
  873. * @returns {void}
  874. */
  875. readRecords(callback) {
  876. if (this.hooks.readRecords.isUsed()) {
  877. if (this.recordsInputPath) {
  878. asyncLib.parallel(
  879. [
  880. cb => this.hooks.readRecords.callAsync(cb),
  881. this._readRecords.bind(this)
  882. ],
  883. err => callback(err)
  884. );
  885. } else {
  886. this.records = {};
  887. this.hooks.readRecords.callAsync(callback);
  888. }
  889. } else {
  890. if (this.recordsInputPath) {
  891. this._readRecords(callback);
  892. } else {
  893. this.records = {};
  894. callback();
  895. }
  896. }
  897. }
  898. /**
  899. * @param {Callback<void>} callback signals when the call finishes
  900. * @returns {void}
  901. */
  902. _readRecords(callback) {
  903. if (!this.recordsInputPath) {
  904. this.records = {};
  905. return callback();
  906. }
  907. this.inputFileSystem.stat(this.recordsInputPath, err => {
  908. // It doesn't exist
  909. // We can ignore this.
  910. if (err) return callback();
  911. this.inputFileSystem.readFile(this.recordsInputPath, (err, content) => {
  912. if (err) return callback(err);
  913. try {
  914. this.records = parseJson(content.toString("utf-8"));
  915. } catch (e) {
  916. return callback(new Error(`Cannot parse records: ${e.message}`));
  917. }
  918. return callback();
  919. });
  920. });
  921. }
  922. /**
  923. * @param {Compilation} compilation the compilation
  924. * @param {string} compilerName the compiler's name
  925. * @param {number} compilerIndex the compiler's index
  926. * @param {OutputOptions=} outputOptions the output options
  927. * @param {WebpackPluginInstance[]=} plugins the plugins to apply
  928. * @returns {Compiler} a child compiler
  929. */
  930. createChildCompiler(
  931. compilation,
  932. compilerName,
  933. compilerIndex,
  934. outputOptions,
  935. plugins
  936. ) {
  937. const childCompiler = new Compiler(this.context, {
  938. ...this.options,
  939. output: {
  940. ...this.options.output,
  941. ...outputOptions
  942. }
  943. });
  944. childCompiler.name = compilerName;
  945. childCompiler.outputPath = this.outputPath;
  946. childCompiler.inputFileSystem = this.inputFileSystem;
  947. childCompiler.outputFileSystem = null;
  948. childCompiler.resolverFactory = this.resolverFactory;
  949. childCompiler.modifiedFiles = this.modifiedFiles;
  950. childCompiler.removedFiles = this.removedFiles;
  951. childCompiler.fileTimestamps = this.fileTimestamps;
  952. childCompiler.contextTimestamps = this.contextTimestamps;
  953. childCompiler.fsStartTime = this.fsStartTime;
  954. childCompiler.cache = this.cache;
  955. childCompiler.compilerPath = `${this.compilerPath}${compilerName}|${compilerIndex}|`;
  956. childCompiler._backCompat = this._backCompat;
  957. const relativeCompilerName = makePathsRelative(
  958. this.context,
  959. compilerName,
  960. this.root
  961. );
  962. if (!this.records[relativeCompilerName]) {
  963. this.records[relativeCompilerName] = [];
  964. }
  965. if (this.records[relativeCompilerName][compilerIndex]) {
  966. childCompiler.records = this.records[relativeCompilerName][compilerIndex];
  967. } else {
  968. this.records[relativeCompilerName].push((childCompiler.records = {}));
  969. }
  970. childCompiler.parentCompilation = compilation;
  971. childCompiler.root = this.root;
  972. if (Array.isArray(plugins)) {
  973. for (const plugin of plugins) {
  974. if (plugin) {
  975. plugin.apply(childCompiler);
  976. }
  977. }
  978. }
  979. for (const name in this.hooks) {
  980. if (
  981. ![
  982. "make",
  983. "compile",
  984. "emit",
  985. "afterEmit",
  986. "invalid",
  987. "done",
  988. "thisCompilation"
  989. ].includes(name)
  990. ) {
  991. if (childCompiler.hooks[name]) {
  992. childCompiler.hooks[name].taps = this.hooks[name].taps.slice();
  993. }
  994. }
  995. }
  996. compilation.hooks.childCompiler.call(
  997. childCompiler,
  998. compilerName,
  999. compilerIndex
  1000. );
  1001. return childCompiler;
  1002. }
  1003. isChild() {
  1004. return !!this.parentCompilation;
  1005. }
  1006. createCompilation(params) {
  1007. this._cleanupLastCompilation();
  1008. return (this._lastCompilation = new Compilation(this, params));
  1009. }
  1010. /**
  1011. * @param {CompilationParams} params the compilation parameters
  1012. * @returns {Compilation} the created compilation
  1013. */
  1014. newCompilation(params) {
  1015. const compilation = this.createCompilation(params);
  1016. compilation.name = this.name;
  1017. compilation.records = this.records;
  1018. this.hooks.thisCompilation.call(compilation, params);
  1019. this.hooks.compilation.call(compilation, params);
  1020. return compilation;
  1021. }
  1022. createNormalModuleFactory() {
  1023. this._cleanupLastNormalModuleFactory();
  1024. const normalModuleFactory = new NormalModuleFactory({
  1025. context: this.options.context,
  1026. fs: this.inputFileSystem,
  1027. resolverFactory: this.resolverFactory,
  1028. options: this.options.module,
  1029. associatedObjectForCache: this.root,
  1030. layers: this.options.experiments.layers
  1031. });
  1032. this._lastNormalModuleFactory = normalModuleFactory;
  1033. this.hooks.normalModuleFactory.call(normalModuleFactory);
  1034. return normalModuleFactory;
  1035. }
  1036. createContextModuleFactory() {
  1037. const contextModuleFactory = new ContextModuleFactory(this.resolverFactory);
  1038. this.hooks.contextModuleFactory.call(contextModuleFactory);
  1039. return contextModuleFactory;
  1040. }
  1041. newCompilationParams() {
  1042. const params = {
  1043. normalModuleFactory: this.createNormalModuleFactory(),
  1044. contextModuleFactory: this.createContextModuleFactory()
  1045. };
  1046. return params;
  1047. }
  1048. /**
  1049. * @param {Callback<Compilation>} callback signals when the compilation finishes
  1050. * @returns {void}
  1051. */
  1052. compile(callback) {
  1053. const params = this.newCompilationParams();
  1054. this.hooks.beforeCompile.callAsync(params, err => {
  1055. if (err) return callback(err);
  1056. this.hooks.compile.call(params);
  1057. const compilation = this.newCompilation(params);
  1058. const logger = compilation.getLogger("webpack.Compiler");
  1059. logger.time("make hook");
  1060. this.hooks.make.callAsync(compilation, err => {
  1061. logger.timeEnd("make hook");
  1062. if (err) return callback(err);
  1063. logger.time("finish make hook");
  1064. this.hooks.finishMake.callAsync(compilation, err => {
  1065. logger.timeEnd("finish make hook");
  1066. if (err) return callback(err);
  1067. process.nextTick(() => {
  1068. logger.time("finish compilation");
  1069. compilation.finish(err => {
  1070. logger.timeEnd("finish compilation");
  1071. if (err) return callback(err);
  1072. logger.time("seal compilation");
  1073. compilation.seal(err => {
  1074. logger.timeEnd("seal compilation");
  1075. if (err) return callback(err);
  1076. logger.time("afterCompile hook");
  1077. this.hooks.afterCompile.callAsync(compilation, err => {
  1078. logger.timeEnd("afterCompile hook");
  1079. if (err) return callback(err);
  1080. return callback(null, compilation);
  1081. });
  1082. });
  1083. });
  1084. });
  1085. });
  1086. });
  1087. });
  1088. }
  1089. /**
  1090. * @param {Callback<void>} callback signals when the compiler closes
  1091. * @returns {void}
  1092. */
  1093. close(callback) {
  1094. if (this.watching) {
  1095. // When there is still an active watching, close this first
  1096. this.watching.close(err => {
  1097. this.close(callback);
  1098. });
  1099. return;
  1100. }
  1101. this.hooks.shutdown.callAsync(err => {
  1102. if (err) return callback(err);
  1103. // Get rid of reference to last compilation to avoid leaking memory
  1104. // We can't run this._cleanupLastCompilation() as the Stats to this compilation
  1105. // might be still in use. We try to get rid of the reference to the cache instead.
  1106. this._lastCompilation = undefined;
  1107. this._lastNormalModuleFactory = undefined;
  1108. this.cache.shutdown(callback);
  1109. });
  1110. }
  1111. }
  1112. module.exports = Compiler;