forked from daren.hsu/line_push
update
This commit is contained in:
+6
-8
@@ -1,19 +1,17 @@
|
||||
'use strict';
|
||||
"use strict";
|
||||
|
||||
Object.defineProperty(exports, "__esModule", {
|
||||
value: true
|
||||
});
|
||||
exports.default = void 0;
|
||||
|
||||
const stack = (err, worker, workerId) => {
|
||||
const originError = (err.stack || '').split('\n').filter(line => line.trim().startsWith('at'));
|
||||
|
||||
const workerError = worker.split('\n').filter(line => line.trim().startsWith('at'));
|
||||
|
||||
const diff = workerError.slice(0, workerError.length - originError.length).join('\n');
|
||||
|
||||
originError.unshift(diff);
|
||||
originError.unshift(err.message);
|
||||
originError.unshift(`Thread Loader (Worker ${workerId})`);
|
||||
|
||||
return originError.join('\n');
|
||||
};
|
||||
|
||||
@@ -22,11 +20,11 @@ class WorkerError extends Error {
|
||||
super(err);
|
||||
this.name = err.name;
|
||||
this.message = err.message;
|
||||
|
||||
Error.captureStackTrace(this, this.constructor);
|
||||
|
||||
this.stack = stack(err, this.stack, workerId);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
exports.default = WorkerError;
|
||||
var _default = WorkerError;
|
||||
exports.default = _default;
|
||||
+138
-46
@@ -1,32 +1,26 @@
|
||||
'use strict';
|
||||
"use strict";
|
||||
|
||||
Object.defineProperty(exports, "__esModule", {
|
||||
value: true
|
||||
});
|
||||
exports.default = void 0;
|
||||
|
||||
var _child_process = require('child_process');
|
||||
var _child_process = _interopRequireDefault(require("child_process"));
|
||||
|
||||
var _child_process2 = _interopRequireDefault(_child_process);
|
||||
var _queue = _interopRequireDefault(require("neo-async/queue"));
|
||||
|
||||
var _queue = require('neo-async/queue');
|
||||
var _mapSeries = _interopRequireDefault(require("neo-async/mapSeries"));
|
||||
|
||||
var _queue2 = _interopRequireDefault(_queue);
|
||||
var _readBuffer = _interopRequireDefault(require("./readBuffer"));
|
||||
|
||||
var _mapSeries = require('neo-async/mapSeries');
|
||||
var _WorkerError = _interopRequireDefault(require("./WorkerError"));
|
||||
|
||||
var _mapSeries2 = _interopRequireDefault(_mapSeries);
|
||||
|
||||
var _readBuffer = require('./readBuffer');
|
||||
|
||||
var _readBuffer2 = _interopRequireDefault(_readBuffer);
|
||||
|
||||
var _WorkerError = require('./WorkerError');
|
||||
|
||||
var _WorkerError2 = _interopRequireDefault(_WorkerError);
|
||||
var _serializer = require("./serializer");
|
||||
|
||||
function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; }
|
||||
|
||||
const workerPath = require.resolve('./worker'); /* eslint-disable no-console */
|
||||
/* eslint-disable no-console */
|
||||
const workerPath = require.resolve('./worker');
|
||||
|
||||
let workerId = 0;
|
||||
|
||||
@@ -38,21 +32,17 @@ class PoolWorker {
|
||||
this.activeJobs = 0;
|
||||
this.onJobDone = onJobDone;
|
||||
this.id = workerId;
|
||||
workerId += 1; // Empty or invalid node args would break the child process
|
||||
|
||||
workerId += 1;
|
||||
// Empty or invalid node args would break the child process
|
||||
const sanitizedNodeArgs = (options.nodeArgs || []).filter(opt => !!opt);
|
||||
|
||||
this.worker = _child_process2.default.spawn(process.execPath, [].concat(sanitizedNodeArgs).concat(workerPath, options.parallelJobs), {
|
||||
this.worker = _child_process.default.spawn(process.execPath, [].concat(sanitizedNodeArgs).concat(workerPath, options.parallelJobs), {
|
||||
detached: true,
|
||||
stdio: ['ignore', 'pipe', 'pipe', 'pipe', 'pipe']
|
||||
});
|
||||
|
||||
this.worker.unref();
|
||||
|
||||
// This prevents a problem where the worker stdio can be undefined
|
||||
this.worker.unref(); // This prevents a problem where the worker stdio can be undefined
|
||||
// when the kernel hits the limit of open files.
|
||||
// More info can be found on: https://github.com/webpack-contrib/thread-loader/issues/2
|
||||
|
||||
if (!this.worker.stdio) {
|
||||
throw new Error(`Failed to create the worker pool with workerId: ${workerId} and ${''}configuration: ${JSON.stringify(options)}. Please verify if you hit the OS open files limit.`);
|
||||
}
|
||||
@@ -99,7 +89,10 @@ class PoolWorker {
|
||||
run(data, callback) {
|
||||
const jobId = this.nextJobId;
|
||||
this.nextJobId += 1;
|
||||
this.jobs[jobId] = { data, callback };
|
||||
this.jobs[jobId] = {
|
||||
data,
|
||||
callback
|
||||
};
|
||||
this.activeJobs += 1;
|
||||
this.writeJson({
|
||||
type: 'job',
|
||||
@@ -117,7 +110,7 @@ class PoolWorker {
|
||||
|
||||
writeJson(data) {
|
||||
const lengthBuffer = Buffer.alloc(4);
|
||||
const messageBuffer = Buffer.from(JSON.stringify(data), 'utf-8');
|
||||
const messageBuffer = Buffer.from(JSON.stringify(data, _serializer.replacer), 'utf-8');
|
||||
lengthBuffer.writeInt32BE(messageBuffer.length, 0);
|
||||
this.writePipe.write(lengthBuffer);
|
||||
this.writePipe.write(messageBuffer);
|
||||
@@ -136,24 +129,26 @@ class PoolWorker {
|
||||
console.error(`Failed to communicate with worker (read length) ${lengthReadError}`);
|
||||
return;
|
||||
}
|
||||
|
||||
this.state = 'length read';
|
||||
const length = lengthBuffer.readInt32BE(0);
|
||||
|
||||
this.state = 'read message';
|
||||
this.readBuffer(length, (messageError, messageBuffer) => {
|
||||
if (messageError) {
|
||||
console.error(`Failed to communicate with worker (read message) ${messageError}`);
|
||||
return;
|
||||
}
|
||||
|
||||
this.state = 'message read';
|
||||
const messageString = messageBuffer.toString('utf-8');
|
||||
const message = JSON.parse(messageString);
|
||||
const message = JSON.parse(messageString, _serializer.reviver);
|
||||
this.state = 'process message';
|
||||
this.onWorkerMessage(message, err => {
|
||||
if (err) {
|
||||
console.error(`Failed to communicate with worker (process message) ${err}`);
|
||||
return;
|
||||
}
|
||||
|
||||
this.state = 'soon next';
|
||||
setImmediate(() => this.readNextMessage());
|
||||
});
|
||||
@@ -162,57 +157,85 @@ class PoolWorker {
|
||||
}
|
||||
|
||||
onWorkerMessage(message, finalCallback) {
|
||||
const { type, id } = message;
|
||||
const {
|
||||
type,
|
||||
id
|
||||
} = message;
|
||||
|
||||
switch (type) {
|
||||
case 'job':
|
||||
{
|
||||
const { data, error, result } = message;
|
||||
(0, _mapSeries2.default)(data, (length, callback) => this.readBuffer(length, callback), (eachErr, buffers) => {
|
||||
const { callback: jobCallback } = this.jobs[id];
|
||||
const {
|
||||
data,
|
||||
error,
|
||||
result
|
||||
} = message;
|
||||
(0, _mapSeries.default)(data, (length, callback) => this.readBuffer(length, callback), (eachErr, buffers) => {
|
||||
const {
|
||||
callback: jobCallback
|
||||
} = this.jobs[id];
|
||||
|
||||
const callback = (err, arg) => {
|
||||
if (jobCallback) {
|
||||
delete this.jobs[id];
|
||||
this.activeJobs -= 1;
|
||||
this.onJobDone();
|
||||
|
||||
if (err) {
|
||||
jobCallback(err instanceof Error ? err : new Error(err), arg);
|
||||
} else {
|
||||
jobCallback(null, arg);
|
||||
}
|
||||
}
|
||||
|
||||
finalCallback();
|
||||
};
|
||||
|
||||
if (eachErr) {
|
||||
callback(eachErr);
|
||||
return;
|
||||
}
|
||||
|
||||
let bufferPosition = 0;
|
||||
|
||||
if (result.result) {
|
||||
result.result = result.result.map(r => {
|
||||
if (r.buffer) {
|
||||
const buffer = buffers[bufferPosition];
|
||||
bufferPosition += 1;
|
||||
|
||||
if (r.string) {
|
||||
return buffer.toString('utf-8');
|
||||
}
|
||||
|
||||
return buffer;
|
||||
}
|
||||
|
||||
return r.data;
|
||||
});
|
||||
}
|
||||
|
||||
if (error) {
|
||||
callback(this.fromErrorObj(error), result);
|
||||
return;
|
||||
}
|
||||
|
||||
callback(null, result);
|
||||
});
|
||||
break;
|
||||
}
|
||||
case 'resolve':
|
||||
|
||||
case 'loadModule':
|
||||
{
|
||||
const { context, request, questionId } = message;
|
||||
const { data } = this.jobs[id];
|
||||
data.resolve(context, request, (error, result) => {
|
||||
const {
|
||||
request,
|
||||
questionId
|
||||
} = message;
|
||||
const {
|
||||
data
|
||||
} = this.jobs[id]; // eslint-disable-next-line no-unused-vars
|
||||
|
||||
data.loadModule(request, (error, source, sourceMap, module) => {
|
||||
this.writeJson({
|
||||
type: 'result',
|
||||
id: questionId,
|
||||
@@ -221,28 +244,85 @@ class PoolWorker {
|
||||
details: error.details,
|
||||
missing: error.missing
|
||||
} : null,
|
||||
result
|
||||
result: [source, sourceMap // TODO: Serialize module?
|
||||
// module,
|
||||
]
|
||||
});
|
||||
});
|
||||
finalCallback();
|
||||
break;
|
||||
}
|
||||
|
||||
case 'resolve':
|
||||
{
|
||||
const {
|
||||
context,
|
||||
request,
|
||||
options,
|
||||
questionId
|
||||
} = message;
|
||||
const {
|
||||
data
|
||||
} = this.jobs[id];
|
||||
|
||||
if (options) {
|
||||
data.getResolve(options)(context, request, (error, result) => {
|
||||
this.writeJson({
|
||||
type: 'result',
|
||||
id: questionId,
|
||||
error: error ? {
|
||||
message: error.message,
|
||||
details: error.details,
|
||||
missing: error.missing
|
||||
} : null,
|
||||
result
|
||||
});
|
||||
});
|
||||
} else {
|
||||
data.resolve(context, request, (error, result) => {
|
||||
this.writeJson({
|
||||
type: 'result',
|
||||
id: questionId,
|
||||
error: error ? {
|
||||
message: error.message,
|
||||
details: error.details,
|
||||
missing: error.missing
|
||||
} : null,
|
||||
result
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
finalCallback();
|
||||
break;
|
||||
}
|
||||
|
||||
case 'emitWarning':
|
||||
{
|
||||
const { data } = message;
|
||||
const { data: jobData } = this.jobs[id];
|
||||
const {
|
||||
data
|
||||
} = message;
|
||||
const {
|
||||
data: jobData
|
||||
} = this.jobs[id];
|
||||
jobData.emitWarning(this.fromErrorObj(data));
|
||||
finalCallback();
|
||||
break;
|
||||
}
|
||||
|
||||
case 'emitError':
|
||||
{
|
||||
const { data } = message;
|
||||
const { data: jobData } = this.jobs[id];
|
||||
const {
|
||||
data
|
||||
} = message;
|
||||
const {
|
||||
data: jobData
|
||||
} = this.jobs[id];
|
||||
jobData.emitError(this.fromErrorObj(data));
|
||||
finalCallback();
|
||||
break;
|
||||
}
|
||||
|
||||
default:
|
||||
{
|
||||
console.error(`Unexpected worker message ${type} in WorkerPool.`);
|
||||
@@ -254,16 +334,20 @@ class PoolWorker {
|
||||
|
||||
fromErrorObj(arg) {
|
||||
let obj;
|
||||
|
||||
if (typeof arg === 'string') {
|
||||
obj = { message: arg };
|
||||
obj = {
|
||||
message: arg
|
||||
};
|
||||
} else {
|
||||
obj = arg;
|
||||
}
|
||||
return new _WorkerError2.default(obj, this.id);
|
||||
|
||||
return new _WorkerError.default(obj, this.id);
|
||||
}
|
||||
|
||||
readBuffer(length, callback) {
|
||||
(0, _readBuffer2.default)(this.readPipe, length, callback);
|
||||
(0, _readBuffer.default)(this.readPipe, length, callback);
|
||||
}
|
||||
|
||||
dispose() {
|
||||
@@ -273,6 +357,7 @@ class PoolWorker {
|
||||
this.writeEnd();
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
class WorkerPool {
|
||||
@@ -285,9 +370,8 @@ class WorkerPool {
|
||||
this.workers = new Set();
|
||||
this.activeJobs = 0;
|
||||
this.timeout = null;
|
||||
this.poolQueue = (0, _queue2.default)(this.distributeJob.bind(this), options.poolParallelJobs);
|
||||
this.poolQueue = (0, _queue.default)(this.distributeJob.bind(this), options.poolParallelJobs);
|
||||
this.terminated = false;
|
||||
|
||||
this.setupLifeCycle();
|
||||
}
|
||||
|
||||
@@ -316,6 +400,7 @@ class WorkerPool {
|
||||
clearTimeout(this.timeout);
|
||||
this.timeout = null;
|
||||
}
|
||||
|
||||
this.activeJobs += 1;
|
||||
this.poolQueue.push(data, callback);
|
||||
}
|
||||
@@ -323,15 +408,18 @@ class WorkerPool {
|
||||
distributeJob(data, callback) {
|
||||
// use worker with the fewest jobs
|
||||
let bestWorker;
|
||||
|
||||
for (const worker of this.workers) {
|
||||
if (!bestWorker || worker.activeJobs < bestWorker.activeJobs) {
|
||||
bestWorker = worker;
|
||||
}
|
||||
}
|
||||
|
||||
if (bestWorker && (bestWorker.activeJobs === 0 || this.workers.size >= this.numberOfWorkers)) {
|
||||
bestWorker.run(data, callback);
|
||||
return;
|
||||
}
|
||||
|
||||
const newWorker = this.createWorker();
|
||||
newWorker.run(data, callback);
|
||||
}
|
||||
@@ -354,6 +442,7 @@ class WorkerPool {
|
||||
|
||||
onJobDone() {
|
||||
this.activeJobs -= 1;
|
||||
|
||||
if (this.activeJobs === 0 && isFinite(this.poolTimeout)) {
|
||||
this.timeout = setTimeout(() => this.disposeWorkers(), this.poolTimeout);
|
||||
}
|
||||
@@ -369,8 +458,11 @@ class WorkerPool {
|
||||
for (const worker of this.workers) {
|
||||
worker.dispose();
|
||||
}
|
||||
|
||||
this.workers.clear();
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
exports.default = WorkerPool;
|
||||
+1
-1
@@ -1,3 +1,3 @@
|
||||
'use strict';
|
||||
"use strict";
|
||||
|
||||
module.exports = require('./index');
|
||||
+19
-12
@@ -1,24 +1,26 @@
|
||||
'use strict';
|
||||
"use strict";
|
||||
|
||||
Object.defineProperty(exports, "__esModule", {
|
||||
value: true
|
||||
});
|
||||
exports.warmup = exports.pitch = undefined;
|
||||
exports.pitch = pitch;
|
||||
exports.warmup = warmup;
|
||||
|
||||
var _loaderUtils = require('loader-utils');
|
||||
var _loaderUtils = _interopRequireDefault(require("loader-utils"));
|
||||
|
||||
var _loaderUtils2 = _interopRequireDefault(_loaderUtils);
|
||||
|
||||
var _workerPools = require('./workerPools');
|
||||
var _workerPools = require("./workerPools");
|
||||
|
||||
function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; }
|
||||
|
||||
function pitch() {
|
||||
const options = _loaderUtils2.default.getOptions(this) || {};
|
||||
const options = _loaderUtils.default.getOptions(this);
|
||||
|
||||
const workerPool = (0, _workerPools.getPool)(options);
|
||||
|
||||
if (!workerPool.isAbleToRun()) {
|
||||
return;
|
||||
}
|
||||
|
||||
const callback = this.async();
|
||||
workerPool.run({
|
||||
loaders: this.loaders.slice(this.loaderIndex + 1).map(l => {
|
||||
@@ -32,20 +34,28 @@ function pitch() {
|
||||
sourceMap: this.sourceMap,
|
||||
emitError: this.emitError,
|
||||
emitWarning: this.emitWarning,
|
||||
loadModule: this.loadModule,
|
||||
resolve: this.resolve,
|
||||
getResolve: this.getResolve,
|
||||
target: this.target,
|
||||
minimize: this.minimize,
|
||||
resourceQuery: this.resourceQuery,
|
||||
optionsContext: this.rootContext || this.options.context
|
||||
optionsContext: this.rootContext || this.options.context,
|
||||
rootContext: this.rootContext
|
||||
}, (err, r) => {
|
||||
if (r) {
|
||||
r.fileDependencies.forEach(d => this.addDependency(d));
|
||||
r.contextDependencies.forEach(d => this.addContextDependency(d));
|
||||
r.missingDependencies.forEach(d => this.addMissingDependency(d));
|
||||
r.buildDependencies.forEach(d => // Compatibility with webpack v4
|
||||
this.addBuildDependency ? this.addBuildDependency(d) : this.addDependency(d));
|
||||
}
|
||||
|
||||
if (err) {
|
||||
callback(err);
|
||||
return;
|
||||
}
|
||||
|
||||
callback(null, ...r.result);
|
||||
});
|
||||
}
|
||||
@@ -53,7 +63,4 @@ function pitch() {
|
||||
function warmup(options, requires) {
|
||||
const workerPool = (0, _workerPools.getPool)(options);
|
||||
workerPool.warmup(requires);
|
||||
}
|
||||
|
||||
exports.pitch = pitch;
|
||||
exports.warmup = warmup; // eslint-disable-line import/prefer-default-export
|
||||
} // eslint-disable-line import/prefer-default-export
|
||||
+6
-1
@@ -1,9 +1,10 @@
|
||||
'use strict';
|
||||
"use strict";
|
||||
|
||||
Object.defineProperty(exports, "__esModule", {
|
||||
value: true
|
||||
});
|
||||
exports.default = readBuffer;
|
||||
|
||||
function readBuffer(pipe, length, callback) {
|
||||
if (length === 0) {
|
||||
callback(null, Buffer.alloc(0));
|
||||
@@ -17,6 +18,7 @@ function readBuffer(pipe, length, callback) {
|
||||
const onChunk = arg => {
|
||||
let chunk = arg;
|
||||
let overflow;
|
||||
|
||||
if (chunk.length > remainingLength) {
|
||||
overflow = chunk.slice(remainingLength);
|
||||
chunk = chunk.slice(0, remainingLength);
|
||||
@@ -24,7 +26,9 @@ function readBuffer(pipe, length, callback) {
|
||||
} else {
|
||||
remainingLength -= chunk.length;
|
||||
}
|
||||
|
||||
buffers.push(chunk);
|
||||
|
||||
if (remainingLength === 0) {
|
||||
pipe.removeListener('data', onChunk);
|
||||
pipe.pause();
|
||||
@@ -40,5 +44,6 @@ function readBuffer(pipe, length, callback) {
|
||||
pipe.on('data', onChunk);
|
||||
pipe.resume();
|
||||
};
|
||||
|
||||
readChunk();
|
||||
}
|
||||
+30
@@ -0,0 +1,30 @@
|
||||
"use strict";
|
||||
|
||||
Object.defineProperty(exports, "__esModule", {
|
||||
value: true
|
||||
});
|
||||
exports.replacer = replacer;
|
||||
exports.reviver = reviver;
|
||||
|
||||
function replacer(_key, value) {
|
||||
if (value instanceof RegExp) {
|
||||
return {
|
||||
__serialized_type: 'RegExp',
|
||||
source: value.source,
|
||||
flags: value.flags
|
||||
};
|
||||
}
|
||||
|
||||
return value;
|
||||
}
|
||||
|
||||
function reviver(_key, value) {
|
||||
if (typeof value === 'object' && value !== null) {
|
||||
// eslint-disable-next-line no-underscore-dangle
|
||||
if (value.__serialized_type === 'RegExp') {
|
||||
return new RegExp(value.source, value.flags);
|
||||
}
|
||||
}
|
||||
|
||||
return value;
|
||||
}
|
||||
+157
-43
@@ -1,42 +1,41 @@
|
||||
'use strict';
|
||||
"use strict";
|
||||
|
||||
var _fs = require('fs');
|
||||
var _fs = _interopRequireDefault(require("fs"));
|
||||
|
||||
var _fs2 = _interopRequireDefault(_fs);
|
||||
var _module = _interopRequireDefault(require("module"));
|
||||
|
||||
var _module = require('module');
|
||||
var _querystring = _interopRequireDefault(require("querystring"));
|
||||
|
||||
var _module2 = _interopRequireDefault(_module);
|
||||
var _loaderRunner = _interopRequireDefault(require("loader-runner"));
|
||||
|
||||
var _loaderRunner = require('loader-runner');
|
||||
var _queue = _interopRequireDefault(require("neo-async/queue"));
|
||||
|
||||
var _loaderRunner2 = _interopRequireDefault(_loaderRunner);
|
||||
var _jsonParseBetterErrors = _interopRequireDefault(require("json-parse-better-errors"));
|
||||
|
||||
var _queue = require('neo-async/queue');
|
||||
var _schemaUtils = require("schema-utils");
|
||||
|
||||
var _queue2 = _interopRequireDefault(_queue);
|
||||
var _readBuffer = _interopRequireDefault(require("./readBuffer"));
|
||||
|
||||
var _readBuffer = require('./readBuffer');
|
||||
|
||||
var _readBuffer2 = _interopRequireDefault(_readBuffer);
|
||||
var _serializer = require("./serializer");
|
||||
|
||||
function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; }
|
||||
|
||||
const writePipe = _fs2.default.createWriteStream(null, { fd: 3 }); /* global require */
|
||||
/* eslint-disable no-console */
|
||||
const writePipe = _fs.default.createWriteStream(null, {
|
||||
fd: 3
|
||||
});
|
||||
|
||||
const readPipe = _fs2.default.createReadStream(null, { fd: 4 });
|
||||
const readPipe = _fs.default.createReadStream(null, {
|
||||
fd: 4
|
||||
});
|
||||
|
||||
writePipe.on('finish', onTerminateWrite);
|
||||
readPipe.on('end', onTerminateRead);
|
||||
writePipe.on('close', onTerminateWrite);
|
||||
readPipe.on('close', onTerminateRead);
|
||||
|
||||
readPipe.on('error', onError);
|
||||
writePipe.on('error', onError);
|
||||
|
||||
const PARALLEL_JOBS = +process.argv[2] || 20;
|
||||
|
||||
let terminated = false;
|
||||
let nextQuestionId = 0;
|
||||
const callbackMap = Object.create(null);
|
||||
@@ -108,34 +107,120 @@ function writeJson(data) {
|
||||
process.nextTick(() => {
|
||||
writePipeUncork();
|
||||
});
|
||||
|
||||
const lengthBuffer = Buffer.alloc(4);
|
||||
const messageBuffer = Buffer.from(JSON.stringify(data), 'utf-8');
|
||||
const messageBuffer = Buffer.from(JSON.stringify(data, _serializer.replacer), 'utf-8');
|
||||
lengthBuffer.writeInt32BE(messageBuffer.length, 0);
|
||||
|
||||
writePipeWrite(lengthBuffer);
|
||||
writePipeWrite(messageBuffer);
|
||||
}
|
||||
|
||||
const queue = (0, _queue2.default)(({ id, data }, taskCallback) => {
|
||||
const queue = (0, _queue.default)(({
|
||||
id,
|
||||
data
|
||||
}, taskCallback) => {
|
||||
try {
|
||||
_loaderRunner2.default.runLoaders({
|
||||
const resolveWithOptions = (context, request, callback, options) => {
|
||||
callbackMap[nextQuestionId] = callback;
|
||||
writeJson({
|
||||
type: 'resolve',
|
||||
id,
|
||||
questionId: nextQuestionId,
|
||||
context,
|
||||
request,
|
||||
options
|
||||
});
|
||||
nextQuestionId += 1;
|
||||
};
|
||||
|
||||
const buildDependencies = [];
|
||||
|
||||
_loaderRunner.default.runLoaders({
|
||||
loaders: data.loaders,
|
||||
resource: data.resource,
|
||||
readResource: _fs2.default.readFile.bind(_fs2.default),
|
||||
readResource: _fs.default.readFile.bind(_fs.default),
|
||||
context: {
|
||||
version: 2,
|
||||
resolve: (context, request, callback) => {
|
||||
callbackMap[nextQuestionId] = callback;
|
||||
fs: _fs.default,
|
||||
loadModule: (request, callback) => {
|
||||
callbackMap[nextQuestionId] = (error, result) => callback(error, ...result);
|
||||
|
||||
writeJson({
|
||||
type: 'resolve',
|
||||
type: 'loadModule',
|
||||
id,
|
||||
questionId: nextQuestionId,
|
||||
context,
|
||||
request
|
||||
});
|
||||
nextQuestionId += 1;
|
||||
},
|
||||
resolve: (context, request, callback) => {
|
||||
resolveWithOptions(context, request, callback);
|
||||
},
|
||||
// eslint-disable-next-line consistent-return
|
||||
getResolve: options => (context, request, callback) => {
|
||||
if (callback) {
|
||||
resolveWithOptions(context, request, callback, options);
|
||||
} else {
|
||||
return new Promise((resolve, reject) => {
|
||||
resolveWithOptions(context, request, (err, result) => {
|
||||
if (err) {
|
||||
reject(err);
|
||||
} else {
|
||||
resolve(result);
|
||||
}
|
||||
}, options);
|
||||
});
|
||||
}
|
||||
},
|
||||
|
||||
// Not an arrow function because it uses this
|
||||
getOptions(schema) {
|
||||
// loaders, loaderIndex will be defined by runLoaders
|
||||
const loader = this.loaders[this.loaderIndex]; // Verbatim copy from
|
||||
// https://github.com/webpack/webpack/blob/v5.31.2/lib/NormalModule.js#L471-L508
|
||||
// except eslint/prettier differences
|
||||
// -- unfortunate result of getOptions being synchronous functions.
|
||||
|
||||
let {
|
||||
options
|
||||
} = loader;
|
||||
|
||||
if (typeof options === 'string') {
|
||||
if (options.substr(0, 1) === '{' && options.substr(-1) === '}') {
|
||||
try {
|
||||
options = (0, _jsonParseBetterErrors.default)(options);
|
||||
} catch (e) {
|
||||
throw new Error(`Cannot parse string options: ${e.message}`);
|
||||
}
|
||||
} else {
|
||||
options = _querystring.default.parse(options, '&', '=', {
|
||||
maxKeys: 0
|
||||
});
|
||||
}
|
||||
} // eslint-disable-next-line no-undefined
|
||||
|
||||
|
||||
if (options === null || options === undefined) {
|
||||
options = {};
|
||||
}
|
||||
|
||||
if (schema) {
|
||||
let name = 'Loader';
|
||||
let baseDataPath = 'options';
|
||||
let match; // eslint-disable-next-line no-cond-assign
|
||||
|
||||
if (schema.title && (match = /^(.+) (.+)$/.exec(schema.title))) {
|
||||
[, name, baseDataPath] = match;
|
||||
}
|
||||
|
||||
(0, _schemaUtils.validate)(schema, options, {
|
||||
name,
|
||||
baseDataPath
|
||||
});
|
||||
}
|
||||
|
||||
return options;
|
||||
},
|
||||
|
||||
emitWarning: warning => {
|
||||
writeJson({
|
||||
type: 'emitWarning',
|
||||
@@ -151,12 +236,19 @@ const queue = (0, _queue2.default)(({ id, data }, taskCallback) => {
|
||||
});
|
||||
},
|
||||
exec: (code, filename) => {
|
||||
const module = new _module2.default(filename, undefined);
|
||||
module.paths = _module2.default._nodeModulePaths(undefined.context); // eslint-disable-line no-underscore-dangle
|
||||
const module = new _module.default(filename, void 0);
|
||||
module.paths = _module.default._nodeModulePaths((void 0).context); // eslint-disable-line no-underscore-dangle
|
||||
|
||||
module.filename = filename;
|
||||
|
||||
module._compile(code, filename); // eslint-disable-line no-underscore-dangle
|
||||
|
||||
|
||||
return module.exports;
|
||||
},
|
||||
addBuildDependency: filename => {
|
||||
buildDependencies.push(filename);
|
||||
},
|
||||
options: {
|
||||
context: data.optionsContext
|
||||
},
|
||||
@@ -165,24 +257,28 @@ const queue = (0, _queue2.default)(({ id, data }, taskCallback) => {
|
||||
sourceMap: data.sourceMap,
|
||||
target: data.target,
|
||||
minimize: data.minimize,
|
||||
resourceQuery: data.resourceQuery
|
||||
resourceQuery: data.resourceQuery,
|
||||
rootContext: data.rootContext
|
||||
}
|
||||
}, (err, lrResult) => {
|
||||
const {
|
||||
result,
|
||||
cacheable,
|
||||
fileDependencies,
|
||||
contextDependencies
|
||||
contextDependencies,
|
||||
missingDependencies
|
||||
} = lrResult;
|
||||
const buffersToSend = [];
|
||||
const convertedResult = Array.isArray(result) && result.map(item => {
|
||||
const isBuffer = Buffer.isBuffer(item);
|
||||
|
||||
if (isBuffer) {
|
||||
buffersToSend.push(item);
|
||||
return {
|
||||
buffer: true
|
||||
};
|
||||
}
|
||||
|
||||
if (typeof item === 'string') {
|
||||
const stringBuffer = Buffer.from(item, 'utf-8');
|
||||
buffersToSend.push(stringBuffer);
|
||||
@@ -191,6 +287,7 @@ const queue = (0, _queue2.default)(({ id, data }, taskCallback) => {
|
||||
string: true
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
data: item
|
||||
};
|
||||
@@ -203,7 +300,9 @@ const queue = (0, _queue2.default)(({ id, data }, taskCallback) => {
|
||||
result: convertedResult,
|
||||
cacheable,
|
||||
fileDependencies,
|
||||
contextDependencies
|
||||
contextDependencies,
|
||||
missingDependencies,
|
||||
buildDependencies
|
||||
},
|
||||
data: buffersToSend.map(buffer => buffer.length)
|
||||
});
|
||||
@@ -224,40 +323,54 @@ const queue = (0, _queue2.default)(({ id, data }, taskCallback) => {
|
||||
|
||||
function dispose() {
|
||||
terminate();
|
||||
|
||||
queue.kill();
|
||||
process.exit(0);
|
||||
}
|
||||
|
||||
function onMessage(message) {
|
||||
try {
|
||||
const { type, id } = message;
|
||||
const {
|
||||
type,
|
||||
id
|
||||
} = message;
|
||||
|
||||
switch (type) {
|
||||
case 'job':
|
||||
{
|
||||
queue.push(message);
|
||||
break;
|
||||
}
|
||||
|
||||
case 'result':
|
||||
{
|
||||
const { error, result } = message;
|
||||
const {
|
||||
error,
|
||||
result
|
||||
} = message;
|
||||
const callback = callbackMap[id];
|
||||
|
||||
if (callback) {
|
||||
const nativeError = toNativeError(error);
|
||||
callback(nativeError, result);
|
||||
} else {
|
||||
console.error(`Worker got unexpected result id ${id}`);
|
||||
}
|
||||
|
||||
delete callbackMap[id];
|
||||
break;
|
||||
}
|
||||
|
||||
case 'warmup':
|
||||
{
|
||||
const { requires } = message;
|
||||
// load modules into process
|
||||
const {
|
||||
requires
|
||||
} = message; // load modules into process
|
||||
|
||||
requires.forEach(r => require(r)); // eslint-disable-line import/no-dynamic-require, global-require
|
||||
|
||||
break;
|
||||
}
|
||||
|
||||
default:
|
||||
{
|
||||
console.error(`Worker got unexpected job type ${type}`);
|
||||
@@ -270,7 +383,7 @@ function onMessage(message) {
|
||||
}
|
||||
|
||||
function readNextMessage() {
|
||||
(0, _readBuffer2.default)(readPipe, 4, (lengthReadError, lengthBuffer) => {
|
||||
(0, _readBuffer.default)(readPipe, 4, (lengthReadError, lengthBuffer) => {
|
||||
if (lengthReadError) {
|
||||
console.error(`Failed to communicate with main process (read length) ${lengthReadError}`);
|
||||
return;
|
||||
@@ -283,7 +396,8 @@ function readNextMessage() {
|
||||
dispose();
|
||||
return;
|
||||
}
|
||||
(0, _readBuffer2.default)(readPipe, length, (messageError, messageBuffer) => {
|
||||
|
||||
(0, _readBuffer.default)(readPipe, length, (messageError, messageBuffer) => {
|
||||
if (terminated) {
|
||||
return;
|
||||
}
|
||||
@@ -292,14 +406,14 @@ function readNextMessage() {
|
||||
console.error(`Failed to communicate with main process (read message) ${messageError}`);
|
||||
return;
|
||||
}
|
||||
const messageString = messageBuffer.toString('utf-8');
|
||||
const message = JSON.parse(messageString);
|
||||
|
||||
const messageString = messageBuffer.toString('utf-8');
|
||||
const message = JSON.parse(messageString, _serializer.reviver);
|
||||
onMessage(message);
|
||||
setImmediate(() => readNextMessage());
|
||||
});
|
||||
});
|
||||
}
|
||||
} // start reading messages from main process
|
||||
|
||||
|
||||
// start reading messages from main process
|
||||
readNextMessage();
|
||||
+10
-16
@@ -1,17 +1,13 @@
|
||||
'use strict';
|
||||
"use strict";
|
||||
|
||||
Object.defineProperty(exports, "__esModule", {
|
||||
value: true
|
||||
});
|
||||
exports.getPool = undefined;
|
||||
exports.getPool = getPool;
|
||||
|
||||
var _os = require('os');
|
||||
var _os = _interopRequireDefault(require("os"));
|
||||
|
||||
var _os2 = _interopRequireDefault(_os);
|
||||
|
||||
var _WorkerPool = require('./WorkerPool');
|
||||
|
||||
var _WorkerPool2 = _interopRequireDefault(_WorkerPool);
|
||||
var _WorkerPool = _interopRequireDefault(require("./WorkerPool"));
|
||||
|
||||
function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; }
|
||||
|
||||
@@ -21,8 +17,9 @@ function calculateNumberOfWorkers() {
|
||||
// There are situations when this call will return undefined so
|
||||
// we are fallback here to 1.
|
||||
// More info on: https://github.com/nodejs/node/issues/19022
|
||||
const cpus = _os2.default.cpus() || { length: 1 };
|
||||
|
||||
const cpus = _os.default.cpus() || {
|
||||
length: 1
|
||||
};
|
||||
return Math.max(1, cpus.length - 1);
|
||||
}
|
||||
|
||||
@@ -37,9 +34,6 @@ function getPool(options) {
|
||||
poolRespawn: options.poolRespawn || false
|
||||
};
|
||||
const tpKey = JSON.stringify(workerPoolOptions);
|
||||
workerPools[tpKey] = workerPools[tpKey] || new _WorkerPool2.default(workerPoolOptions);
|
||||
const workerPool = workerPools[tpKey];
|
||||
return workerPool;
|
||||
}
|
||||
|
||||
exports.getPool = getPool; // eslint-disable-line import/prefer-default-export
|
||||
workerPools[tpKey] = workerPools[tpKey] || new _WorkerPool.default(workerPoolOptions);
|
||||
return workerPools[tpKey];
|
||||
} // eslint-disable-line import/prefer-default-export
|
||||
Reference in New Issue
Block a user