Child Processes & Worker Threads

Difficulty: Advanced

Question

When would you use child processes vs worker threads in Node.js? Explain the differences and use cases.

Answer

Child processes and worker threads are both used to handle CPU-intensive work without blocking the event loop, but they work fundamentally differently.

Child processes (child_process module) spawn entirely new OS processes with their own memory space, V8 instance, and event loop. Communication happens via IPC (Inter-Process Communication) or stdin/stdout. They can run any executable, not just JavaScript.

Worker threads (worker_threads module) create new threads within the same process. They share memory (via SharedArrayBuffer and Atomics) and are lighter than child processes. However, they can only run JavaScript/WASM.

Use child processes when: running external commands (ffmpeg, ImageMagick), isolating untrusted code, or needing full process isolation. Use worker threads when: performing CPU-intensive JavaScript computation (crypto, parsing, compression) and you need shared memory or lower overhead.

Code examples

Child Process Methods

const { exec, execFile, spawn, fork } = require('child_process');

// exec: run shell command, buffer output (small output)
exec('ls -la /tmp', (error, stdout, stderr) => {
  if (error) console.error('Error:', error.message);
  console.log('Output:', stdout);
});

// execFile: run executable directly (no shell, safer)
execFile('node', ['--version'], (error, stdout) => {
  console.log('Node:', stdout.trim());
});

// spawn: stream output (large output, long-running)
const child = spawn('ffmpeg', [
  '-i', 'input.mp4',
  '-codec:v', 'libx264',
  'output.mp4',
]);

child.stdout.on('data', (data) => console.log('stdout:', data.toString()));
child.stderr.on('data', (data) => console.log('progress:', data.toString()));
child.on('close', (code) => console.log('Exit code:', code));

// fork: spawn Node.js process with IPC channel
const worker = fork('./heavy-computation.js');
worker.send({ numbers: [1, 2, 3, 4, 5] });
worker.on('message', (result) => {
  console.log('Result:', result);
});

// heavy-computation.js
// process.on('message', (data) => {
//   const result = heavyWork(data.numbers);
//   process.send(result);
// });

exec buffers output (good for small results). spawn streams output (good for large data). fork creates Node.js child with built-in IPC.

Worker Threads for CPU Work

// main.js
const { Worker, isMainThread, parentPort, workerData } = require('worker_threads');

if (isMainThread) {
  // Main thread - create workers
  function runInWorker(task) {
    return new Promise((resolve, reject) => {
      const worker = new Worker(__filename, { workerData: task });
      worker.on('message', resolve);
      worker.on('error', reject);
      worker.on('exit', (code) => {
        if (code !== 0) reject(new Error(`Worker exited with code ${code}`));
      });
    });
  }

  // Parallel computation
  async function main() {
    const results = await Promise.all([
      runInWorker({ type: 'hash', data: 'password1' }),
      runInWorker({ type: 'hash', data: 'password2' }),
      runInWorker({ type: 'hash', data: 'password3' }),
    ]);
    console.log('All hashes:', results);
  }
  main();

} else {
  // Worker thread
  const crypto = require('crypto');
  const { type, data } = workerData;

  if (type === 'hash') {
    const hash = crypto.pbkdf2Sync(data, 'salt', 100000, 64, 'sha512');
    parentPort.postMessage(hash.toString('hex'));
  }
}

Worker threads run in parallel. Using __filename lets the same file act as both main thread and worker. Promise.all runs all workers concurrently.

Worker Thread Pool Pattern

const { Worker } = require('worker_threads');
const os = require('os');

class WorkerPool {
  constructor(workerScript, poolSize = os.cpus().length) {
    this.workers = [];
    this.queue = [];

    for (let i = 0; i < poolSize; i++) {
      this.workers.push({ busy: false, worker: this.createWorker(workerScript) });
    }
  }

  createWorker(script) {
    const worker = new Worker(script);
    worker.on('message', (result) => {
      const entry = this.workers.find(w => w.worker === worker);
      entry.busy = false;
      entry.resolve(result);
      this.processQueue();
    });
    return worker;
  }

  execute(data) {
    return new Promise((resolve, reject) => {
      const freeWorker = this.workers.find(w => !w.busy);
      if (freeWorker) {
        freeWorker.busy = true;
        freeWorker.resolve = resolve;
        freeWorker.worker.postMessage(data);
      } else {
        this.queue.push({ data, resolve, reject });
      }
    });
  }

  processQueue() {
    if (this.queue.length === 0) return;
    const freeWorker = this.workers.find(w => !w.busy);
    if (!freeWorker) return;
    const { data, resolve } = this.queue.shift();
    freeWorker.busy = true;
    freeWorker.resolve = resolve;
    freeWorker.worker.postMessage(data);
  }
}

// Usage
const pool = new WorkerPool('./compute-worker.js', 4);
const result = await pool.execute({ input: 'data' });

A worker pool reuses threads instead of creating new ones for each task. This avoids the overhead of thread creation and limits concurrency.

Key points

Concepts covered

child_process, Worker Threads, exec, spawn, fork