'use strict';
// Archive serialization: `createZipArchive()`/`createZipArchiveSync()` and
// the `zipFiles()` on-disk variant, the `generateZipArchive()` generator they
// build on (auto-switching to Zip64 as offsets/counts overflow), and the
// shared archive-option normalizer.
const {
ArrayPrototypePush,
JSONStringify,
NumberMAX_SAFE_INTEGER,
StringPrototypeEndsWith,
SymbolAsyncDispose,
SymbolAsyncIterator,
SymbolDispose,
SymbolIterator,
} = primordials;
const {
codes: {
ERR_ZIP_ARCHIVE_TOO_LARGE,
ERR_ZIP_UNSUPPORTED_FEATURE,
},
} = require('internal/errors');
const {
validateBoolean,
validateInteger,
validateObject,
validateString,
} = require('internal/validators');
const { isUint8Array } = require('internal/util/types');
const { Buffer } = require('buffer');
const { Readable } = require('stream');
const fs = require('fs');
const {
EMPTY_BUFFER,
SENTINEL16,
kFinalize,
} = require('internal/zip/constants');
const {
buildArchiveTrailer,
} = require('internal/zip/header-builders');
const {
fsStatAsync,
fsLstatAsync,
fsReadlinkAsync,
} = require('internal/zip/fs-util');
const { ZipEntry } = require('internal/zip/entry');
/**
* `createZipArchive()`/`createZipArchiveSync()` (and the `ZipBuffer`
* `toBuffer()`/`toBufferSync()` methods that forward to them) take a single
* optional `options` argument that doubles as a plain archive comment: a
* string is shorthand for `{ comment: options }`.
* @param {string | { comment?: string, baseOffset?: number }} [options]
* @returns {{ comment: string | undefined, baseOffset: number }}
*/
function normalizeArchiveOptions(options) {
if (options === undefined) return { comment: undefined, baseOffset: 0 };
if (typeof options === 'string') return { comment: options, baseOffset: 0 };
validateObject(options, 'options');
const { comment, baseOffset = 0 } = options;
// A Buffer comment is an internal convenience (used when round-tripping an
// existing archive) that preserves the original bytes without forcing them
// through a decode/re-encode cycle that would corrupt non-UTF-8 comments.
if (comment !== undefined && !isUint8Array(comment)) {
validateString(comment, 'options.comment');
}
validateInteger(baseOffset, 'options.baseOffset', 0, NumberMAX_SAFE_INTEGER);
return { comment, baseOffset };
}
/**
* Serializes `entries` (a (async) iterable of `ZipEntry`) into a `Readable`
* stream of archive byte chunks, automatically switching to Zip64 structures
* once the entry count or any offset/size exceeds the classic 32-/16-bit
* limits.
*
* `options.baseOffset` shifts every local/central header offset the archive
* records by that many bytes, so the emitted bytes are self-describing even
* when something else is written before them - for example, appending the
* archive after `baseOffset` bytes already written to the same file, rather
* than at its start.
* @param {Iterable<ZipEntry> | AsyncIterable<ZipEntry>} entries
* @param {string | { comment?: string, baseOffset?: number }} [options]
* @returns {Readable}
*/
function createZipArchive(entries, options) {
return Readable.from(generateZipArchive(entries, options), { objectMode: false });
}
/**
* Creates an archive from files on disk. `files` is an iterable of
* `[sourcePath, entryName]` pairs - an array, a `Map`, the result of
* `Object.entries()`, a generator, and so on. Each entry captures the file's
* Unix mode and modification time; a directory becomes a directory entry and a
* regular file's contents are streamed in without being buffered in memory.
*
* With `options.followSymlinks` (default `true`) a symbolic link is resolved
* and archived as the file it points to; with it `false` the link itself is
* stored as a symlink entry whose content is the target path.
* @param {Iterable<[string, string]>} files
* @param {string | { followSymlinks?: boolean, comment?: string, baseOffset?: number }} [options]
* @returns {import('stream').Readable}
*/
function zipFiles(files, options) {
const followSymlinks = options?.followSymlinks ?? true;
validateBoolean(followSymlinks, 'options.followSymlinks');
return createZipArchive(fileEntries(files, followSymlinks), options);
}
// Turn each `[sourcePath, entryName]` pair into a `ZipEntry`, stat-ing the
// source to capture its mode/mtime and picking the symlink/directory/file
// entry shape; the file-backed variant streams contents rather than buffering.
async function* fileEntries(files, followSymlinks) {
for await (const pair of files) {
const sourcePath = pair[0];
const name = pair[1];
validateString(sourcePath, 'sourcePath');
validateString(name, 'name');
// Following links resolves through to the target (stat); otherwise the
// link itself is inspected (lstat) and stored as a symlink entry.
const stats = followSymlinks ? await fsStatAsync(sourcePath) : await fsLstatAsync(sourcePath);
const options = { __proto__: null, mode: stats.mode & 0o7777, modified: stats.mtime };
if (stats.isSymbolicLink()) {
yield ZipEntry.createSymlink(name, await fsReadlinkAsync(sourcePath), options);
} else if (stats.isDirectory()) {
const dirName = StringPrototypeEndsWith(name, '/') ? name : `${name}/`;
yield await ZipEntry.create(dirName, EMPTY_BUFFER, options);
} else {
// Open the file ourselves and stream from that exact descriptor instead
// of re-resolving the path with createReadStream(). Re-opening by path
// would let a symlink swapped in after the classification above redirect
// the read - a TOCTOU that defeats followSymlinks:false. O_NOFOLLOW makes
// a final-component symlink fail the open outright when not following;
// O_NONBLOCK keeps the open itself from blocking on a FIFO or a slow
// device (it has no effect on a regular file's reads), so the fstat can
// then reject anything that is not a regular file - a FIFO, device, or
// socket, which as a stream source could block or emit unbounded data.
// Metadata comes from that same fstat so it describes the inode we read.
const flags = (followSymlinks ?
fs.constants.O_RDONLY :
fs.constants.O_RDONLY | (fs.constants.O_NOFOLLOW || 0)) |
(fs.constants.O_NONBLOCK || 0);
const handle = await fs.promises.open(sourcePath, flags);
let ownsHandle = true;
try {
const stat = await handle.stat();
if (!stat.isFile()) {
throw new ERR_ZIP_UNSUPPORTED_FEATURE(
`cannot archive ${JSONStringify(sourcePath)}: not a regular file`);
}
const fileOptions = {
__proto__: null,
mode: stat.mode & 0o7777,
modified: stat.mtime,
};
// The stream adopts the handle and closes it when it ends or is
// destroyed (autoClose); if the consumer stops before taking the
// entry, the finally destroys the stream, releasing the descriptor.
const stream = fs.createReadStream(undefined, { fd: handle });
ownsHandle = false;
let handedOff = false;
try {
yield ZipEntry.createStream(name, stream, fileOptions);
handedOff = true;
} finally {
if (!handedOff) stream.destroy();
}
} finally {
if (ownsHandle) await handle.close();
}
}
}
}
// Encode the archive comment (a string, or raw bytes when round-tripping a
// non-UTF-8 comment) and enforce the 16-bit length field (sec. 4.3.16).
function normalizeCommentBuffer(comment) {
if (comment === undefined) return EMPTY_BUFFER;
const buffer = isUint8Array(comment) ? comment : Buffer.from(comment, 'utf8');
if (buffer.length > SENTINEL16) {
throw new ERR_ZIP_ARCHIVE_TOO_LARGE(
'the archive comment must not exceed 65535 bytes when encoded as UTF-8');
}
return buffer;
}
// Dispose the entries still queued behind an interrupted serialization. A
// ZipEntry handed to the archive is owned by it, so when the consumer destroys
// the output stream early - which runs `generateZipArchive`'s `finally` - the
// entries it never reached must be released too (a streaming entry may hold a
// file read stream). The entry currently being serialized is torn down by the
// `for await ... of entry` loop's own return propagation, so only the queue
// behind it is handled here.
//
// A materialized source (an array, `Map`, `Object.entries()`) has already
// created every entry, so each remaining one is pulled and disposed. A lazy
// async source is instead `return()`ed: forcing it to produce every remaining
// entry could open thousands of descriptors just to close them, so it is
// signalled to stop and release its own in-flight resource (see
// `fileEntries()`).
async function disposeQueuedEntries(iterator, isAsync) {
if (isAsync) {
if (typeof iterator.return === 'function') await iterator.return();
return;
}
for (let next = iterator.next(); !next.done; next = iterator.next()) {
const entry = next.value;
if (typeof entry?.[SymbolAsyncDispose] === 'function') await entry[SymbolAsyncDispose]();
}
}
// Core serializer backing `createZipArchive()`: emits each entry's local header
// + data, then the central directory, then the end-of-central-directory record
// (APPNOTE sec. 4.3.6 archive layout), promoting to Zip64 end records
// (sec. 4.3.14/4.3.15) once any count/offset/size overflows its classic field.
//
// The entries are owned by this generator: it drives the iterator by hand
// (rather than `for await`) so that if the consumer destroys the returned
// stream partway - `Readable.from()` then calls this generator's `return()` -
// the `finally` can dispose every entry that was never fully serialized. A
// synchronous source is stepped without `await` so a pulled-but-not-yet-used
// entry cannot be stranded by a return injected at the `await`.
async function* generateZipArchive(entries, options) {
const { comment, baseOffset } = normalizeArchiveOptions(options);
const commentBuffer = normalizeCommentBuffer(comment);
const centralHeaders = [];
let pos = baseOffset;
const isAsync = entries[SymbolAsyncIterator] !== undefined;
const iterator = isAsync ? entries[SymbolAsyncIterator]() : entries[SymbolIterator]();
let completed = false;
try {
while (true) {
const next = isAsync ? await iterator.next() : iterator.next();
if (next.done) break;
const entry = next.value;
const start = pos;
for await (const chunk of entry) {
yield chunk;
pos += chunk.length;
}
ArrayPrototypePush(centralHeaders, entry[kFinalize](start));
}
const centralDirectoryOffset = pos;
for (let i = 0; i < centralHeaders.length; i++) {
const chunk = centralHeaders[i];
yield chunk;
pos += chunk.length;
}
const centralDirectorySize = pos - centralDirectoryOffset;
const count = centralHeaders.length;
const trailer = buildArchiveTrailer(count, centralDirectorySize, centralDirectoryOffset, commentBuffer);
for (let i = 0; i < trailer.length; i++) yield trailer[i];
completed = true;
} finally {
if (!completed) await disposeQueuedEntries(iterator, isAsync);
}
}
/**
* The synchronous counterpart of `createZipArchive()`. `entries` must be a
* plain (synchronous) `Iterable` of entries that don't require an
* asynchronous serialization pass - a streaming entry created with
* `ZipEntry.createStream()` throws when its turn to serialize comes up, the
* same as calling `entry[Symbol.iterator]()` on one directly. Blocks the
* event loop and further JavaScript execution until the whole archive
* (including any deflate passes) has been produced; see
* `zipEntry.contentSync()`.
* @param {Iterable<ZipEntry>} entries
* @param {string | { comment?: string, baseOffset?: number }} [options]
* @yields {Buffer}
*/
function* createZipArchiveSync(entries, options) {
const { comment, baseOffset } = normalizeArchiveOptions(options);
const commentBuffer = normalizeCommentBuffer(comment);
const centralHeaders = [];
let pos = baseOffset;
const iterator = entries[SymbolIterator]();
let completed = false;
// As in generateZipArchive(), the entries are owned here: a streaming entry
// throws when serialized synchronously, so on that throw (or an early
// return of this generator) dispose the entry that failed and every entry
// still queued behind it, releasing any sources they hold.
let current = null;
try {
for (let next = iterator.next(); !next.done; next = iterator.next()) {
current = next.value;
const start = pos;
for (const chunk of current) {
yield chunk;
pos += chunk.length;
}
ArrayPrototypePush(centralHeaders, current[kFinalize](start));
current = null;
}
const centralDirectoryOffset = pos;
for (let i = 0; i < centralHeaders.length; i++) {
const chunk = centralHeaders[i];
yield chunk;
pos += chunk.length;
}
const centralDirectorySize = pos - centralDirectoryOffset;
const count = centralHeaders.length;
const trailer = buildArchiveTrailer(count, centralDirectorySize, centralDirectoryOffset, commentBuffer);
for (let i = 0; i < trailer.length; i++) yield trailer[i];
completed = true;
} finally {
if (!completed) {
if (typeof current?.[SymbolDispose] === 'function') current[SymbolDispose]();
for (let next = iterator.next(); !next.done; next = iterator.next()) {
const entry = next.value;
if (typeof entry?.[SymbolDispose] === 'function') entry[SymbolDispose]();
}
}
}
}
module.exports = {
normalizeArchiveOptions,
createZipArchive,
createZipArchiveSync,
zipFiles,
};