mirror of
https://github.com/nodejs/node.git
synced 2025-04-28 05:25:19 +00:00
zlib: add zstd support
Some checks are pending
Coverage Linux (without intl) / coverage-linux-without-intl (push) Waiting to run
Coverage Linux / coverage-linux (push) Waiting to run
Coverage Windows / coverage-windows (push) Waiting to run
Test and upload documentation to artifacts / build-docs (push) Waiting to run
Linters / lint-addon-docs (push) Waiting to run
Linters / lint-cpp (push) Waiting to run
Linters / format-cpp (push) Waiting to run
Linters / lint-js-and-md (push) Waiting to run
Linters / lint-py (push) Waiting to run
Linters / lint-yaml (push) Waiting to run
Linters / lint-sh (push) Waiting to run
Linters / lint-codeowners (push) Waiting to run
Linters / lint-pr-url (push) Waiting to run
Linters / lint-readme (push) Waiting to run
Notify on Push / Notify on Force Push on `main` (push) Waiting to run
Notify on Push / Notify on Push on `main` that lacks metadata (push) Waiting to run
Scorecard supply-chain security / Scorecard analysis (push) Waiting to run
Some checks are pending
Coverage Linux (without intl) / coverage-linux-without-intl (push) Waiting to run
Coverage Linux / coverage-linux (push) Waiting to run
Coverage Windows / coverage-windows (push) Waiting to run
Test and upload documentation to artifacts / build-docs (push) Waiting to run
Linters / lint-addon-docs (push) Waiting to run
Linters / lint-cpp (push) Waiting to run
Linters / format-cpp (push) Waiting to run
Linters / lint-js-and-md (push) Waiting to run
Linters / lint-py (push) Waiting to run
Linters / lint-yaml (push) Waiting to run
Linters / lint-sh (push) Waiting to run
Linters / lint-codeowners (push) Waiting to run
Linters / lint-pr-url (push) Waiting to run
Linters / lint-readme (push) Waiting to run
Notify on Push / Notify on Force Push on `main` (push) Waiting to run
Notify on Push / Notify on Push on `main` that lacks metadata (push) Waiting to run
Scorecard supply-chain security / Scorecard analysis (push) Waiting to run
Fixes: https://github.com/nodejs/node/issues/48412 PR-URL: https://github.com/nodejs/node/pull/52100 Reviewed-By: Yagiz Nizipli <yagiz@nizipli.com>
This commit is contained in:
parent
f5353100be
commit
bf12d72faa
@ -5,7 +5,7 @@ const zlib = require('zlib');
|
||||
const bench = common.createBenchmark(main, {
|
||||
type: [
|
||||
'Deflate', 'DeflateRaw', 'Inflate', 'InflateRaw', 'Gzip', 'Gunzip', 'Unzip',
|
||||
'BrotliCompress', 'BrotliDecompress',
|
||||
'BrotliCompress', 'BrotliDecompress', 'ZstdCompress', 'ZstdDecompress',
|
||||
],
|
||||
options: ['true', 'false'],
|
||||
n: [5e5],
|
||||
|
@ -7,7 +7,7 @@ const bench = common.createBenchmark(main, {
|
||||
inputLen: [1024],
|
||||
duration: [5],
|
||||
type: ['string', 'buffer'],
|
||||
algorithm: ['gzip', 'brotli'],
|
||||
algorithm: ['gzip', 'brotli', 'zstd'],
|
||||
}, {
|
||||
test: {
|
||||
inputLen: 1024,
|
||||
@ -15,14 +15,19 @@ const bench = common.createBenchmark(main, {
|
||||
},
|
||||
});
|
||||
|
||||
const algorithms = {
|
||||
'gzip': [zlib.createGzip, zlib.createGunzip],
|
||||
'brotli': [zlib.createBrotliCompress, zlib.createBrotliDecompress],
|
||||
'zstd': [zlib.createZstdCompress, zlib.createZstdDecompress],
|
||||
};
|
||||
|
||||
function main({ inputLen, duration, type, algorithm }) {
|
||||
const buffer = Buffer.alloc(inputLen, fs.readFileSync(__filename));
|
||||
const chunk = type === 'buffer' ? buffer : buffer.toString('utf8');
|
||||
|
||||
const input = algorithm === 'gzip' ?
|
||||
zlib.createGzip() : zlib.createBrotliCompress();
|
||||
const output = algorithm === 'gzip' ?
|
||||
zlib.createGunzip() : zlib.createBrotliDecompress();
|
||||
const [createCompress, createUncompress] = algorithms[algorithm];
|
||||
const input = createCompress();
|
||||
const output = createUncompress();
|
||||
|
||||
let readFromOutput = 0;
|
||||
input.pipe(output);
|
||||
|
@ -3334,6 +3334,12 @@ The requested functionality is not supported in worker threads.
|
||||
|
||||
Creation of a [`zlib`][] object failed due to incorrect configuration.
|
||||
|
||||
<a id="ERR_ZSTD_INVALID_PARAM"></a>
|
||||
|
||||
### `ERR_ZSTD_INVALID_PARAM`
|
||||
|
||||
An invalid parameter key was passed during construction of a Zstd stream.
|
||||
|
||||
<a id="HPE_CHUNK_EXTENSIONS_OVERFLOW"></a>
|
||||
|
||||
### `HPE_CHUNK_EXTENSIONS_OVERFLOW`
|
||||
|
179
doc/api/zlib.md
179
doc/api/zlib.md
@ -7,7 +7,7 @@
|
||||
<!-- source_link=lib/zlib.js -->
|
||||
|
||||
The `node:zlib` module provides compression functionality implemented using
|
||||
Gzip, Deflate/Inflate, and Brotli.
|
||||
Gzip, Deflate/Inflate, Brotli, and Zstd.
|
||||
|
||||
To access it:
|
||||
|
||||
@ -220,8 +220,8 @@ operations be cached to avoid duplication of effort.
|
||||
|
||||
## Compressing HTTP requests and responses
|
||||
|
||||
The `node:zlib` module can be used to implement support for the `gzip`, `deflate`
|
||||
and `br` content-encoding mechanisms defined by
|
||||
The `node:zlib` module can be used to implement support for the `gzip`, `deflate`,
|
||||
`br`, and `zstd` content-encoding mechanisms defined by
|
||||
[HTTP](https://tools.ietf.org/html/rfc7230#section-4.2).
|
||||
|
||||
The HTTP [`Accept-Encoding`][] header is used within an HTTP request to identify
|
||||
@ -284,7 +284,7 @@ const { pipeline } = require('node:stream');
|
||||
const request = http.get({ host: 'example.com',
|
||||
path: '/',
|
||||
port: 80,
|
||||
headers: { 'Accept-Encoding': 'br,gzip,deflate' } });
|
||||
headers: { 'Accept-Encoding': 'br,gzip,deflate,zstd' } });
|
||||
request.on('response', (response) => {
|
||||
const output = fs.createWriteStream('example.com_index.html');
|
||||
|
||||
@ -306,6 +306,9 @@ request.on('response', (response) => {
|
||||
case 'deflate':
|
||||
pipeline(response, zlib.createInflate(), output, onError);
|
||||
break;
|
||||
case 'zstd':
|
||||
pipeline(response, zlib.createZstdDecompress(), output, onError);
|
||||
break;
|
||||
default:
|
||||
pipeline(response, output, onError);
|
||||
break;
|
||||
@ -396,6 +399,9 @@ http.createServer((request, response) => {
|
||||
} else if (/\bbr\b/.test(acceptEncoding)) {
|
||||
response.writeHead(200, { 'Content-Encoding': 'br' });
|
||||
pipeline(raw, zlib.createBrotliCompress(), response, onError);
|
||||
} else if (/\bzstd\b/.test(acceptEncoding)) {
|
||||
response.writeHead(200, { 'Content-Encoding': 'zstd' });
|
||||
pipeline(raw, zlib.createZstdCompress(), response, onError);
|
||||
} else {
|
||||
response.writeHead(200, {});
|
||||
pipeline(raw, response, onError);
|
||||
@ -416,6 +422,7 @@ const buffer = Buffer.from('eJzT0yMA', 'base64');
|
||||
zlib.unzip(
|
||||
buffer,
|
||||
// For Brotli, the equivalent is zlib.constants.BROTLI_OPERATION_FLUSH.
|
||||
// For Zstd, the equivalent is zlib.constants.ZSTD_e_flush.
|
||||
{ finishFlush: zlib.constants.Z_SYNC_FLUSH },
|
||||
(err, buffer) => {
|
||||
if (err) {
|
||||
@ -487,6 +494,16 @@ these options have different ranges than the zlib ones:
|
||||
|
||||
See [below][Brotli parameters] for more details on Brotli-specific options.
|
||||
|
||||
### For Zstd-based streams
|
||||
|
||||
There are equivalents to the zlib options for Zstd-based streams, although
|
||||
these options have different ranges than the zlib ones:
|
||||
|
||||
* zlib's `level` option matches Zstd's `ZSTD_c_compressionLevel` option.
|
||||
* zlib's `windowBits` option matches Zstd's `ZSTD_c_windowLog` option.
|
||||
|
||||
See [below][Zstd parameters] for more details on Zstd-specific options.
|
||||
|
||||
## Flushing
|
||||
|
||||
Calling [`.flush()`][] on a compression stream will make `zlib` return as much
|
||||
@ -701,6 +718,50 @@ These advanced options are available for controlling decompression:
|
||||
* Boolean flag enabling “Large Window Brotli” mode (not compatible with the
|
||||
Brotli format as standardized in [RFC 7932][]).
|
||||
|
||||
### Zstd constants
|
||||
|
||||
<!-- YAML
|
||||
added: REPLACEME
|
||||
-->
|
||||
|
||||
There are several options and other constants available for Zstd-based
|
||||
streams:
|
||||
|
||||
#### Flush operations
|
||||
|
||||
The following values are valid flush operations for Zstd-based streams:
|
||||
|
||||
* `zlib.constants.ZSTD_e_continue` (default for all operations)
|
||||
* `zlib.constants.ZSTD_e_flush` (default when calling `.flush()`)
|
||||
* `zlib.constants.ZSTD_e_end` (default for the last chunk)
|
||||
|
||||
#### Compressor options
|
||||
|
||||
There are several options that can be set on Zstd encoders, affecting
|
||||
compression efficiency and speed. Both the keys and the values can be accessed
|
||||
as properties of the `zlib.constants` object.
|
||||
|
||||
The most important options are:
|
||||
|
||||
* `ZSTD_c_compressionLevel`
|
||||
* Set compression parameters according to pre-defined cLevel table. Default
|
||||
level is ZSTD\_CLEVEL\_DEFAULT==3.
|
||||
|
||||
#### Pledged Source Size
|
||||
|
||||
It's possible to specify the expected total size of the uncompressed input via
|
||||
`opts.pledgedSrcSize`. If the size doesn't match at the end of the input,
|
||||
compression will fail with the code `ZSTD_error_srcSize_wrong`.
|
||||
|
||||
#### Decompressor options
|
||||
|
||||
These advanced options are available for controlling decompression:
|
||||
|
||||
* `ZSTD_d_windowLogMax`
|
||||
* Select a size limit (in power of 2) beyond which the streaming API will
|
||||
refuse to allocate memory buffer in order to protect the host from
|
||||
unreasonable memory requirements.
|
||||
|
||||
## Class: `Options`
|
||||
|
||||
<!-- YAML
|
||||
@ -962,6 +1023,51 @@ added: v0.7.0
|
||||
Reset the compressor/decompressor to factory defaults. Only applicable to
|
||||
the inflate and deflate algorithms.
|
||||
|
||||
## Class: `ZstdOptions`
|
||||
|
||||
<!-- YAML
|
||||
added: REPLACEME
|
||||
-->
|
||||
|
||||
<!--type=misc-->
|
||||
|
||||
Each Zstd-based class takes an `options` object. All options are optional.
|
||||
|
||||
* `flush` {integer} **Default:** `zlib.constants.ZSTD_e_continue`
|
||||
* `finishFlush` {integer} **Default:** `zlib.constants.ZSTD_e_end`
|
||||
* `chunkSize` {integer} **Default:** `16 * 1024`
|
||||
* `params` {Object} Key-value object containing indexed [Zstd parameters][].
|
||||
* `maxOutputLength` {integer} Limits output size when using
|
||||
[convenience methods][]. **Default:** [`buffer.kMaxLength`][]
|
||||
|
||||
For example:
|
||||
|
||||
```js
|
||||
const stream = zlib.createZstdCompress({
|
||||
chunkSize: 32 * 1024,
|
||||
params: {
|
||||
[zlib.constants.ZSTD_c_compressionLevel]: 10,
|
||||
[zlib.constants.ZSTD_c_checksumFlag]: 1,
|
||||
},
|
||||
});
|
||||
```
|
||||
|
||||
## Class: `zlib.ZstdCompress`
|
||||
|
||||
<!-- YAML
|
||||
added: REPLACEME
|
||||
-->
|
||||
|
||||
Compress data using the Zstd algorithm.
|
||||
|
||||
## Class: `zlib.ZstdDecompress`
|
||||
|
||||
<!-- YAML
|
||||
added: REPLACEME
|
||||
-->
|
||||
|
||||
Decompress data using the Zstd algorithm.
|
||||
|
||||
## `zlib.constants`
|
||||
|
||||
<!-- YAML
|
||||
@ -1135,6 +1241,26 @@ added: v0.5.8
|
||||
|
||||
Creates and returns a new [`Unzip`][] object.
|
||||
|
||||
## `zlib.createZstdCompress([options])`
|
||||
|
||||
<!-- YAML
|
||||
added: REPLACEME
|
||||
-->
|
||||
|
||||
* `options` {zstd options}
|
||||
|
||||
Creates and returns a new [`ZstdCompress`][] object.
|
||||
|
||||
## `zlib.createZstdDecompress([options])`
|
||||
|
||||
<!-- YAML
|
||||
added: REPLACEME
|
||||
-->
|
||||
|
||||
* `options` {zstd options}
|
||||
|
||||
Creates and returns a new [`ZstdDecompress`][] object.
|
||||
|
||||
## Convenience methods
|
||||
|
||||
<!--type=misc-->
|
||||
@ -1481,11 +1607,54 @@ changes:
|
||||
|
||||
Decompress a chunk of data with [`Unzip`][].
|
||||
|
||||
### `zlib.zstdCompress(buffer[, options], callback)`
|
||||
|
||||
<!-- YAML
|
||||
added: REPLACEME
|
||||
-->
|
||||
|
||||
* `buffer` {Buffer|TypedArray|DataView|ArrayBuffer|string}
|
||||
* `options` {zstd options}
|
||||
* `callback` {Function}
|
||||
|
||||
### `zlib.zstdCompressSync(buffer[, options])`
|
||||
|
||||
<!-- YAML
|
||||
added: REPLACEME
|
||||
-->
|
||||
|
||||
* `buffer` {Buffer|TypedArray|DataView|ArrayBuffer|string}
|
||||
* `options` {zstd options}
|
||||
|
||||
Compress a chunk of data with [`ZstdCompress`][].
|
||||
|
||||
### `zlib.zstdDecompress(buffer[, options], callback)`
|
||||
|
||||
<!-- YAML
|
||||
added: REPLACEME
|
||||
-->
|
||||
|
||||
* `buffer` {Buffer|TypedArray|DataView|ArrayBuffer|string}
|
||||
* `options` {zstd options}
|
||||
* `callback` {Function}
|
||||
|
||||
### `zlib.zstdDecompressSync(buffer[, options])`
|
||||
|
||||
<!-- YAML
|
||||
added: REPLACEME
|
||||
-->
|
||||
|
||||
* `buffer` {Buffer|TypedArray|DataView|ArrayBuffer|string}
|
||||
* `options` {zstd options}
|
||||
|
||||
Decompress a chunk of data with [`ZstdDecompress`][].
|
||||
|
||||
[Brotli parameters]: #brotli-constants
|
||||
[Cyclic redundancy check]: https://en.wikipedia.org/wiki/Cyclic_redundancy_check
|
||||
[Memory usage tuning]: #memory-usage-tuning
|
||||
[RFC 7932]: https://www.rfc-editor.org/rfc/rfc7932.txt
|
||||
[Streams API]: stream.md
|
||||
[Zstd parameters]: #zstd-constants
|
||||
[`.flush()`]: #zlibflushkind-callback
|
||||
[`Accept-Encoding`]: https://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.3
|
||||
[`BrotliCompress`]: #class-zlibbrotlicompress
|
||||
@ -1498,6 +1667,8 @@ Decompress a chunk of data with [`Unzip`][].
|
||||
[`InflateRaw`]: #class-zlibinflateraw
|
||||
[`Inflate`]: #class-zlibinflate
|
||||
[`Unzip`]: #class-zlibunzip
|
||||
[`ZstdCompress`]: #class-zlibzstdcompress
|
||||
[`ZstdDecompress`]: #class-zlibzstddecompress
|
||||
[`buffer.kMaxLength`]: buffer.md#bufferkmaxlength
|
||||
[`deflateInit2` and `inflateInit2`]: https://zlib.net/manual.html#Advanced
|
||||
[`stream.Transform`]: stream.md#class-streamtransform
|
||||
|
@ -1889,3 +1889,4 @@ E('ERR_WORKER_UNSERIALIZABLE_ERROR',
|
||||
'Serializing an uncaught exception failed', Error);
|
||||
E('ERR_WORKER_UNSUPPORTED_OPERATION',
|
||||
'%s is not supported in workers', TypeError);
|
||||
E('ERR_ZSTD_INVALID_PARAM', '%s is not a valid zstd parameter', RangeError);
|
||||
|
107
lib/zlib.js
107
lib/zlib.js
@ -42,6 +42,7 @@ const {
|
||||
ERR_BUFFER_TOO_LARGE,
|
||||
ERR_INVALID_ARG_TYPE,
|
||||
ERR_OUT_OF_RANGE,
|
||||
ERR_ZSTD_INVALID_PARAM,
|
||||
},
|
||||
genericNodeError,
|
||||
} = require('internal/errors');
|
||||
@ -83,9 +84,12 @@ const {
|
||||
// Node's compression stream modes (node_zlib_mode)
|
||||
DEFLATE, DEFLATERAW, INFLATE, INFLATERAW, GZIP, GUNZIP, UNZIP,
|
||||
BROTLI_DECODE, BROTLI_ENCODE,
|
||||
ZSTD_COMPRESS, ZSTD_DECOMPRESS,
|
||||
// Brotli operations (~flush levels)
|
||||
BROTLI_OPERATION_PROCESS, BROTLI_OPERATION_FLUSH,
|
||||
BROTLI_OPERATION_FINISH, BROTLI_OPERATION_EMIT_METADATA,
|
||||
// Zstd end directives (~flush levels)
|
||||
ZSTD_e_continue, ZSTD_e_flush, ZSTD_e_end,
|
||||
} = constants;
|
||||
|
||||
// Translation table for return codes.
|
||||
@ -192,9 +196,11 @@ function zlibOnError(message, errno, code) {
|
||||
const FLUSH_BOUND = [
|
||||
[ Z_NO_FLUSH, Z_BLOCK ],
|
||||
[ BROTLI_OPERATION_PROCESS, BROTLI_OPERATION_EMIT_METADATA ],
|
||||
[ ZSTD_e_continue, ZSTD_e_end ],
|
||||
];
|
||||
const FLUSH_BOUND_IDX_NORMAL = 0;
|
||||
const FLUSH_BOUND_IDX_BROTLI = 1;
|
||||
const FLUSH_BOUND_IDX_ZSTD = 2;
|
||||
|
||||
// The base class for all Zlib-style streams.
|
||||
function ZlibBase(opts, mode, handle, { flush, finishFlush, fullFlush }) {
|
||||
@ -203,13 +209,15 @@ function ZlibBase(opts, mode, handle, { flush, finishFlush, fullFlush }) {
|
||||
// The ZlibBase class is not exported to user land, the mode should only be
|
||||
// passed in by us.
|
||||
assert(typeof mode === 'number');
|
||||
assert(mode >= DEFLATE && mode <= BROTLI_ENCODE);
|
||||
assert(mode >= DEFLATE && mode <= ZSTD_DECOMPRESS);
|
||||
|
||||
let flushBoundIdx;
|
||||
if (mode !== BROTLI_ENCODE && mode !== BROTLI_DECODE) {
|
||||
flushBoundIdx = FLUSH_BOUND_IDX_NORMAL;
|
||||
} else {
|
||||
if (mode === BROTLI_ENCODE || mode === BROTLI_DECODE) {
|
||||
flushBoundIdx = FLUSH_BOUND_IDX_BROTLI;
|
||||
} else if (mode === ZSTD_COMPRESS || mode === ZSTD_DECOMPRESS) {
|
||||
flushBoundIdx = FLUSH_BOUND_IDX_ZSTD;
|
||||
} else {
|
||||
flushBoundIdx = FLUSH_BOUND_IDX_NORMAL;
|
||||
}
|
||||
|
||||
if (opts) {
|
||||
@ -829,6 +837,89 @@ ObjectSetPrototypeOf(BrotliDecompress.prototype, Brotli.prototype);
|
||||
ObjectSetPrototypeOf(BrotliDecompress, Brotli);
|
||||
|
||||
|
||||
const zstdDefaultOpts = {
|
||||
flush: ZSTD_e_continue,
|
||||
finishFlush: ZSTD_e_end,
|
||||
fullFlush: ZSTD_e_flush,
|
||||
};
|
||||
function Zstd(opts, mode, initParamsArray, maxParam) {
|
||||
assert(mode === ZSTD_COMPRESS || mode === ZSTD_DECOMPRESS);
|
||||
|
||||
initParamsArray.fill(-1);
|
||||
if (opts?.params) {
|
||||
ObjectKeys(opts.params).forEach((origKey) => {
|
||||
const key = +origKey;
|
||||
if (NumberIsNaN(key) || key < 0 || key > maxParam ||
|
||||
(initParamsArray[key] | 0) !== -1) {
|
||||
throw new ERR_ZSTD_INVALID_PARAM(origKey);
|
||||
}
|
||||
|
||||
const value = opts.params[origKey];
|
||||
if (typeof value !== 'number' && typeof value !== 'boolean') {
|
||||
throw new ERR_INVALID_ARG_TYPE('options.params[key]',
|
||||
'number', opts.params[origKey]);
|
||||
}
|
||||
initParamsArray[key] = value;
|
||||
});
|
||||
}
|
||||
|
||||
const handle = mode === ZSTD_COMPRESS ?
|
||||
new binding.ZstdCompress() : new binding.ZstdDecompress();
|
||||
|
||||
const pledgedSrcSize = opts?.pledgedSrcSize ?? undefined;
|
||||
|
||||
this._writeState = new Uint32Array(2);
|
||||
handle.init(
|
||||
initParamsArray,
|
||||
pledgedSrcSize,
|
||||
this._writeState,
|
||||
processCallback,
|
||||
);
|
||||
|
||||
ReflectApply(ZlibBase, this, [opts, mode, handle, zstdDefaultOpts]);
|
||||
}
|
||||
ObjectSetPrototypeOf(Zstd.prototype, ZlibBase.prototype);
|
||||
ObjectSetPrototypeOf(Zstd, ZlibBase);
|
||||
|
||||
|
||||
const kMaxZstdCParam = MathMax(...ObjectKeys(constants).map(
|
||||
(key) => (key.startsWith('ZSTD_c_') ?
|
||||
constants[key] :
|
||||
0),
|
||||
));
|
||||
|
||||
const zstdInitCParamsArray = new Uint32Array(kMaxZstdCParam + 1);
|
||||
|
||||
function ZstdCompress(opts) {
|
||||
if (!(this instanceof ZstdCompress))
|
||||
return new ZstdCompress(opts);
|
||||
|
||||
ReflectApply(Zstd, this,
|
||||
[opts, ZSTD_COMPRESS, zstdInitCParamsArray, kMaxZstdCParam]);
|
||||
}
|
||||
ObjectSetPrototypeOf(ZstdCompress.prototype, Zstd.prototype);
|
||||
ObjectSetPrototypeOf(ZstdCompress, Zstd);
|
||||
|
||||
|
||||
const kMaxZstdDParam = MathMax(...ObjectKeys(constants).map(
|
||||
(key) => (key.startsWith('ZSTD_d_') ?
|
||||
constants[key] :
|
||||
0),
|
||||
));
|
||||
|
||||
const zstdInitDParamsArray = new Uint32Array(kMaxZstdDParam + 1);
|
||||
|
||||
function ZstdDecompress(opts) {
|
||||
if (!(this instanceof ZstdDecompress))
|
||||
return new ZstdDecompress(opts);
|
||||
|
||||
ReflectApply(Zstd, this,
|
||||
[opts, ZSTD_DECOMPRESS, zstdInitDParamsArray, kMaxZstdDParam]);
|
||||
}
|
||||
ObjectSetPrototypeOf(ZstdDecompress.prototype, Zstd.prototype);
|
||||
ObjectSetPrototypeOf(ZstdDecompress, Zstd);
|
||||
|
||||
|
||||
function createProperty(ctor) {
|
||||
return {
|
||||
__proto__: null,
|
||||
@ -867,6 +958,8 @@ module.exports = {
|
||||
Unzip,
|
||||
BrotliCompress,
|
||||
BrotliDecompress,
|
||||
ZstdCompress,
|
||||
ZstdDecompress,
|
||||
|
||||
// Convenience methods.
|
||||
// compress/decompress a string or buffer in one step.
|
||||
@ -888,6 +981,10 @@ module.exports = {
|
||||
brotliCompressSync: createConvenienceMethod(BrotliCompress, true),
|
||||
brotliDecompress: createConvenienceMethod(BrotliDecompress, false),
|
||||
brotliDecompressSync: createConvenienceMethod(BrotliDecompress, true),
|
||||
zstdCompress: createConvenienceMethod(ZstdCompress, false),
|
||||
zstdCompressSync: createConvenienceMethod(ZstdCompress, true),
|
||||
zstdDecompress: createConvenienceMethod(ZstdDecompress, false),
|
||||
zstdDecompressSync: createConvenienceMethod(ZstdDecompress, true),
|
||||
};
|
||||
|
||||
ObjectDefineProperties(module.exports, {
|
||||
@ -900,6 +997,8 @@ ObjectDefineProperties(module.exports, {
|
||||
createUnzip: createProperty(Unzip),
|
||||
createBrotliCompress: createProperty(BrotliCompress),
|
||||
createBrotliDecompress: createProperty(BrotliDecompress),
|
||||
createZstdCompress: createProperty(ZstdCompress),
|
||||
createZstdDecompress: createProperty(ZstdDecompress),
|
||||
constants: {
|
||||
__proto__: null,
|
||||
configurable: false,
|
||||
|
384
src/node_zlib.cc
384
src/node_zlib.cc
@ -32,9 +32,11 @@
|
||||
|
||||
#include "v8.h"
|
||||
|
||||
#include "brotli/encode.h"
|
||||
#include "brotli/decode.h"
|
||||
#include "brotli/encode.h"
|
||||
#include "zlib.h"
|
||||
#include "zstd.h"
|
||||
#include "zstd_errors.h"
|
||||
|
||||
#include <sys/types.h>
|
||||
|
||||
@ -95,6 +97,44 @@ inline const char* ZlibStrerror(int err) {
|
||||
return "Z_UNKNOWN_ERROR";
|
||||
}
|
||||
|
||||
#define ZSTD_ERROR_CODES(V) \
|
||||
V(ZSTD_error_no_error) \
|
||||
V(ZSTD_error_GENERIC) \
|
||||
V(ZSTD_error_prefix_unknown) \
|
||||
V(ZSTD_error_version_unsupported) \
|
||||
V(ZSTD_error_frameParameter_unsupported) \
|
||||
V(ZSTD_error_frameParameter_windowTooLarge) \
|
||||
V(ZSTD_error_corruption_detected) \
|
||||
V(ZSTD_error_checksum_wrong) \
|
||||
V(ZSTD_error_literals_headerWrong) \
|
||||
V(ZSTD_error_dictionary_corrupted) \
|
||||
V(ZSTD_error_dictionary_wrong) \
|
||||
V(ZSTD_error_dictionaryCreation_failed) \
|
||||
V(ZSTD_error_parameter_unsupported) \
|
||||
V(ZSTD_error_parameter_combination_unsupported) \
|
||||
V(ZSTD_error_parameter_outOfBound) \
|
||||
V(ZSTD_error_tableLog_tooLarge) \
|
||||
V(ZSTD_error_maxSymbolValue_tooLarge) \
|
||||
V(ZSTD_error_maxSymbolValue_tooSmall) \
|
||||
V(ZSTD_error_stabilityCondition_notRespected) \
|
||||
V(ZSTD_error_stage_wrong) \
|
||||
V(ZSTD_error_init_missing) \
|
||||
V(ZSTD_error_memory_allocation) \
|
||||
V(ZSTD_error_workSpace_tooSmall) \
|
||||
V(ZSTD_error_dstSize_tooSmall) \
|
||||
V(ZSTD_error_srcSize_wrong) \
|
||||
V(ZSTD_error_dstBuffer_null) \
|
||||
V(ZSTD_error_noForwardProgress_destFull) \
|
||||
V(ZSTD_error_noForwardProgress_inputEmpty)
|
||||
|
||||
inline const char* ZstdStrerror(int err) {
|
||||
#define V(code) \
|
||||
if (err == code) return #code;
|
||||
ZSTD_ERROR_CODES(V)
|
||||
#undef V
|
||||
return "ZSTD_error_GENERIC";
|
||||
}
|
||||
|
||||
enum node_zlib_mode {
|
||||
NONE,
|
||||
DEFLATE,
|
||||
@ -105,7 +145,9 @@ enum node_zlib_mode {
|
||||
INFLATERAW,
|
||||
UNZIP,
|
||||
BROTLI_DECODE,
|
||||
BROTLI_ENCODE
|
||||
BROTLI_ENCODE,
|
||||
ZSTD_COMPRESS,
|
||||
ZSTD_DECOMPRESS
|
||||
};
|
||||
|
||||
constexpr uint8_t GZIP_HEADER_ID1 = 0x1f;
|
||||
@ -249,6 +291,79 @@ class BrotliDecoderContext final : public BrotliContext {
|
||||
DeleteFnPtr<BrotliDecoderState, BrotliDecoderDestroyInstance> state_;
|
||||
};
|
||||
|
||||
class ZstdContext : public MemoryRetainer {
|
||||
public:
|
||||
ZstdContext() = default;
|
||||
|
||||
// Streaming-related, should be available for all compression libraries:
|
||||
void Close();
|
||||
void SetBuffers(const char* in, uint32_t in_len, char* out, uint32_t out_len);
|
||||
void SetFlush(int flush);
|
||||
void GetAfterWriteOffsets(uint32_t* avail_in, uint32_t* avail_out) const;
|
||||
CompressionError GetErrorInfo() const;
|
||||
|
||||
ZstdContext(const ZstdContext&) = delete;
|
||||
ZstdContext& operator=(const ZstdContext&) = delete;
|
||||
|
||||
protected:
|
||||
ZSTD_EndDirective flush_ = ZSTD_e_continue;
|
||||
|
||||
ZSTD_inBuffer input_ = {nullptr, 0, 0};
|
||||
ZSTD_outBuffer output_ = {nullptr, 0, 0};
|
||||
|
||||
ZSTD_ErrorCode error_ = ZSTD_error_no_error;
|
||||
std::string error_string_;
|
||||
std::string error_code_string_;
|
||||
};
|
||||
|
||||
class ZstdCompressContext final : public ZstdContext {
|
||||
public:
|
||||
ZstdCompressContext() = default;
|
||||
|
||||
// Streaming-related, should be available for all compression libraries:
|
||||
void DoThreadPoolWork();
|
||||
CompressionError ResetStream();
|
||||
|
||||
// Zstd specific:
|
||||
CompressionError Init(uint64_t pledged_src_size);
|
||||
CompressionError SetParameter(int key, int value);
|
||||
|
||||
// Wrap ZSTD_freeCCtx to remove the return type.
|
||||
static void FreeZstd(ZSTD_CCtx* cctx) { ZSTD_freeCCtx(cctx); }
|
||||
|
||||
SET_MEMORY_INFO_NAME(ZstdCompressContext)
|
||||
SET_SELF_SIZE(ZstdCompressContext)
|
||||
SET_NO_MEMORY_INFO()
|
||||
|
||||
private:
|
||||
DeleteFnPtr<ZSTD_CCtx, ZstdCompressContext::FreeZstd> cctx_;
|
||||
|
||||
uint64_t pledged_src_size_ = ZSTD_CONTENTSIZE_UNKNOWN;
|
||||
};
|
||||
|
||||
class ZstdDecompressContext final : public ZstdContext {
|
||||
public:
|
||||
ZstdDecompressContext() = default;
|
||||
|
||||
// Streaming-related, should be available for all compression libraries:
|
||||
void DoThreadPoolWork();
|
||||
CompressionError ResetStream();
|
||||
|
||||
// Zstd specific:
|
||||
CompressionError Init(uint64_t pledged_src_size);
|
||||
CompressionError SetParameter(int key, int value);
|
||||
|
||||
// Wrap ZSTD_freeDCtx to remove the return type.
|
||||
static void FreeZstd(ZSTD_DCtx* dctx) { ZSTD_freeDCtx(dctx); }
|
||||
|
||||
SET_MEMORY_INFO_NAME(ZstdDecompressContext)
|
||||
SET_SELF_SIZE(ZstdDecompressContext)
|
||||
SET_NO_MEMORY_INFO()
|
||||
|
||||
private:
|
||||
DeleteFnPtr<ZSTD_DCtx, ZstdDecompressContext::FreeZstd> dctx_;
|
||||
};
|
||||
|
||||
template <typename CompressionContext>
|
||||
class CompressionStream : public AsyncWrap, public ThreadPoolWork {
|
||||
public:
|
||||
@ -740,6 +855,92 @@ class BrotliCompressionStream final :
|
||||
using BrotliEncoderStream = BrotliCompressionStream<BrotliEncoderContext>;
|
||||
using BrotliDecoderStream = BrotliCompressionStream<BrotliDecoderContext>;
|
||||
|
||||
template <typename CompressionContext>
|
||||
class ZstdStream final : public CompressionStream<CompressionContext> {
|
||||
public:
|
||||
ZstdStream(Environment* env, Local<Object> wrap)
|
||||
: CompressionStream<CompressionContext>(env, wrap) {}
|
||||
|
||||
inline CompressionContext* context() {
|
||||
return this->CompressionStream<CompressionContext>::context();
|
||||
}
|
||||
typedef typename CompressionStream<CompressionContext>::AllocScope AllocScope;
|
||||
|
||||
static void New(const FunctionCallbackInfo<Value>& args) {
|
||||
Environment* env = Environment::GetCurrent(args);
|
||||
new ZstdStream(env, args.This());
|
||||
}
|
||||
|
||||
static void Init(const FunctionCallbackInfo<Value>& args) {
|
||||
Environment* env = Environment::GetCurrent(args);
|
||||
Local<Context> context = env->context();
|
||||
|
||||
CHECK(args.Length() == 4 &&
|
||||
"init(params, pledgedSrcSize, writeResult, writeCallback)");
|
||||
ZstdStream* wrap;
|
||||
ASSIGN_OR_RETURN_UNWRAP(&wrap, args.Holder());
|
||||
|
||||
CHECK(args[2]->IsUint32Array());
|
||||
uint32_t* write_result = reinterpret_cast<uint32_t*>(Buffer::Data(args[2]));
|
||||
|
||||
CHECK(args[3]->IsFunction());
|
||||
Local<Function> write_js_callback = args[3].As<Function>();
|
||||
wrap->InitStream(write_result, write_js_callback);
|
||||
|
||||
uint64_t pledged_src_size = ZSTD_CONTENTSIZE_UNKNOWN;
|
||||
if (args[1]->IsNumber()) {
|
||||
int64_t signed_pledged_src_size;
|
||||
if (!args[1]->IntegerValue(context).To(&signed_pledged_src_size)) {
|
||||
THROW_ERR_INVALID_ARG_VALUE(wrap->env(),
|
||||
"pledgedSrcSize should be an integer");
|
||||
return;
|
||||
}
|
||||
if (signed_pledged_src_size < 0) {
|
||||
THROW_ERR_INVALID_ARG_VALUE(wrap->env(),
|
||||
"pledgedSrcSize may not be negative");
|
||||
return;
|
||||
}
|
||||
pledged_src_size = signed_pledged_src_size;
|
||||
}
|
||||
|
||||
AllocScope alloc_scope(wrap);
|
||||
CompressionError err = wrap->context()->Init(pledged_src_size);
|
||||
if (err.IsError()) {
|
||||
wrap->EmitError(err);
|
||||
THROW_ERR_ZLIB_INITIALIZATION_FAILED(wrap->env(), err.message);
|
||||
return;
|
||||
}
|
||||
|
||||
CHECK(args[0]->IsUint32Array());
|
||||
const uint32_t* data = reinterpret_cast<uint32_t*>(Buffer::Data(args[0]));
|
||||
size_t len = args[0].As<Uint32Array>()->Length();
|
||||
|
||||
for (int i = 0; static_cast<size_t>(i) < len; i++) {
|
||||
if (data[i] == static_cast<uint32_t>(-1)) continue;
|
||||
|
||||
CompressionError err = wrap->context()->SetParameter(i, data[i]);
|
||||
if (err.IsError()) {
|
||||
wrap->EmitError(err);
|
||||
THROW_ERR_ZLIB_INITIALIZATION_FAILED(wrap->env(), err.message);
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
static void Params(const FunctionCallbackInfo<Value>& args) {
|
||||
// Currently a no-op, and not accessed from JS land.
|
||||
// At some point zstd may support changing parameters on the fly,
|
||||
// in which case we can implement this and a JS equivalent similar to
|
||||
// the zlib Params() function.
|
||||
}
|
||||
|
||||
SET_MEMORY_INFO_NAME(ZstdStream)
|
||||
SET_SELF_SIZE(ZstdStream)
|
||||
};
|
||||
|
||||
using ZstdCompressStream = ZstdStream<ZstdCompressContext>;
|
||||
using ZstdDecompressStream = ZstdStream<ZstdDecompressContext>;
|
||||
|
||||
void ZlibContext::Close() {
|
||||
{
|
||||
Mutex::ScopedLock lock(mutex_);
|
||||
@ -1263,6 +1464,115 @@ CompressionError BrotliDecoderContext::GetErrorInfo() const {
|
||||
}
|
||||
}
|
||||
|
||||
void ZstdContext::Close() {}
|
||||
|
||||
void ZstdContext::SetBuffers(const char* in,
|
||||
uint32_t in_len,
|
||||
char* out,
|
||||
uint32_t out_len) {
|
||||
input_.src = reinterpret_cast<const uint8_t*>(in);
|
||||
input_.size = in_len;
|
||||
input_.pos = 0;
|
||||
|
||||
output_.dst = reinterpret_cast<uint8_t*>(out);
|
||||
output_.size = out_len;
|
||||
output_.pos = 0;
|
||||
}
|
||||
|
||||
void ZstdContext::SetFlush(int flush) {
|
||||
flush_ = static_cast<ZSTD_EndDirective>(flush);
|
||||
}
|
||||
|
||||
void ZstdContext::GetAfterWriteOffsets(uint32_t* avail_in,
|
||||
uint32_t* avail_out) const {
|
||||
*avail_in = input_.size - input_.pos;
|
||||
*avail_out = output_.size - output_.pos;
|
||||
}
|
||||
|
||||
CompressionError ZstdContext::GetErrorInfo() const {
|
||||
if (error_ != ZSTD_error_no_error) {
|
||||
return CompressionError(error_string_.c_str(),
|
||||
error_code_string_.c_str(),
|
||||
static_cast<int>(error_));
|
||||
} else {
|
||||
return {};
|
||||
}
|
||||
}
|
||||
|
||||
CompressionError ZstdCompressContext::SetParameter(int key, int value) {
|
||||
size_t result = ZSTD_CCtx_setParameter(
|
||||
cctx_.get(), static_cast<ZSTD_cParameter>(key), value);
|
||||
if (ZSTD_isError(result)) {
|
||||
return CompressionError(
|
||||
"Setting parameter failed", "ERR_ZSTD_PARAM_SET_FAILED", -1);
|
||||
}
|
||||
return {};
|
||||
}
|
||||
|
||||
CompressionError ZstdCompressContext::Init(uint64_t pledged_src_size) {
|
||||
pledged_src_size_ = pledged_src_size;
|
||||
cctx_.reset(ZSTD_createCCtx());
|
||||
if (!cctx_) {
|
||||
return CompressionError("Could not initialize zstd instance",
|
||||
"ERR_ZLIB_INITIALIZATION_FAILED",
|
||||
-1);
|
||||
}
|
||||
size_t result = ZSTD_CCtx_setPledgedSrcSize(cctx_.get(), pledged_src_size);
|
||||
if (ZSTD_isError(result)) {
|
||||
return CompressionError(
|
||||
"Could not set pledged src size", "ERR_ZLIB_INITIALIZATION_FAILED", -1);
|
||||
}
|
||||
return {};
|
||||
}
|
||||
|
||||
CompressionError ZstdCompressContext::ResetStream() {
|
||||
return Init(pledged_src_size_);
|
||||
}
|
||||
|
||||
void ZstdCompressContext::DoThreadPoolWork() {
|
||||
size_t const remaining =
|
||||
ZSTD_compressStream2(cctx_.get(), &output_, &input_, flush_);
|
||||
if (ZSTD_isError(remaining)) {
|
||||
error_ = ZSTD_getErrorCode(remaining);
|
||||
error_code_string_ = ZstdStrerror(error_);
|
||||
error_string_ = ZSTD_getErrorString(error_);
|
||||
}
|
||||
}
|
||||
|
||||
CompressionError ZstdDecompressContext::SetParameter(int key, int value) {
|
||||
size_t result = ZSTD_DCtx_setParameter(
|
||||
dctx_.get(), static_cast<ZSTD_dParameter>(key), value);
|
||||
if (ZSTD_isError(result)) {
|
||||
return CompressionError(
|
||||
"Setting parameter failed", "ERR_ZSTD_PARAM_SET_FAILED", -1);
|
||||
}
|
||||
return {};
|
||||
}
|
||||
|
||||
CompressionError ZstdDecompressContext::Init(uint64_t pledged_src_size) {
|
||||
dctx_.reset(ZSTD_createDCtx());
|
||||
if (!dctx_) {
|
||||
return CompressionError("Could not initialize zstd instance",
|
||||
"ERR_ZLIB_INITIALIZATION_FAILED",
|
||||
-1);
|
||||
}
|
||||
return {};
|
||||
}
|
||||
|
||||
CompressionError ZstdDecompressContext::ResetStream() {
|
||||
// We pass ZSTD_CONTENTSIZE_UNKNOWN because the argument is ignored for
|
||||
// decompression.
|
||||
return Init(ZSTD_CONTENTSIZE_UNKNOWN);
|
||||
}
|
||||
|
||||
void ZstdDecompressContext::DoThreadPoolWork() {
|
||||
size_t const ret = ZSTD_decompressStream(dctx_.get(), &output_, &input_);
|
||||
if (ZSTD_isError(ret)) {
|
||||
error_ = ZSTD_getErrorCode(ret);
|
||||
error_code_string_ = ZstdStrerror(error_);
|
||||
error_string_ = ZSTD_getErrorString(error_);
|
||||
}
|
||||
}
|
||||
|
||||
template <typename Stream>
|
||||
struct MakeClass {
|
||||
@ -1332,6 +1642,8 @@ void Initialize(Local<Object> target,
|
||||
MakeClass<ZlibStream>::Make(env, target, "Zlib");
|
||||
MakeClass<BrotliEncoderStream>::Make(env, target, "BrotliEncoder");
|
||||
MakeClass<BrotliDecoderStream>::Make(env, target, "BrotliDecoder");
|
||||
MakeClass<ZstdCompressStream>::Make(env, target, "ZstdCompress");
|
||||
MakeClass<ZstdDecompressStream>::Make(env, target, "ZstdDecompress");
|
||||
|
||||
SetMethod(context, target, "crc32", CRC32);
|
||||
target->Set(env->context(),
|
||||
@ -1343,6 +1655,8 @@ void RegisterExternalReferences(ExternalReferenceRegistry* registry) {
|
||||
MakeClass<ZlibStream>::Make(registry);
|
||||
MakeClass<BrotliEncoderStream>::Make(registry);
|
||||
MakeClass<BrotliDecoderStream>::Make(registry);
|
||||
MakeClass<ZstdCompressStream>::Make(registry);
|
||||
MakeClass<ZstdDecompressStream>::Make(registry);
|
||||
registry->Register(CRC32);
|
||||
}
|
||||
|
||||
@ -1387,6 +1701,8 @@ void DefineZlibConstants(Local<Object> target) {
|
||||
NODE_DEFINE_CONSTANT(target, UNZIP);
|
||||
NODE_DEFINE_CONSTANT(target, BROTLI_DECODE);
|
||||
NODE_DEFINE_CONSTANT(target, BROTLI_ENCODE);
|
||||
NODE_DEFINE_CONSTANT(target, ZSTD_DECOMPRESS);
|
||||
NODE_DEFINE_CONSTANT(target, ZSTD_COMPRESS);
|
||||
|
||||
NODE_DEFINE_CONSTANT(target, Z_MIN_WINDOWBITS);
|
||||
NODE_DEFINE_CONSTANT(target, Z_MAX_WINDOWBITS);
|
||||
@ -1466,6 +1782,70 @@ void DefineZlibConstants(Local<Object> target) {
|
||||
NODE_DEFINE_CONSTANT(target, BROTLI_DECODER_ERROR_ALLOC_RING_BUFFER_2);
|
||||
NODE_DEFINE_CONSTANT(target, BROTLI_DECODER_ERROR_ALLOC_BLOCK_TYPE_TREES);
|
||||
NODE_DEFINE_CONSTANT(target, BROTLI_DECODER_ERROR_UNREACHABLE);
|
||||
|
||||
// Zstd constants
|
||||
NODE_DEFINE_CONSTANT(target, ZSTD_e_continue);
|
||||
NODE_DEFINE_CONSTANT(target, ZSTD_e_flush);
|
||||
NODE_DEFINE_CONSTANT(target, ZSTD_e_end);
|
||||
NODE_DEFINE_CONSTANT(target, ZSTD_fast);
|
||||
NODE_DEFINE_CONSTANT(target, ZSTD_dfast);
|
||||
NODE_DEFINE_CONSTANT(target, ZSTD_greedy);
|
||||
NODE_DEFINE_CONSTANT(target, ZSTD_lazy);
|
||||
NODE_DEFINE_CONSTANT(target, ZSTD_lazy2);
|
||||
NODE_DEFINE_CONSTANT(target, ZSTD_btlazy2);
|
||||
NODE_DEFINE_CONSTANT(target, ZSTD_btopt);
|
||||
NODE_DEFINE_CONSTANT(target, ZSTD_btultra);
|
||||
NODE_DEFINE_CONSTANT(target, ZSTD_btultra2);
|
||||
NODE_DEFINE_CONSTANT(target, ZSTD_c_compressionLevel);
|
||||
NODE_DEFINE_CONSTANT(target, ZSTD_c_windowLog);
|
||||
NODE_DEFINE_CONSTANT(target, ZSTD_c_hashLog);
|
||||
NODE_DEFINE_CONSTANT(target, ZSTD_c_chainLog);
|
||||
NODE_DEFINE_CONSTANT(target, ZSTD_c_searchLog);
|
||||
NODE_DEFINE_CONSTANT(target, ZSTD_c_minMatch);
|
||||
NODE_DEFINE_CONSTANT(target, ZSTD_c_targetLength);
|
||||
NODE_DEFINE_CONSTANT(target, ZSTD_c_strategy);
|
||||
NODE_DEFINE_CONSTANT(target, ZSTD_c_enableLongDistanceMatching);
|
||||
NODE_DEFINE_CONSTANT(target, ZSTD_c_ldmHashLog);
|
||||
NODE_DEFINE_CONSTANT(target, ZSTD_c_ldmMinMatch);
|
||||
NODE_DEFINE_CONSTANT(target, ZSTD_c_ldmBucketSizeLog);
|
||||
NODE_DEFINE_CONSTANT(target, ZSTD_c_ldmHashRateLog);
|
||||
NODE_DEFINE_CONSTANT(target, ZSTD_c_contentSizeFlag);
|
||||
NODE_DEFINE_CONSTANT(target, ZSTD_c_checksumFlag);
|
||||
NODE_DEFINE_CONSTANT(target, ZSTD_c_dictIDFlag);
|
||||
NODE_DEFINE_CONSTANT(target, ZSTD_c_nbWorkers);
|
||||
NODE_DEFINE_CONSTANT(target, ZSTD_c_jobSize);
|
||||
NODE_DEFINE_CONSTANT(target, ZSTD_c_overlapLog);
|
||||
NODE_DEFINE_CONSTANT(target, ZSTD_d_windowLogMax);
|
||||
NODE_DEFINE_CONSTANT(target, ZSTD_CLEVEL_DEFAULT);
|
||||
// Error codes
|
||||
NODE_DEFINE_CONSTANT(target, ZSTD_error_no_error);
|
||||
NODE_DEFINE_CONSTANT(target, ZSTD_error_GENERIC);
|
||||
NODE_DEFINE_CONSTANT(target, ZSTD_error_prefix_unknown);
|
||||
NODE_DEFINE_CONSTANT(target, ZSTD_error_version_unsupported);
|
||||
NODE_DEFINE_CONSTANT(target, ZSTD_error_frameParameter_unsupported);
|
||||
NODE_DEFINE_CONSTANT(target, ZSTD_error_frameParameter_windowTooLarge);
|
||||
NODE_DEFINE_CONSTANT(target, ZSTD_error_corruption_detected);
|
||||
NODE_DEFINE_CONSTANT(target, ZSTD_error_checksum_wrong);
|
||||
NODE_DEFINE_CONSTANT(target, ZSTD_error_literals_headerWrong);
|
||||
NODE_DEFINE_CONSTANT(target, ZSTD_error_dictionary_corrupted);
|
||||
NODE_DEFINE_CONSTANT(target, ZSTD_error_dictionary_wrong);
|
||||
NODE_DEFINE_CONSTANT(target, ZSTD_error_dictionaryCreation_failed);
|
||||
NODE_DEFINE_CONSTANT(target, ZSTD_error_parameter_unsupported);
|
||||
NODE_DEFINE_CONSTANT(target, ZSTD_error_parameter_combination_unsupported);
|
||||
NODE_DEFINE_CONSTANT(target, ZSTD_error_parameter_outOfBound);
|
||||
NODE_DEFINE_CONSTANT(target, ZSTD_error_tableLog_tooLarge);
|
||||
NODE_DEFINE_CONSTANT(target, ZSTD_error_maxSymbolValue_tooLarge);
|
||||
NODE_DEFINE_CONSTANT(target, ZSTD_error_maxSymbolValue_tooSmall);
|
||||
NODE_DEFINE_CONSTANT(target, ZSTD_error_stabilityCondition_notRespected);
|
||||
NODE_DEFINE_CONSTANT(target, ZSTD_error_stage_wrong);
|
||||
NODE_DEFINE_CONSTANT(target, ZSTD_error_init_missing);
|
||||
NODE_DEFINE_CONSTANT(target, ZSTD_error_memory_allocation);
|
||||
NODE_DEFINE_CONSTANT(target, ZSTD_error_workSpace_tooSmall);
|
||||
NODE_DEFINE_CONSTANT(target, ZSTD_error_dstSize_tooSmall);
|
||||
NODE_DEFINE_CONSTANT(target, ZSTD_error_srcSize_wrong);
|
||||
NODE_DEFINE_CONSTANT(target, ZSTD_error_dstBuffer_null);
|
||||
NODE_DEFINE_CONSTANT(target, ZSTD_error_noForwardProgress_destFull);
|
||||
NODE_DEFINE_CONSTANT(target, ZSTD_error_noForwardProgress_inputEmpty);
|
||||
}
|
||||
|
||||
} // namespace node
|
||||
|
BIN
test/fixtures/person.jpg.zst
vendored
Normal file
BIN
test/fixtures/person.jpg.zst
vendored
Normal file
Binary file not shown.
@ -54,6 +54,8 @@ for (const [type, expect] of [
|
||||
['deflateRaw', 'inflateRaw', 'DeflateRaw', 'InflateRaw'],
|
||||
['brotliCompress', 'brotliDecompress',
|
||||
'BrotliCompress', 'BrotliDecompress'],
|
||||
['zstdCompress', 'zstdDecompress',
|
||||
'ZstdCompress', 'ZstdDecompress'],
|
||||
]) {
|
||||
zlib[method[0]](expect, opts, common.mustCall((err, result) => {
|
||||
zlib[method[1]](result, opts, common.mustCall((err, result) => {
|
||||
|
@ -11,10 +11,12 @@ const emptyBuffer = Buffer.alloc(0);
|
||||
[ zlib.deflateSync, zlib.inflateSync, 'deflate sync' ],
|
||||
[ zlib.gzipSync, zlib.gunzipSync, 'gzip sync' ],
|
||||
[ zlib.brotliCompressSync, zlib.brotliDecompressSync, 'br sync' ],
|
||||
[ zlib.zstdCompressSync, zlib.zstdDecompressSync, 'zstd sync' ],
|
||||
[ promisify(zlib.deflateRaw), promisify(zlib.inflateRaw), 'raw' ],
|
||||
[ promisify(zlib.deflate), promisify(zlib.inflate), 'deflate' ],
|
||||
[ promisify(zlib.gzip), promisify(zlib.gunzip), 'gzip' ],
|
||||
[ promisify(zlib.brotliCompress), promisify(zlib.brotliDecompress), 'br' ],
|
||||
[ promisify(zlib.zstdCompress), promisify(zlib.zstdDecompress), 'zstd' ],
|
||||
]) {
|
||||
const compressed = await compress(emptyBuffer);
|
||||
const decompressed = await decompress(compressed);
|
||||
|
@ -40,6 +40,7 @@ const unzips = [
|
||||
zlib.Inflate(),
|
||||
zlib.InflateRaw(),
|
||||
zlib.BrotliDecompress(),
|
||||
zlib.ZstdDecompress(),
|
||||
];
|
||||
|
||||
nonStringInputs.forEach(common.mustCall((input) => {
|
||||
|
@ -144,6 +144,7 @@ class HashStream extends Stream {
|
||||
for (const [ createCompress, createDecompress ] of [
|
||||
[ zlib.createGzip, zlib.createGunzip ],
|
||||
[ zlib.createBrotliCompress, zlib.createBrotliDecompress ],
|
||||
[ zlib.createZstdCompress, zlib.createZstdDecompress ],
|
||||
]) {
|
||||
const inp = new RandomReadStream({ total: 1024, block: 256, jitter: 16 });
|
||||
const out = new HashStream();
|
||||
|
@ -28,9 +28,10 @@ const zlib = require('node:zlib');
|
||||
const { test } = require('node:test');
|
||||
|
||||
test('zlib should accept writing after flush', async () => {
|
||||
for (const [createCompress, createDecompress] of [
|
||||
[zlib.createGzip, zlib.createGunzip],
|
||||
[zlib.createBrotliCompress, zlib.createBrotliDecompress],
|
||||
for (const [ createCompress, createDecompress ] of [
|
||||
[ zlib.createGzip, zlib.createGunzip ],
|
||||
[ zlib.createBrotliCompress, zlib.createBrotliDecompress ],
|
||||
[ zlib.createZstdCompress, zlib.createZstdDecompress ],
|
||||
]) {
|
||||
const { promise, resolve, reject } = Promise.withResolvers();
|
||||
const gzip = createCompress();
|
||||
|
@ -28,7 +28,13 @@ const zlib = require('node:zlib');
|
||||
const { test } = require('node:test');
|
||||
|
||||
test('zlib should properly handle zero byte input', async () => {
|
||||
for (const Compressor of [zlib.Gzip, zlib.BrotliCompress]) {
|
||||
const compressors = [
|
||||
[zlib.Gzip, 20],
|
||||
[zlib.BrotliCompress, 1],
|
||||
[zlib.ZstdCompress, 9],
|
||||
];
|
||||
|
||||
for (const [Compressor, expected] of compressors) {
|
||||
const { promise, resolve, reject } = Promise.withResolvers();
|
||||
const gz = Compressor();
|
||||
const emptyBuffer = Buffer.alloc(0);
|
||||
@ -38,7 +44,6 @@ test('zlib should properly handle zero byte input', async () => {
|
||||
});
|
||||
gz.on('error', reject);
|
||||
gz.on('end', function() {
|
||||
const expected = Compressor === zlib.Gzip ? 20 : 1;
|
||||
assert.strictEqual(received, expected,
|
||||
`${received}, ${expected}, ${Compressor.name}`);
|
||||
resolve();
|
||||
|
28
test/parallel/test-zlib-zstd-flush.js
Normal file
28
test/parallel/test-zlib-zstd-flush.js
Normal file
@ -0,0 +1,28 @@
|
||||
'use strict';
|
||||
require('../common');
|
||||
const assert = require('assert');
|
||||
const zlib = require('zlib');
|
||||
const fixtures = require('../common/fixtures');
|
||||
|
||||
const file = fixtures.readSync('person.jpg');
|
||||
const chunkSize = 16;
|
||||
const compress = new zlib.ZstdCompress();
|
||||
|
||||
const chunk = file.slice(0, chunkSize);
|
||||
const expectedFull = Buffer.from('KLUv/QBYgAAA/9j/4AAQSkZJRgABAQEASA==', 'base64');
|
||||
let actualFull;
|
||||
|
||||
compress.write(chunk, function() {
|
||||
compress.flush(function() {
|
||||
const bufs = [];
|
||||
let buf;
|
||||
while ((buf = compress.read()) !== null)
|
||||
bufs.push(buf);
|
||||
actualFull = Buffer.concat(bufs);
|
||||
});
|
||||
});
|
||||
|
||||
process.once('exit', function() {
|
||||
assert.deepStrictEqual(actualFull.toString('base64'), expectedFull.toString('base64'));
|
||||
assert.deepStrictEqual(actualFull, expectedFull);
|
||||
});
|
38
test/parallel/test-zlib-zstd-from-string.js
Normal file
38
test/parallel/test-zlib-zstd-from-string.js
Normal file
@ -0,0 +1,38 @@
|
||||
'use strict';
|
||||
// Test compressing and uncompressing a string with zstd
|
||||
|
||||
const common = require('../common');
|
||||
const assert = require('assert');
|
||||
const zlib = require('zlib');
|
||||
|
||||
const inputString = 'ΩΩLorem ipsum dolor sit amet, consectetur adipiscing eli' +
|
||||
't. Morbi faucibus, purus at gravida dictum, libero arcu ' +
|
||||
'convallis lacus, in commodo libero metus eu nisi. Nullam' +
|
||||
' commodo, neque nec porta placerat, nisi est fermentum a' +
|
||||
'ugue, vitae gravida tellus sapien sit amet tellus. Aenea' +
|
||||
'n non diam orci. Proin quis elit turpis. Suspendisse non' +
|
||||
' diam ipsum. Suspendisse nec ullamcorper odio. Vestibulu' +
|
||||
'm arcu mi, sodales non suscipit id, ultrices ut massa. S' +
|
||||
'ed ac sem sit amet arcu malesuada fermentum. Nunc sed. ';
|
||||
const compressedString = 'KLUv/QRYRQkA9tc9H6AlhTb/z/7/gbTI3kaWLKnbCtkZu/hXm0j' +
|
||||
'FpNz/VQM2ADMANQBHTuQOpIYzfVv7XGwXrpoIfgXNAB98xW4wV3' +
|
||||
'vnCF2bjcvWZF2wIZ1vr1mSHHvPHU0TgMGBwUFrF0xqReWcWPO8z' +
|
||||
'Ny6wMwFUilN+Lg987Zvs2GSRMy6uYvtovK9Uuhgst6l9FQrXLnA' +
|
||||
'5gpZL7PdI8bO9sDH3tHm73XBzaUK+LjSPNKRmzQ3ZMYEPozdof1' +
|
||||
'2KcZGfIcLa0PTsdkYqhGcAx/E9mWa8EGEeq0Qou2LTmzgg3YJz/' +
|
||||
'21OuXSF+TOd662d60Qyb04xC5dOF4b8JFH8mpHAxAAELu3tg1oa' +
|
||||
'bBEIWaRHdE0l/+0RdEWWIVMAku8TgbiX/4bU+OpLo4UuY1FKDR8' +
|
||||
'RgBc';
|
||||
|
||||
zlib.zstdCompress(inputString, common.mustCall((err, buffer) => {
|
||||
assert(inputString.length > buffer.length);
|
||||
|
||||
zlib.zstdDecompress(buffer, common.mustCall((err, buffer) => {
|
||||
assert.strictEqual(buffer.toString(), inputString);
|
||||
}));
|
||||
}));
|
||||
|
||||
const buffer = Buffer.from(compressedString, 'base64');
|
||||
zlib.zstdDecompress(buffer, common.mustCall((err, buffer) => {
|
||||
assert.strictEqual(buffer.toString(), inputString);
|
||||
}));
|
34
test/parallel/test-zlib-zstd-from-zstd.js
Normal file
34
test/parallel/test-zlib-zstd-from-zstd.js
Normal file
@ -0,0 +1,34 @@
|
||||
'use strict';
|
||||
// Test unzipping a file that was created with a non-node zstd lib,
|
||||
// piped in as fast as possible.
|
||||
//
|
||||
// The compressed fixture was created using the reference CLI:
|
||||
// $ zstd -19 test/fixtures/person.jpg -o test/fixtures/person.jpg.zst
|
||||
|
||||
const common = require('../common');
|
||||
const assert = require('assert');
|
||||
const zlib = require('zlib');
|
||||
const fixtures = require('../common/fixtures');
|
||||
|
||||
const tmpdir = require('../common/tmpdir');
|
||||
tmpdir.refresh();
|
||||
|
||||
const decompress = new zlib.ZstdDecompress();
|
||||
|
||||
const fs = require('fs');
|
||||
|
||||
const fixture = fixtures.path('person.jpg.zst');
|
||||
const unzippedFixture = fixtures.path('person.jpg');
|
||||
const outputFile = tmpdir.resolve('person.jpg');
|
||||
const expect = fs.readFileSync(unzippedFixture);
|
||||
const inp = fs.createReadStream(fixture);
|
||||
const out = fs.createWriteStream(outputFile);
|
||||
|
||||
inp.pipe(decompress).pipe(out);
|
||||
out.on('close', common.mustCall(() => {
|
||||
const actual = fs.readFileSync(outputFile);
|
||||
assert.strictEqual(actual.length, expect.length);
|
||||
for (let i = 0, l = actual.length; i < l; i++) {
|
||||
assert.strictEqual(actual[i], expect[i], `byte[${i}]`);
|
||||
}
|
||||
}));
|
29
test/parallel/test-zlib-zstd-kmaxlength-rangeerror.js
Normal file
29
test/parallel/test-zlib-zstd-kmaxlength-rangeerror.js
Normal file
@ -0,0 +1,29 @@
|
||||
'use strict';
|
||||
require('../common');
|
||||
|
||||
// This test ensures that zlib throws a RangeError if the final buffer needs to
|
||||
// be larger than kMaxLength and concatenation fails.
|
||||
// https://github.com/nodejs/node/pull/1811
|
||||
|
||||
const assert = require('assert');
|
||||
|
||||
// Change kMaxLength for zlib to trigger the error without having to allocate
|
||||
// large Buffers.
|
||||
const buffer = require('buffer');
|
||||
const oldkMaxLength = buffer.kMaxLength;
|
||||
buffer.kMaxLength = 64;
|
||||
const zlib = require('zlib');
|
||||
buffer.kMaxLength = oldkMaxLength;
|
||||
|
||||
// "a".repeat(128), compressed using zstd.
|
||||
const encoded = Buffer.from('KLUv/SCARQAAEGFhAQA7BVg=', 'base64');
|
||||
|
||||
// Async
|
||||
zlib.zstdDecompress(encoded, function(err) {
|
||||
assert.ok(err instanceof RangeError);
|
||||
});
|
||||
|
||||
// Sync
|
||||
assert.throws(function() {
|
||||
zlib.zstdDecompressSync(encoded);
|
||||
}, RangeError);
|
37
test/parallel/test-zlib-zstd-pledged-src-size.js
Normal file
37
test/parallel/test-zlib-zstd-pledged-src-size.js
Normal file
@ -0,0 +1,37 @@
|
||||
'use strict';
|
||||
const common = require('../common');
|
||||
const assert = require('assert');
|
||||
const zlib = require('zlib');
|
||||
|
||||
function compressWithPledgedSrcSize({ pledgedSrcSize, actualSrcSize }) {
|
||||
return new Promise((resolve, reject) => {
|
||||
const compressor = zlib.createZstdCompress({ pledgedSrcSize });
|
||||
compressor.on('error', (e) => {
|
||||
reject(e);
|
||||
});
|
||||
compressor.on('end', resolve);
|
||||
compressor.write('x'.repeat(actualSrcSize), () => {
|
||||
compressor.end();
|
||||
compressor.resume();
|
||||
});
|
||||
}).then(() => {
|
||||
// Compression should only succeed if sizes match
|
||||
assert.strictEqual(pledgedSrcSize, actualSrcSize);
|
||||
}, (error) => {
|
||||
assert.strictEqual(error.code, 'ZSTD_error_srcSize_wrong');
|
||||
// Size error should only happen when sizes do not match
|
||||
assert.notStrictEqual(pledgedSrcSize, actualSrcSize);
|
||||
}).then(common.mustCall());
|
||||
}
|
||||
|
||||
compressWithPledgedSrcSize({ pledgedSrcSize: 0, actualSrcSize: 0 });
|
||||
|
||||
compressWithPledgedSrcSize({ pledgedSrcSize: 0, actualSrcSize: 42 });
|
||||
|
||||
compressWithPledgedSrcSize({ pledgedSrcSize: 13, actualSrcSize: 42 });
|
||||
|
||||
compressWithPledgedSrcSize({ pledgedSrcSize: 42, actualSrcSize: 0 });
|
||||
|
||||
compressWithPledgedSrcSize({ pledgedSrcSize: 42, actualSrcSize: 13 });
|
||||
|
||||
compressWithPledgedSrcSize({ pledgedSrcSize: 42, actualSrcSize: 42 });
|
134
test/parallel/test-zlib-zstd.js
Normal file
134
test/parallel/test-zlib-zstd.js
Normal file
@ -0,0 +1,134 @@
|
||||
'use strict';
|
||||
require('../common');
|
||||
const fixtures = require('../common/fixtures');
|
||||
const assert = require('assert');
|
||||
const zlib = require('zlib');
|
||||
|
||||
// Test some zstd-specific properties of the zstd streams that can not
|
||||
// be easily covered through expanding zlib-only tests.
|
||||
|
||||
const sampleBuffer = fixtures.readSync('/pss-vectors.json');
|
||||
|
||||
{
|
||||
// Test setting the quality parameter at stream creation:
|
||||
const sizes = [];
|
||||
for (let quality = 1;
|
||||
quality <= 22;
|
||||
quality++) {
|
||||
const encoded = zlib.zstdCompressSync(sampleBuffer, {
|
||||
params: {
|
||||
[zlib.constants.ZSTD_c_compressionLevel]: quality
|
||||
}
|
||||
});
|
||||
sizes.push(encoded.length);
|
||||
}
|
||||
|
||||
// Increasing quality should roughly correspond to decreasing compressed size:
|
||||
for (let i = 0; i < sizes.length - 1; i++) {
|
||||
assert(sizes[i + 1] <= sizes[i] * 1.05, sizes); // 5 % margin of error.
|
||||
}
|
||||
assert(sizes[0] > sizes[sizes.length - 1], sizes);
|
||||
}
|
||||
|
||||
{
|
||||
// Test that setting out-of-bounds option values or keys fails.
|
||||
assert.throws(() => {
|
||||
zlib.createZstdCompress({
|
||||
params: {
|
||||
10000: 0
|
||||
}
|
||||
});
|
||||
}, {
|
||||
code: 'ERR_ZSTD_INVALID_PARAM',
|
||||
name: 'RangeError',
|
||||
message: '10000 is not a valid zstd parameter'
|
||||
});
|
||||
|
||||
// Test that accidentally using duplicate keys fails.
|
||||
assert.throws(() => {
|
||||
zlib.createZstdCompress({
|
||||
params: {
|
||||
'0': 0,
|
||||
'00': 0
|
||||
}
|
||||
});
|
||||
}, {
|
||||
code: 'ERR_ZSTD_INVALID_PARAM',
|
||||
name: 'RangeError',
|
||||
message: '00 is not a valid zstd parameter'
|
||||
});
|
||||
|
||||
assert.throws(() => {
|
||||
zlib.createZstdCompress({
|
||||
params: {
|
||||
// This param must be a valid ZSTD_strategy value.
|
||||
[zlib.constants.ZSTD_c_strategy]: 130
|
||||
}
|
||||
});
|
||||
}, {
|
||||
code: 'ERR_ZLIB_INITIALIZATION_FAILED',
|
||||
name: 'Error',
|
||||
message: 'Setting parameter failed'
|
||||
});
|
||||
|
||||
// Test that setting out-of-bounds option values or keys fails.
|
||||
assert.throws(() => {
|
||||
zlib.createZstdDecompress({
|
||||
params: {
|
||||
10000: 0
|
||||
}
|
||||
});
|
||||
}, {
|
||||
code: 'ERR_ZSTD_INVALID_PARAM',
|
||||
name: 'RangeError',
|
||||
message: '10000 is not a valid zstd parameter'
|
||||
});
|
||||
|
||||
// Test that accidentally using duplicate keys fails.
|
||||
assert.throws(() => {
|
||||
zlib.createZstdDecompress({
|
||||
params: {
|
||||
'0': 0,
|
||||
'00': 0
|
||||
}
|
||||
});
|
||||
}, {
|
||||
code: 'ERR_ZSTD_INVALID_PARAM',
|
||||
name: 'RangeError',
|
||||
message: '00 is not a valid zstd parameter'
|
||||
});
|
||||
|
||||
assert.throws(() => {
|
||||
zlib.createZstdDecompress({
|
||||
params: {
|
||||
// This param must be >= 10 (ZSTD_WINDOWLOG_ABSOLUTEMIN).
|
||||
[zlib.constants.ZSTD_d_windowLogMax]: 1
|
||||
}
|
||||
});
|
||||
}, {
|
||||
code: 'ERR_ZLIB_INITIALIZATION_FAILED',
|
||||
name: 'Error',
|
||||
message: 'Setting parameter failed'
|
||||
});
|
||||
}
|
||||
|
||||
{
|
||||
// Test options.flush range
|
||||
assert.throws(() => {
|
||||
zlib.zstdCompressSync('', { flush: zlib.constants.Z_FINISH });
|
||||
}, {
|
||||
code: 'ERR_OUT_OF_RANGE',
|
||||
name: 'RangeError',
|
||||
message: 'The value of "options.flush" is out of range. It must be >= 0 ' +
|
||||
'and <= 2. Received 4',
|
||||
});
|
||||
|
||||
assert.throws(() => {
|
||||
zlib.zstdCompressSync('', { finishFlush: zlib.constants.Z_FINISH });
|
||||
}, {
|
||||
code: 'ERR_OUT_OF_RANGE',
|
||||
name: 'RangeError',
|
||||
message: 'The value of "options.finishFlush" is out of range. It must be ' +
|
||||
'>= 0 and <= 2. Received 4',
|
||||
});
|
||||
}
|
@ -43,6 +43,7 @@ let zlibPairs = [
|
||||
[zlib.Gzip, zlib.Unzip],
|
||||
[zlib.DeflateRaw, zlib.InflateRaw],
|
||||
[zlib.BrotliCompress, zlib.BrotliDecompress],
|
||||
[zlib.ZstdCompress, zlib.ZstdDecompress],
|
||||
];
|
||||
|
||||
// How fast to trickle through the slowstream
|
||||
|
@ -252,6 +252,7 @@ const customTypesMap = {
|
||||
'X509Certificate': 'crypto.html#class-x509certificate',
|
||||
|
||||
'zlib options': 'zlib.html#class-options',
|
||||
'zstd options': 'zlib.html#class-zstdoptions',
|
||||
|
||||
'ReadableStream':
|
||||
'webstreams.html#class-readablestream',
|
||||
|
Loading…
Reference in New Issue
Block a user