mirror of
https://github.com/nodejs/node.git
synced 2025-04-29 06:19:07 +00:00
zlib: add zstd support
Some checks are pending
Coverage Linux (without intl) / coverage-linux-without-intl (push) Waiting to run
Coverage Linux / coverage-linux (push) Waiting to run
Coverage Windows / coverage-windows (push) Waiting to run
Test and upload documentation to artifacts / build-docs (push) Waiting to run
Linters / lint-addon-docs (push) Waiting to run
Linters / lint-cpp (push) Waiting to run
Linters / format-cpp (push) Waiting to run
Linters / lint-js-and-md (push) Waiting to run
Linters / lint-py (push) Waiting to run
Linters / lint-yaml (push) Waiting to run
Linters / lint-sh (push) Waiting to run
Linters / lint-codeowners (push) Waiting to run
Linters / lint-pr-url (push) Waiting to run
Linters / lint-readme (push) Waiting to run
Notify on Push / Notify on Force Push on `main` (push) Waiting to run
Notify on Push / Notify on Push on `main` that lacks metadata (push) Waiting to run
Scorecard supply-chain security / Scorecard analysis (push) Waiting to run
Some checks are pending
Coverage Linux (without intl) / coverage-linux-without-intl (push) Waiting to run
Coverage Linux / coverage-linux (push) Waiting to run
Coverage Windows / coverage-windows (push) Waiting to run
Test and upload documentation to artifacts / build-docs (push) Waiting to run
Linters / lint-addon-docs (push) Waiting to run
Linters / lint-cpp (push) Waiting to run
Linters / format-cpp (push) Waiting to run
Linters / lint-js-and-md (push) Waiting to run
Linters / lint-py (push) Waiting to run
Linters / lint-yaml (push) Waiting to run
Linters / lint-sh (push) Waiting to run
Linters / lint-codeowners (push) Waiting to run
Linters / lint-pr-url (push) Waiting to run
Linters / lint-readme (push) Waiting to run
Notify on Push / Notify on Force Push on `main` (push) Waiting to run
Notify on Push / Notify on Push on `main` that lacks metadata (push) Waiting to run
Scorecard supply-chain security / Scorecard analysis (push) Waiting to run
Fixes: https://github.com/nodejs/node/issues/48412 PR-URL: https://github.com/nodejs/node/pull/52100 Reviewed-By: Yagiz Nizipli <yagiz@nizipli.com>
This commit is contained in:
parent
f5353100be
commit
bf12d72faa
@ -5,7 +5,7 @@ const zlib = require('zlib');
|
|||||||
const bench = common.createBenchmark(main, {
|
const bench = common.createBenchmark(main, {
|
||||||
type: [
|
type: [
|
||||||
'Deflate', 'DeflateRaw', 'Inflate', 'InflateRaw', 'Gzip', 'Gunzip', 'Unzip',
|
'Deflate', 'DeflateRaw', 'Inflate', 'InflateRaw', 'Gzip', 'Gunzip', 'Unzip',
|
||||||
'BrotliCompress', 'BrotliDecompress',
|
'BrotliCompress', 'BrotliDecompress', 'ZstdCompress', 'ZstdDecompress',
|
||||||
],
|
],
|
||||||
options: ['true', 'false'],
|
options: ['true', 'false'],
|
||||||
n: [5e5],
|
n: [5e5],
|
||||||
|
@ -7,7 +7,7 @@ const bench = common.createBenchmark(main, {
|
|||||||
inputLen: [1024],
|
inputLen: [1024],
|
||||||
duration: [5],
|
duration: [5],
|
||||||
type: ['string', 'buffer'],
|
type: ['string', 'buffer'],
|
||||||
algorithm: ['gzip', 'brotli'],
|
algorithm: ['gzip', 'brotli', 'zstd'],
|
||||||
}, {
|
}, {
|
||||||
test: {
|
test: {
|
||||||
inputLen: 1024,
|
inputLen: 1024,
|
||||||
@ -15,14 +15,19 @@ const bench = common.createBenchmark(main, {
|
|||||||
},
|
},
|
||||||
});
|
});
|
||||||
|
|
||||||
|
const algorithms = {
|
||||||
|
'gzip': [zlib.createGzip, zlib.createGunzip],
|
||||||
|
'brotli': [zlib.createBrotliCompress, zlib.createBrotliDecompress],
|
||||||
|
'zstd': [zlib.createZstdCompress, zlib.createZstdDecompress],
|
||||||
|
};
|
||||||
|
|
||||||
function main({ inputLen, duration, type, algorithm }) {
|
function main({ inputLen, duration, type, algorithm }) {
|
||||||
const buffer = Buffer.alloc(inputLen, fs.readFileSync(__filename));
|
const buffer = Buffer.alloc(inputLen, fs.readFileSync(__filename));
|
||||||
const chunk = type === 'buffer' ? buffer : buffer.toString('utf8');
|
const chunk = type === 'buffer' ? buffer : buffer.toString('utf8');
|
||||||
|
|
||||||
const input = algorithm === 'gzip' ?
|
const [createCompress, createUncompress] = algorithms[algorithm];
|
||||||
zlib.createGzip() : zlib.createBrotliCompress();
|
const input = createCompress();
|
||||||
const output = algorithm === 'gzip' ?
|
const output = createUncompress();
|
||||||
zlib.createGunzip() : zlib.createBrotliDecompress();
|
|
||||||
|
|
||||||
let readFromOutput = 0;
|
let readFromOutput = 0;
|
||||||
input.pipe(output);
|
input.pipe(output);
|
||||||
|
@ -3334,6 +3334,12 @@ The requested functionality is not supported in worker threads.
|
|||||||
|
|
||||||
Creation of a [`zlib`][] object failed due to incorrect configuration.
|
Creation of a [`zlib`][] object failed due to incorrect configuration.
|
||||||
|
|
||||||
|
<a id="ERR_ZSTD_INVALID_PARAM"></a>
|
||||||
|
|
||||||
|
### `ERR_ZSTD_INVALID_PARAM`
|
||||||
|
|
||||||
|
An invalid parameter key was passed during construction of a Zstd stream.
|
||||||
|
|
||||||
<a id="HPE_CHUNK_EXTENSIONS_OVERFLOW"></a>
|
<a id="HPE_CHUNK_EXTENSIONS_OVERFLOW"></a>
|
||||||
|
|
||||||
### `HPE_CHUNK_EXTENSIONS_OVERFLOW`
|
### `HPE_CHUNK_EXTENSIONS_OVERFLOW`
|
||||||
|
179
doc/api/zlib.md
179
doc/api/zlib.md
@ -7,7 +7,7 @@
|
|||||||
<!-- source_link=lib/zlib.js -->
|
<!-- source_link=lib/zlib.js -->
|
||||||
|
|
||||||
The `node:zlib` module provides compression functionality implemented using
|
The `node:zlib` module provides compression functionality implemented using
|
||||||
Gzip, Deflate/Inflate, and Brotli.
|
Gzip, Deflate/Inflate, Brotli, and Zstd.
|
||||||
|
|
||||||
To access it:
|
To access it:
|
||||||
|
|
||||||
@ -220,8 +220,8 @@ operations be cached to avoid duplication of effort.
|
|||||||
|
|
||||||
## Compressing HTTP requests and responses
|
## Compressing HTTP requests and responses
|
||||||
|
|
||||||
The `node:zlib` module can be used to implement support for the `gzip`, `deflate`
|
The `node:zlib` module can be used to implement support for the `gzip`, `deflate`,
|
||||||
and `br` content-encoding mechanisms defined by
|
`br`, and `zstd` content-encoding mechanisms defined by
|
||||||
[HTTP](https://tools.ietf.org/html/rfc7230#section-4.2).
|
[HTTP](https://tools.ietf.org/html/rfc7230#section-4.2).
|
||||||
|
|
||||||
The HTTP [`Accept-Encoding`][] header is used within an HTTP request to identify
|
The HTTP [`Accept-Encoding`][] header is used within an HTTP request to identify
|
||||||
@ -284,7 +284,7 @@ const { pipeline } = require('node:stream');
|
|||||||
const request = http.get({ host: 'example.com',
|
const request = http.get({ host: 'example.com',
|
||||||
path: '/',
|
path: '/',
|
||||||
port: 80,
|
port: 80,
|
||||||
headers: { 'Accept-Encoding': 'br,gzip,deflate' } });
|
headers: { 'Accept-Encoding': 'br,gzip,deflate,zstd' } });
|
||||||
request.on('response', (response) => {
|
request.on('response', (response) => {
|
||||||
const output = fs.createWriteStream('example.com_index.html');
|
const output = fs.createWriteStream('example.com_index.html');
|
||||||
|
|
||||||
@ -306,6 +306,9 @@ request.on('response', (response) => {
|
|||||||
case 'deflate':
|
case 'deflate':
|
||||||
pipeline(response, zlib.createInflate(), output, onError);
|
pipeline(response, zlib.createInflate(), output, onError);
|
||||||
break;
|
break;
|
||||||
|
case 'zstd':
|
||||||
|
pipeline(response, zlib.createZstdDecompress(), output, onError);
|
||||||
|
break;
|
||||||
default:
|
default:
|
||||||
pipeline(response, output, onError);
|
pipeline(response, output, onError);
|
||||||
break;
|
break;
|
||||||
@ -396,6 +399,9 @@ http.createServer((request, response) => {
|
|||||||
} else if (/\bbr\b/.test(acceptEncoding)) {
|
} else if (/\bbr\b/.test(acceptEncoding)) {
|
||||||
response.writeHead(200, { 'Content-Encoding': 'br' });
|
response.writeHead(200, { 'Content-Encoding': 'br' });
|
||||||
pipeline(raw, zlib.createBrotliCompress(), response, onError);
|
pipeline(raw, zlib.createBrotliCompress(), response, onError);
|
||||||
|
} else if (/\bzstd\b/.test(acceptEncoding)) {
|
||||||
|
response.writeHead(200, { 'Content-Encoding': 'zstd' });
|
||||||
|
pipeline(raw, zlib.createZstdCompress(), response, onError);
|
||||||
} else {
|
} else {
|
||||||
response.writeHead(200, {});
|
response.writeHead(200, {});
|
||||||
pipeline(raw, response, onError);
|
pipeline(raw, response, onError);
|
||||||
@ -416,6 +422,7 @@ const buffer = Buffer.from('eJzT0yMA', 'base64');
|
|||||||
zlib.unzip(
|
zlib.unzip(
|
||||||
buffer,
|
buffer,
|
||||||
// For Brotli, the equivalent is zlib.constants.BROTLI_OPERATION_FLUSH.
|
// For Brotli, the equivalent is zlib.constants.BROTLI_OPERATION_FLUSH.
|
||||||
|
// For Zstd, the equivalent is zlib.constants.ZSTD_e_flush.
|
||||||
{ finishFlush: zlib.constants.Z_SYNC_FLUSH },
|
{ finishFlush: zlib.constants.Z_SYNC_FLUSH },
|
||||||
(err, buffer) => {
|
(err, buffer) => {
|
||||||
if (err) {
|
if (err) {
|
||||||
@ -487,6 +494,16 @@ these options have different ranges than the zlib ones:
|
|||||||
|
|
||||||
See [below][Brotli parameters] for more details on Brotli-specific options.
|
See [below][Brotli parameters] for more details on Brotli-specific options.
|
||||||
|
|
||||||
|
### For Zstd-based streams
|
||||||
|
|
||||||
|
There are equivalents to the zlib options for Zstd-based streams, although
|
||||||
|
these options have different ranges than the zlib ones:
|
||||||
|
|
||||||
|
* zlib's `level` option matches Zstd's `ZSTD_c_compressionLevel` option.
|
||||||
|
* zlib's `windowBits` option matches Zstd's `ZSTD_c_windowLog` option.
|
||||||
|
|
||||||
|
See [below][Zstd parameters] for more details on Zstd-specific options.
|
||||||
|
|
||||||
## Flushing
|
## Flushing
|
||||||
|
|
||||||
Calling [`.flush()`][] on a compression stream will make `zlib` return as much
|
Calling [`.flush()`][] on a compression stream will make `zlib` return as much
|
||||||
@ -701,6 +718,50 @@ These advanced options are available for controlling decompression:
|
|||||||
* Boolean flag enabling “Large Window Brotli” mode (not compatible with the
|
* Boolean flag enabling “Large Window Brotli” mode (not compatible with the
|
||||||
Brotli format as standardized in [RFC 7932][]).
|
Brotli format as standardized in [RFC 7932][]).
|
||||||
|
|
||||||
|
### Zstd constants
|
||||||
|
|
||||||
|
<!-- YAML
|
||||||
|
added: REPLACEME
|
||||||
|
-->
|
||||||
|
|
||||||
|
There are several options and other constants available for Zstd-based
|
||||||
|
streams:
|
||||||
|
|
||||||
|
#### Flush operations
|
||||||
|
|
||||||
|
The following values are valid flush operations for Zstd-based streams:
|
||||||
|
|
||||||
|
* `zlib.constants.ZSTD_e_continue` (default for all operations)
|
||||||
|
* `zlib.constants.ZSTD_e_flush` (default when calling `.flush()`)
|
||||||
|
* `zlib.constants.ZSTD_e_end` (default for the last chunk)
|
||||||
|
|
||||||
|
#### Compressor options
|
||||||
|
|
||||||
|
There are several options that can be set on Zstd encoders, affecting
|
||||||
|
compression efficiency and speed. Both the keys and the values can be accessed
|
||||||
|
as properties of the `zlib.constants` object.
|
||||||
|
|
||||||
|
The most important options are:
|
||||||
|
|
||||||
|
* `ZSTD_c_compressionLevel`
|
||||||
|
* Set compression parameters according to pre-defined cLevel table. Default
|
||||||
|
level is ZSTD\_CLEVEL\_DEFAULT==3.
|
||||||
|
|
||||||
|
#### Pledged Source Size
|
||||||
|
|
||||||
|
It's possible to specify the expected total size of the uncompressed input via
|
||||||
|
`opts.pledgedSrcSize`. If the size doesn't match at the end of the input,
|
||||||
|
compression will fail with the code `ZSTD_error_srcSize_wrong`.
|
||||||
|
|
||||||
|
#### Decompressor options
|
||||||
|
|
||||||
|
These advanced options are available for controlling decompression:
|
||||||
|
|
||||||
|
* `ZSTD_d_windowLogMax`
|
||||||
|
* Select a size limit (in power of 2) beyond which the streaming API will
|
||||||
|
refuse to allocate memory buffer in order to protect the host from
|
||||||
|
unreasonable memory requirements.
|
||||||
|
|
||||||
## Class: `Options`
|
## Class: `Options`
|
||||||
|
|
||||||
<!-- YAML
|
<!-- YAML
|
||||||
@ -962,6 +1023,51 @@ added: v0.7.0
|
|||||||
Reset the compressor/decompressor to factory defaults. Only applicable to
|
Reset the compressor/decompressor to factory defaults. Only applicable to
|
||||||
the inflate and deflate algorithms.
|
the inflate and deflate algorithms.
|
||||||
|
|
||||||
|
## Class: `ZstdOptions`
|
||||||
|
|
||||||
|
<!-- YAML
|
||||||
|
added: REPLACEME
|
||||||
|
-->
|
||||||
|
|
||||||
|
<!--type=misc-->
|
||||||
|
|
||||||
|
Each Zstd-based class takes an `options` object. All options are optional.
|
||||||
|
|
||||||
|
* `flush` {integer} **Default:** `zlib.constants.ZSTD_e_continue`
|
||||||
|
* `finishFlush` {integer} **Default:** `zlib.constants.ZSTD_e_end`
|
||||||
|
* `chunkSize` {integer} **Default:** `16 * 1024`
|
||||||
|
* `params` {Object} Key-value object containing indexed [Zstd parameters][].
|
||||||
|
* `maxOutputLength` {integer} Limits output size when using
|
||||||
|
[convenience methods][]. **Default:** [`buffer.kMaxLength`][]
|
||||||
|
|
||||||
|
For example:
|
||||||
|
|
||||||
|
```js
|
||||||
|
const stream = zlib.createZstdCompress({
|
||||||
|
chunkSize: 32 * 1024,
|
||||||
|
params: {
|
||||||
|
[zlib.constants.ZSTD_c_compressionLevel]: 10,
|
||||||
|
[zlib.constants.ZSTD_c_checksumFlag]: 1,
|
||||||
|
},
|
||||||
|
});
|
||||||
|
```
|
||||||
|
|
||||||
|
## Class: `zlib.ZstdCompress`
|
||||||
|
|
||||||
|
<!-- YAML
|
||||||
|
added: REPLACEME
|
||||||
|
-->
|
||||||
|
|
||||||
|
Compress data using the Zstd algorithm.
|
||||||
|
|
||||||
|
## Class: `zlib.ZstdDecompress`
|
||||||
|
|
||||||
|
<!-- YAML
|
||||||
|
added: REPLACEME
|
||||||
|
-->
|
||||||
|
|
||||||
|
Decompress data using the Zstd algorithm.
|
||||||
|
|
||||||
## `zlib.constants`
|
## `zlib.constants`
|
||||||
|
|
||||||
<!-- YAML
|
<!-- YAML
|
||||||
@ -1135,6 +1241,26 @@ added: v0.5.8
|
|||||||
|
|
||||||
Creates and returns a new [`Unzip`][] object.
|
Creates and returns a new [`Unzip`][] object.
|
||||||
|
|
||||||
|
## `zlib.createZstdCompress([options])`
|
||||||
|
|
||||||
|
<!-- YAML
|
||||||
|
added: REPLACEME
|
||||||
|
-->
|
||||||
|
|
||||||
|
* `options` {zstd options}
|
||||||
|
|
||||||
|
Creates and returns a new [`ZstdCompress`][] object.
|
||||||
|
|
||||||
|
## `zlib.createZstdDecompress([options])`
|
||||||
|
|
||||||
|
<!-- YAML
|
||||||
|
added: REPLACEME
|
||||||
|
-->
|
||||||
|
|
||||||
|
* `options` {zstd options}
|
||||||
|
|
||||||
|
Creates and returns a new [`ZstdDecompress`][] object.
|
||||||
|
|
||||||
## Convenience methods
|
## Convenience methods
|
||||||
|
|
||||||
<!--type=misc-->
|
<!--type=misc-->
|
||||||
@ -1481,11 +1607,54 @@ changes:
|
|||||||
|
|
||||||
Decompress a chunk of data with [`Unzip`][].
|
Decompress a chunk of data with [`Unzip`][].
|
||||||
|
|
||||||
|
### `zlib.zstdCompress(buffer[, options], callback)`
|
||||||
|
|
||||||
|
<!-- YAML
|
||||||
|
added: REPLACEME
|
||||||
|
-->
|
||||||
|
|
||||||
|
* `buffer` {Buffer|TypedArray|DataView|ArrayBuffer|string}
|
||||||
|
* `options` {zstd options}
|
||||||
|
* `callback` {Function}
|
||||||
|
|
||||||
|
### `zlib.zstdCompressSync(buffer[, options])`
|
||||||
|
|
||||||
|
<!-- YAML
|
||||||
|
added: REPLACEME
|
||||||
|
-->
|
||||||
|
|
||||||
|
* `buffer` {Buffer|TypedArray|DataView|ArrayBuffer|string}
|
||||||
|
* `options` {zstd options}
|
||||||
|
|
||||||
|
Compress a chunk of data with [`ZstdCompress`][].
|
||||||
|
|
||||||
|
### `zlib.zstdDecompress(buffer[, options], callback)`
|
||||||
|
|
||||||
|
<!-- YAML
|
||||||
|
added: REPLACEME
|
||||||
|
-->
|
||||||
|
|
||||||
|
* `buffer` {Buffer|TypedArray|DataView|ArrayBuffer|string}
|
||||||
|
* `options` {zstd options}
|
||||||
|
* `callback` {Function}
|
||||||
|
|
||||||
|
### `zlib.zstdDecompressSync(buffer[, options])`
|
||||||
|
|
||||||
|
<!-- YAML
|
||||||
|
added: REPLACEME
|
||||||
|
-->
|
||||||
|
|
||||||
|
* `buffer` {Buffer|TypedArray|DataView|ArrayBuffer|string}
|
||||||
|
* `options` {zstd options}
|
||||||
|
|
||||||
|
Decompress a chunk of data with [`ZstdDecompress`][].
|
||||||
|
|
||||||
[Brotli parameters]: #brotli-constants
|
[Brotli parameters]: #brotli-constants
|
||||||
[Cyclic redundancy check]: https://en.wikipedia.org/wiki/Cyclic_redundancy_check
|
[Cyclic redundancy check]: https://en.wikipedia.org/wiki/Cyclic_redundancy_check
|
||||||
[Memory usage tuning]: #memory-usage-tuning
|
[Memory usage tuning]: #memory-usage-tuning
|
||||||
[RFC 7932]: https://www.rfc-editor.org/rfc/rfc7932.txt
|
[RFC 7932]: https://www.rfc-editor.org/rfc/rfc7932.txt
|
||||||
[Streams API]: stream.md
|
[Streams API]: stream.md
|
||||||
|
[Zstd parameters]: #zstd-constants
|
||||||
[`.flush()`]: #zlibflushkind-callback
|
[`.flush()`]: #zlibflushkind-callback
|
||||||
[`Accept-Encoding`]: https://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.3
|
[`Accept-Encoding`]: https://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.3
|
||||||
[`BrotliCompress`]: #class-zlibbrotlicompress
|
[`BrotliCompress`]: #class-zlibbrotlicompress
|
||||||
@ -1498,6 +1667,8 @@ Decompress a chunk of data with [`Unzip`][].
|
|||||||
[`InflateRaw`]: #class-zlibinflateraw
|
[`InflateRaw`]: #class-zlibinflateraw
|
||||||
[`Inflate`]: #class-zlibinflate
|
[`Inflate`]: #class-zlibinflate
|
||||||
[`Unzip`]: #class-zlibunzip
|
[`Unzip`]: #class-zlibunzip
|
||||||
|
[`ZstdCompress`]: #class-zlibzstdcompress
|
||||||
|
[`ZstdDecompress`]: #class-zlibzstddecompress
|
||||||
[`buffer.kMaxLength`]: buffer.md#bufferkmaxlength
|
[`buffer.kMaxLength`]: buffer.md#bufferkmaxlength
|
||||||
[`deflateInit2` and `inflateInit2`]: https://zlib.net/manual.html#Advanced
|
[`deflateInit2` and `inflateInit2`]: https://zlib.net/manual.html#Advanced
|
||||||
[`stream.Transform`]: stream.md#class-streamtransform
|
[`stream.Transform`]: stream.md#class-streamtransform
|
||||||
|
@ -1889,3 +1889,4 @@ E('ERR_WORKER_UNSERIALIZABLE_ERROR',
|
|||||||
'Serializing an uncaught exception failed', Error);
|
'Serializing an uncaught exception failed', Error);
|
||||||
E('ERR_WORKER_UNSUPPORTED_OPERATION',
|
E('ERR_WORKER_UNSUPPORTED_OPERATION',
|
||||||
'%s is not supported in workers', TypeError);
|
'%s is not supported in workers', TypeError);
|
||||||
|
E('ERR_ZSTD_INVALID_PARAM', '%s is not a valid zstd parameter', RangeError);
|
||||||
|
107
lib/zlib.js
107
lib/zlib.js
@ -42,6 +42,7 @@ const {
|
|||||||
ERR_BUFFER_TOO_LARGE,
|
ERR_BUFFER_TOO_LARGE,
|
||||||
ERR_INVALID_ARG_TYPE,
|
ERR_INVALID_ARG_TYPE,
|
||||||
ERR_OUT_OF_RANGE,
|
ERR_OUT_OF_RANGE,
|
||||||
|
ERR_ZSTD_INVALID_PARAM,
|
||||||
},
|
},
|
||||||
genericNodeError,
|
genericNodeError,
|
||||||
} = require('internal/errors');
|
} = require('internal/errors');
|
||||||
@ -83,9 +84,12 @@ const {
|
|||||||
// Node's compression stream modes (node_zlib_mode)
|
// Node's compression stream modes (node_zlib_mode)
|
||||||
DEFLATE, DEFLATERAW, INFLATE, INFLATERAW, GZIP, GUNZIP, UNZIP,
|
DEFLATE, DEFLATERAW, INFLATE, INFLATERAW, GZIP, GUNZIP, UNZIP,
|
||||||
BROTLI_DECODE, BROTLI_ENCODE,
|
BROTLI_DECODE, BROTLI_ENCODE,
|
||||||
|
ZSTD_COMPRESS, ZSTD_DECOMPRESS,
|
||||||
// Brotli operations (~flush levels)
|
// Brotli operations (~flush levels)
|
||||||
BROTLI_OPERATION_PROCESS, BROTLI_OPERATION_FLUSH,
|
BROTLI_OPERATION_PROCESS, BROTLI_OPERATION_FLUSH,
|
||||||
BROTLI_OPERATION_FINISH, BROTLI_OPERATION_EMIT_METADATA,
|
BROTLI_OPERATION_FINISH, BROTLI_OPERATION_EMIT_METADATA,
|
||||||
|
// Zstd end directives (~flush levels)
|
||||||
|
ZSTD_e_continue, ZSTD_e_flush, ZSTD_e_end,
|
||||||
} = constants;
|
} = constants;
|
||||||
|
|
||||||
// Translation table for return codes.
|
// Translation table for return codes.
|
||||||
@ -192,9 +196,11 @@ function zlibOnError(message, errno, code) {
|
|||||||
const FLUSH_BOUND = [
|
const FLUSH_BOUND = [
|
||||||
[ Z_NO_FLUSH, Z_BLOCK ],
|
[ Z_NO_FLUSH, Z_BLOCK ],
|
||||||
[ BROTLI_OPERATION_PROCESS, BROTLI_OPERATION_EMIT_METADATA ],
|
[ BROTLI_OPERATION_PROCESS, BROTLI_OPERATION_EMIT_METADATA ],
|
||||||
|
[ ZSTD_e_continue, ZSTD_e_end ],
|
||||||
];
|
];
|
||||||
const FLUSH_BOUND_IDX_NORMAL = 0;
|
const FLUSH_BOUND_IDX_NORMAL = 0;
|
||||||
const FLUSH_BOUND_IDX_BROTLI = 1;
|
const FLUSH_BOUND_IDX_BROTLI = 1;
|
||||||
|
const FLUSH_BOUND_IDX_ZSTD = 2;
|
||||||
|
|
||||||
// The base class for all Zlib-style streams.
|
// The base class for all Zlib-style streams.
|
||||||
function ZlibBase(opts, mode, handle, { flush, finishFlush, fullFlush }) {
|
function ZlibBase(opts, mode, handle, { flush, finishFlush, fullFlush }) {
|
||||||
@ -203,13 +209,15 @@ function ZlibBase(opts, mode, handle, { flush, finishFlush, fullFlush }) {
|
|||||||
// The ZlibBase class is not exported to user land, the mode should only be
|
// The ZlibBase class is not exported to user land, the mode should only be
|
||||||
// passed in by us.
|
// passed in by us.
|
||||||
assert(typeof mode === 'number');
|
assert(typeof mode === 'number');
|
||||||
assert(mode >= DEFLATE && mode <= BROTLI_ENCODE);
|
assert(mode >= DEFLATE && mode <= ZSTD_DECOMPRESS);
|
||||||
|
|
||||||
let flushBoundIdx;
|
let flushBoundIdx;
|
||||||
if (mode !== BROTLI_ENCODE && mode !== BROTLI_DECODE) {
|
if (mode === BROTLI_ENCODE || mode === BROTLI_DECODE) {
|
||||||
flushBoundIdx = FLUSH_BOUND_IDX_NORMAL;
|
|
||||||
} else {
|
|
||||||
flushBoundIdx = FLUSH_BOUND_IDX_BROTLI;
|
flushBoundIdx = FLUSH_BOUND_IDX_BROTLI;
|
||||||
|
} else if (mode === ZSTD_COMPRESS || mode === ZSTD_DECOMPRESS) {
|
||||||
|
flushBoundIdx = FLUSH_BOUND_IDX_ZSTD;
|
||||||
|
} else {
|
||||||
|
flushBoundIdx = FLUSH_BOUND_IDX_NORMAL;
|
||||||
}
|
}
|
||||||
|
|
||||||
if (opts) {
|
if (opts) {
|
||||||
@ -829,6 +837,89 @@ ObjectSetPrototypeOf(BrotliDecompress.prototype, Brotli.prototype);
|
|||||||
ObjectSetPrototypeOf(BrotliDecompress, Brotli);
|
ObjectSetPrototypeOf(BrotliDecompress, Brotli);
|
||||||
|
|
||||||
|
|
||||||
|
const zstdDefaultOpts = {
|
||||||
|
flush: ZSTD_e_continue,
|
||||||
|
finishFlush: ZSTD_e_end,
|
||||||
|
fullFlush: ZSTD_e_flush,
|
||||||
|
};
|
||||||
|
function Zstd(opts, mode, initParamsArray, maxParam) {
|
||||||
|
assert(mode === ZSTD_COMPRESS || mode === ZSTD_DECOMPRESS);
|
||||||
|
|
||||||
|
initParamsArray.fill(-1);
|
||||||
|
if (opts?.params) {
|
||||||
|
ObjectKeys(opts.params).forEach((origKey) => {
|
||||||
|
const key = +origKey;
|
||||||
|
if (NumberIsNaN(key) || key < 0 || key > maxParam ||
|
||||||
|
(initParamsArray[key] | 0) !== -1) {
|
||||||
|
throw new ERR_ZSTD_INVALID_PARAM(origKey);
|
||||||
|
}
|
||||||
|
|
||||||
|
const value = opts.params[origKey];
|
||||||
|
if (typeof value !== 'number' && typeof value !== 'boolean') {
|
||||||
|
throw new ERR_INVALID_ARG_TYPE('options.params[key]',
|
||||||
|
'number', opts.params[origKey]);
|
||||||
|
}
|
||||||
|
initParamsArray[key] = value;
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
const handle = mode === ZSTD_COMPRESS ?
|
||||||
|
new binding.ZstdCompress() : new binding.ZstdDecompress();
|
||||||
|
|
||||||
|
const pledgedSrcSize = opts?.pledgedSrcSize ?? undefined;
|
||||||
|
|
||||||
|
this._writeState = new Uint32Array(2);
|
||||||
|
handle.init(
|
||||||
|
initParamsArray,
|
||||||
|
pledgedSrcSize,
|
||||||
|
this._writeState,
|
||||||
|
processCallback,
|
||||||
|
);
|
||||||
|
|
||||||
|
ReflectApply(ZlibBase, this, [opts, mode, handle, zstdDefaultOpts]);
|
||||||
|
}
|
||||||
|
ObjectSetPrototypeOf(Zstd.prototype, ZlibBase.prototype);
|
||||||
|
ObjectSetPrototypeOf(Zstd, ZlibBase);
|
||||||
|
|
||||||
|
|
||||||
|
const kMaxZstdCParam = MathMax(...ObjectKeys(constants).map(
|
||||||
|
(key) => (key.startsWith('ZSTD_c_') ?
|
||||||
|
constants[key] :
|
||||||
|
0),
|
||||||
|
));
|
||||||
|
|
||||||
|
const zstdInitCParamsArray = new Uint32Array(kMaxZstdCParam + 1);
|
||||||
|
|
||||||
|
function ZstdCompress(opts) {
|
||||||
|
if (!(this instanceof ZstdCompress))
|
||||||
|
return new ZstdCompress(opts);
|
||||||
|
|
||||||
|
ReflectApply(Zstd, this,
|
||||||
|
[opts, ZSTD_COMPRESS, zstdInitCParamsArray, kMaxZstdCParam]);
|
||||||
|
}
|
||||||
|
ObjectSetPrototypeOf(ZstdCompress.prototype, Zstd.prototype);
|
||||||
|
ObjectSetPrototypeOf(ZstdCompress, Zstd);
|
||||||
|
|
||||||
|
|
||||||
|
const kMaxZstdDParam = MathMax(...ObjectKeys(constants).map(
|
||||||
|
(key) => (key.startsWith('ZSTD_d_') ?
|
||||||
|
constants[key] :
|
||||||
|
0),
|
||||||
|
));
|
||||||
|
|
||||||
|
const zstdInitDParamsArray = new Uint32Array(kMaxZstdDParam + 1);
|
||||||
|
|
||||||
|
function ZstdDecompress(opts) {
|
||||||
|
if (!(this instanceof ZstdDecompress))
|
||||||
|
return new ZstdDecompress(opts);
|
||||||
|
|
||||||
|
ReflectApply(Zstd, this,
|
||||||
|
[opts, ZSTD_DECOMPRESS, zstdInitDParamsArray, kMaxZstdDParam]);
|
||||||
|
}
|
||||||
|
ObjectSetPrototypeOf(ZstdDecompress.prototype, Zstd.prototype);
|
||||||
|
ObjectSetPrototypeOf(ZstdDecompress, Zstd);
|
||||||
|
|
||||||
|
|
||||||
function createProperty(ctor) {
|
function createProperty(ctor) {
|
||||||
return {
|
return {
|
||||||
__proto__: null,
|
__proto__: null,
|
||||||
@ -867,6 +958,8 @@ module.exports = {
|
|||||||
Unzip,
|
Unzip,
|
||||||
BrotliCompress,
|
BrotliCompress,
|
||||||
BrotliDecompress,
|
BrotliDecompress,
|
||||||
|
ZstdCompress,
|
||||||
|
ZstdDecompress,
|
||||||
|
|
||||||
// Convenience methods.
|
// Convenience methods.
|
||||||
// compress/decompress a string or buffer in one step.
|
// compress/decompress a string or buffer in one step.
|
||||||
@ -888,6 +981,10 @@ module.exports = {
|
|||||||
brotliCompressSync: createConvenienceMethod(BrotliCompress, true),
|
brotliCompressSync: createConvenienceMethod(BrotliCompress, true),
|
||||||
brotliDecompress: createConvenienceMethod(BrotliDecompress, false),
|
brotliDecompress: createConvenienceMethod(BrotliDecompress, false),
|
||||||
brotliDecompressSync: createConvenienceMethod(BrotliDecompress, true),
|
brotliDecompressSync: createConvenienceMethod(BrotliDecompress, true),
|
||||||
|
zstdCompress: createConvenienceMethod(ZstdCompress, false),
|
||||||
|
zstdCompressSync: createConvenienceMethod(ZstdCompress, true),
|
||||||
|
zstdDecompress: createConvenienceMethod(ZstdDecompress, false),
|
||||||
|
zstdDecompressSync: createConvenienceMethod(ZstdDecompress, true),
|
||||||
};
|
};
|
||||||
|
|
||||||
ObjectDefineProperties(module.exports, {
|
ObjectDefineProperties(module.exports, {
|
||||||
@ -900,6 +997,8 @@ ObjectDefineProperties(module.exports, {
|
|||||||
createUnzip: createProperty(Unzip),
|
createUnzip: createProperty(Unzip),
|
||||||
createBrotliCompress: createProperty(BrotliCompress),
|
createBrotliCompress: createProperty(BrotliCompress),
|
||||||
createBrotliDecompress: createProperty(BrotliDecompress),
|
createBrotliDecompress: createProperty(BrotliDecompress),
|
||||||
|
createZstdCompress: createProperty(ZstdCompress),
|
||||||
|
createZstdDecompress: createProperty(ZstdDecompress),
|
||||||
constants: {
|
constants: {
|
||||||
__proto__: null,
|
__proto__: null,
|
||||||
configurable: false,
|
configurable: false,
|
||||||
|
384
src/node_zlib.cc
384
src/node_zlib.cc
@ -32,9 +32,11 @@
|
|||||||
|
|
||||||
#include "v8.h"
|
#include "v8.h"
|
||||||
|
|
||||||
#include "brotli/encode.h"
|
|
||||||
#include "brotli/decode.h"
|
#include "brotli/decode.h"
|
||||||
|
#include "brotli/encode.h"
|
||||||
#include "zlib.h"
|
#include "zlib.h"
|
||||||
|
#include "zstd.h"
|
||||||
|
#include "zstd_errors.h"
|
||||||
|
|
||||||
#include <sys/types.h>
|
#include <sys/types.h>
|
||||||
|
|
||||||
@ -95,6 +97,44 @@ inline const char* ZlibStrerror(int err) {
|
|||||||
return "Z_UNKNOWN_ERROR";
|
return "Z_UNKNOWN_ERROR";
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#define ZSTD_ERROR_CODES(V) \
|
||||||
|
V(ZSTD_error_no_error) \
|
||||||
|
V(ZSTD_error_GENERIC) \
|
||||||
|
V(ZSTD_error_prefix_unknown) \
|
||||||
|
V(ZSTD_error_version_unsupported) \
|
||||||
|
V(ZSTD_error_frameParameter_unsupported) \
|
||||||
|
V(ZSTD_error_frameParameter_windowTooLarge) \
|
||||||
|
V(ZSTD_error_corruption_detected) \
|
||||||
|
V(ZSTD_error_checksum_wrong) \
|
||||||
|
V(ZSTD_error_literals_headerWrong) \
|
||||||
|
V(ZSTD_error_dictionary_corrupted) \
|
||||||
|
V(ZSTD_error_dictionary_wrong) \
|
||||||
|
V(ZSTD_error_dictionaryCreation_failed) \
|
||||||
|
V(ZSTD_error_parameter_unsupported) \
|
||||||
|
V(ZSTD_error_parameter_combination_unsupported) \
|
||||||
|
V(ZSTD_error_parameter_outOfBound) \
|
||||||
|
V(ZSTD_error_tableLog_tooLarge) \
|
||||||
|
V(ZSTD_error_maxSymbolValue_tooLarge) \
|
||||||
|
V(ZSTD_error_maxSymbolValue_tooSmall) \
|
||||||
|
V(ZSTD_error_stabilityCondition_notRespected) \
|
||||||
|
V(ZSTD_error_stage_wrong) \
|
||||||
|
V(ZSTD_error_init_missing) \
|
||||||
|
V(ZSTD_error_memory_allocation) \
|
||||||
|
V(ZSTD_error_workSpace_tooSmall) \
|
||||||
|
V(ZSTD_error_dstSize_tooSmall) \
|
||||||
|
V(ZSTD_error_srcSize_wrong) \
|
||||||
|
V(ZSTD_error_dstBuffer_null) \
|
||||||
|
V(ZSTD_error_noForwardProgress_destFull) \
|
||||||
|
V(ZSTD_error_noForwardProgress_inputEmpty)
|
||||||
|
|
||||||
|
inline const char* ZstdStrerror(int err) {
|
||||||
|
#define V(code) \
|
||||||
|
if (err == code) return #code;
|
||||||
|
ZSTD_ERROR_CODES(V)
|
||||||
|
#undef V
|
||||||
|
return "ZSTD_error_GENERIC";
|
||||||
|
}
|
||||||
|
|
||||||
enum node_zlib_mode {
|
enum node_zlib_mode {
|
||||||
NONE,
|
NONE,
|
||||||
DEFLATE,
|
DEFLATE,
|
||||||
@ -105,7 +145,9 @@ enum node_zlib_mode {
|
|||||||
INFLATERAW,
|
INFLATERAW,
|
||||||
UNZIP,
|
UNZIP,
|
||||||
BROTLI_DECODE,
|
BROTLI_DECODE,
|
||||||
BROTLI_ENCODE
|
BROTLI_ENCODE,
|
||||||
|
ZSTD_COMPRESS,
|
||||||
|
ZSTD_DECOMPRESS
|
||||||
};
|
};
|
||||||
|
|
||||||
constexpr uint8_t GZIP_HEADER_ID1 = 0x1f;
|
constexpr uint8_t GZIP_HEADER_ID1 = 0x1f;
|
||||||
@ -249,6 +291,79 @@ class BrotliDecoderContext final : public BrotliContext {
|
|||||||
DeleteFnPtr<BrotliDecoderState, BrotliDecoderDestroyInstance> state_;
|
DeleteFnPtr<BrotliDecoderState, BrotliDecoderDestroyInstance> state_;
|
||||||
};
|
};
|
||||||
|
|
||||||
|
class ZstdContext : public MemoryRetainer {
|
||||||
|
public:
|
||||||
|
ZstdContext() = default;
|
||||||
|
|
||||||
|
// Streaming-related, should be available for all compression libraries:
|
||||||
|
void Close();
|
||||||
|
void SetBuffers(const char* in, uint32_t in_len, char* out, uint32_t out_len);
|
||||||
|
void SetFlush(int flush);
|
||||||
|
void GetAfterWriteOffsets(uint32_t* avail_in, uint32_t* avail_out) const;
|
||||||
|
CompressionError GetErrorInfo() const;
|
||||||
|
|
||||||
|
ZstdContext(const ZstdContext&) = delete;
|
||||||
|
ZstdContext& operator=(const ZstdContext&) = delete;
|
||||||
|
|
||||||
|
protected:
|
||||||
|
ZSTD_EndDirective flush_ = ZSTD_e_continue;
|
||||||
|
|
||||||
|
ZSTD_inBuffer input_ = {nullptr, 0, 0};
|
||||||
|
ZSTD_outBuffer output_ = {nullptr, 0, 0};
|
||||||
|
|
||||||
|
ZSTD_ErrorCode error_ = ZSTD_error_no_error;
|
||||||
|
std::string error_string_;
|
||||||
|
std::string error_code_string_;
|
||||||
|
};
|
||||||
|
|
||||||
|
class ZstdCompressContext final : public ZstdContext {
|
||||||
|
public:
|
||||||
|
ZstdCompressContext() = default;
|
||||||
|
|
||||||
|
// Streaming-related, should be available for all compression libraries:
|
||||||
|
void DoThreadPoolWork();
|
||||||
|
CompressionError ResetStream();
|
||||||
|
|
||||||
|
// Zstd specific:
|
||||||
|
CompressionError Init(uint64_t pledged_src_size);
|
||||||
|
CompressionError SetParameter(int key, int value);
|
||||||
|
|
||||||
|
// Wrap ZSTD_freeCCtx to remove the return type.
|
||||||
|
static void FreeZstd(ZSTD_CCtx* cctx) { ZSTD_freeCCtx(cctx); }
|
||||||
|
|
||||||
|
SET_MEMORY_INFO_NAME(ZstdCompressContext)
|
||||||
|
SET_SELF_SIZE(ZstdCompressContext)
|
||||||
|
SET_NO_MEMORY_INFO()
|
||||||
|
|
||||||
|
private:
|
||||||
|
DeleteFnPtr<ZSTD_CCtx, ZstdCompressContext::FreeZstd> cctx_;
|
||||||
|
|
||||||
|
uint64_t pledged_src_size_ = ZSTD_CONTENTSIZE_UNKNOWN;
|
||||||
|
};
|
||||||
|
|
||||||
|
class ZstdDecompressContext final : public ZstdContext {
|
||||||
|
public:
|
||||||
|
ZstdDecompressContext() = default;
|
||||||
|
|
||||||
|
// Streaming-related, should be available for all compression libraries:
|
||||||
|
void DoThreadPoolWork();
|
||||||
|
CompressionError ResetStream();
|
||||||
|
|
||||||
|
// Zstd specific:
|
||||||
|
CompressionError Init(uint64_t pledged_src_size);
|
||||||
|
CompressionError SetParameter(int key, int value);
|
||||||
|
|
||||||
|
// Wrap ZSTD_freeDCtx to remove the return type.
|
||||||
|
static void FreeZstd(ZSTD_DCtx* dctx) { ZSTD_freeDCtx(dctx); }
|
||||||
|
|
||||||
|
SET_MEMORY_INFO_NAME(ZstdDecompressContext)
|
||||||
|
SET_SELF_SIZE(ZstdDecompressContext)
|
||||||
|
SET_NO_MEMORY_INFO()
|
||||||
|
|
||||||
|
private:
|
||||||
|
DeleteFnPtr<ZSTD_DCtx, ZstdDecompressContext::FreeZstd> dctx_;
|
||||||
|
};
|
||||||
|
|
||||||
template <typename CompressionContext>
|
template <typename CompressionContext>
|
||||||
class CompressionStream : public AsyncWrap, public ThreadPoolWork {
|
class CompressionStream : public AsyncWrap, public ThreadPoolWork {
|
||||||
public:
|
public:
|
||||||
@ -740,6 +855,92 @@ class BrotliCompressionStream final :
|
|||||||
using BrotliEncoderStream = BrotliCompressionStream<BrotliEncoderContext>;
|
using BrotliEncoderStream = BrotliCompressionStream<BrotliEncoderContext>;
|
||||||
using BrotliDecoderStream = BrotliCompressionStream<BrotliDecoderContext>;
|
using BrotliDecoderStream = BrotliCompressionStream<BrotliDecoderContext>;
|
||||||
|
|
||||||
|
template <typename CompressionContext>
|
||||||
|
class ZstdStream final : public CompressionStream<CompressionContext> {
|
||||||
|
public:
|
||||||
|
ZstdStream(Environment* env, Local<Object> wrap)
|
||||||
|
: CompressionStream<CompressionContext>(env, wrap) {}
|
||||||
|
|
||||||
|
inline CompressionContext* context() {
|
||||||
|
return this->CompressionStream<CompressionContext>::context();
|
||||||
|
}
|
||||||
|
typedef typename CompressionStream<CompressionContext>::AllocScope AllocScope;
|
||||||
|
|
||||||
|
static void New(const FunctionCallbackInfo<Value>& args) {
|
||||||
|
Environment* env = Environment::GetCurrent(args);
|
||||||
|
new ZstdStream(env, args.This());
|
||||||
|
}
|
||||||
|
|
||||||
|
static void Init(const FunctionCallbackInfo<Value>& args) {
|
||||||
|
Environment* env = Environment::GetCurrent(args);
|
||||||
|
Local<Context> context = env->context();
|
||||||
|
|
||||||
|
CHECK(args.Length() == 4 &&
|
||||||
|
"init(params, pledgedSrcSize, writeResult, writeCallback)");
|
||||||
|
ZstdStream* wrap;
|
||||||
|
ASSIGN_OR_RETURN_UNWRAP(&wrap, args.Holder());
|
||||||
|
|
||||||
|
CHECK(args[2]->IsUint32Array());
|
||||||
|
uint32_t* write_result = reinterpret_cast<uint32_t*>(Buffer::Data(args[2]));
|
||||||
|
|
||||||
|
CHECK(args[3]->IsFunction());
|
||||||
|
Local<Function> write_js_callback = args[3].As<Function>();
|
||||||
|
wrap->InitStream(write_result, write_js_callback);
|
||||||
|
|
||||||
|
uint64_t pledged_src_size = ZSTD_CONTENTSIZE_UNKNOWN;
|
||||||
|
if (args[1]->IsNumber()) {
|
||||||
|
int64_t signed_pledged_src_size;
|
||||||
|
if (!args[1]->IntegerValue(context).To(&signed_pledged_src_size)) {
|
||||||
|
THROW_ERR_INVALID_ARG_VALUE(wrap->env(),
|
||||||
|
"pledgedSrcSize should be an integer");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if (signed_pledged_src_size < 0) {
|
||||||
|
THROW_ERR_INVALID_ARG_VALUE(wrap->env(),
|
||||||
|
"pledgedSrcSize may not be negative");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
pledged_src_size = signed_pledged_src_size;
|
||||||
|
}
|
||||||
|
|
||||||
|
AllocScope alloc_scope(wrap);
|
||||||
|
CompressionError err = wrap->context()->Init(pledged_src_size);
|
||||||
|
if (err.IsError()) {
|
||||||
|
wrap->EmitError(err);
|
||||||
|
THROW_ERR_ZLIB_INITIALIZATION_FAILED(wrap->env(), err.message);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
CHECK(args[0]->IsUint32Array());
|
||||||
|
const uint32_t* data = reinterpret_cast<uint32_t*>(Buffer::Data(args[0]));
|
||||||
|
size_t len = args[0].As<Uint32Array>()->Length();
|
||||||
|
|
||||||
|
for (int i = 0; static_cast<size_t>(i) < len; i++) {
|
||||||
|
if (data[i] == static_cast<uint32_t>(-1)) continue;
|
||||||
|
|
||||||
|
CompressionError err = wrap->context()->SetParameter(i, data[i]);
|
||||||
|
if (err.IsError()) {
|
||||||
|
wrap->EmitError(err);
|
||||||
|
THROW_ERR_ZLIB_INITIALIZATION_FAILED(wrap->env(), err.message);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
static void Params(const FunctionCallbackInfo<Value>& args) {
|
||||||
|
// Currently a no-op, and not accessed from JS land.
|
||||||
|
// At some point zstd may support changing parameters on the fly,
|
||||||
|
// in which case we can implement this and a JS equivalent similar to
|
||||||
|
// the zlib Params() function.
|
||||||
|
}
|
||||||
|
|
||||||
|
SET_MEMORY_INFO_NAME(ZstdStream)
|
||||||
|
SET_SELF_SIZE(ZstdStream)
|
||||||
|
};
|
||||||
|
|
||||||
|
using ZstdCompressStream = ZstdStream<ZstdCompressContext>;
|
||||||
|
using ZstdDecompressStream = ZstdStream<ZstdDecompressContext>;
|
||||||
|
|
||||||
void ZlibContext::Close() {
|
void ZlibContext::Close() {
|
||||||
{
|
{
|
||||||
Mutex::ScopedLock lock(mutex_);
|
Mutex::ScopedLock lock(mutex_);
|
||||||
@ -1263,6 +1464,115 @@ CompressionError BrotliDecoderContext::GetErrorInfo() const {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
void ZstdContext::Close() {}
|
||||||
|
|
||||||
|
void ZstdContext::SetBuffers(const char* in,
|
||||||
|
uint32_t in_len,
|
||||||
|
char* out,
|
||||||
|
uint32_t out_len) {
|
||||||
|
input_.src = reinterpret_cast<const uint8_t*>(in);
|
||||||
|
input_.size = in_len;
|
||||||
|
input_.pos = 0;
|
||||||
|
|
||||||
|
output_.dst = reinterpret_cast<uint8_t*>(out);
|
||||||
|
output_.size = out_len;
|
||||||
|
output_.pos = 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
void ZstdContext::SetFlush(int flush) {
|
||||||
|
flush_ = static_cast<ZSTD_EndDirective>(flush);
|
||||||
|
}
|
||||||
|
|
||||||
|
void ZstdContext::GetAfterWriteOffsets(uint32_t* avail_in,
|
||||||
|
uint32_t* avail_out) const {
|
||||||
|
*avail_in = input_.size - input_.pos;
|
||||||
|
*avail_out = output_.size - output_.pos;
|
||||||
|
}
|
||||||
|
|
||||||
|
CompressionError ZstdContext::GetErrorInfo() const {
|
||||||
|
if (error_ != ZSTD_error_no_error) {
|
||||||
|
return CompressionError(error_string_.c_str(),
|
||||||
|
error_code_string_.c_str(),
|
||||||
|
static_cast<int>(error_));
|
||||||
|
} else {
|
||||||
|
return {};
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
CompressionError ZstdCompressContext::SetParameter(int key, int value) {
|
||||||
|
size_t result = ZSTD_CCtx_setParameter(
|
||||||
|
cctx_.get(), static_cast<ZSTD_cParameter>(key), value);
|
||||||
|
if (ZSTD_isError(result)) {
|
||||||
|
return CompressionError(
|
||||||
|
"Setting parameter failed", "ERR_ZSTD_PARAM_SET_FAILED", -1);
|
||||||
|
}
|
||||||
|
return {};
|
||||||
|
}
|
||||||
|
|
||||||
|
CompressionError ZstdCompressContext::Init(uint64_t pledged_src_size) {
|
||||||
|
pledged_src_size_ = pledged_src_size;
|
||||||
|
cctx_.reset(ZSTD_createCCtx());
|
||||||
|
if (!cctx_) {
|
||||||
|
return CompressionError("Could not initialize zstd instance",
|
||||||
|
"ERR_ZLIB_INITIALIZATION_FAILED",
|
||||||
|
-1);
|
||||||
|
}
|
||||||
|
size_t result = ZSTD_CCtx_setPledgedSrcSize(cctx_.get(), pledged_src_size);
|
||||||
|
if (ZSTD_isError(result)) {
|
||||||
|
return CompressionError(
|
||||||
|
"Could not set pledged src size", "ERR_ZLIB_INITIALIZATION_FAILED", -1);
|
||||||
|
}
|
||||||
|
return {};
|
||||||
|
}
|
||||||
|
|
||||||
|
CompressionError ZstdCompressContext::ResetStream() {
|
||||||
|
return Init(pledged_src_size_);
|
||||||
|
}
|
||||||
|
|
||||||
|
void ZstdCompressContext::DoThreadPoolWork() {
|
||||||
|
size_t const remaining =
|
||||||
|
ZSTD_compressStream2(cctx_.get(), &output_, &input_, flush_);
|
||||||
|
if (ZSTD_isError(remaining)) {
|
||||||
|
error_ = ZSTD_getErrorCode(remaining);
|
||||||
|
error_code_string_ = ZstdStrerror(error_);
|
||||||
|
error_string_ = ZSTD_getErrorString(error_);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
CompressionError ZstdDecompressContext::SetParameter(int key, int value) {
|
||||||
|
size_t result = ZSTD_DCtx_setParameter(
|
||||||
|
dctx_.get(), static_cast<ZSTD_dParameter>(key), value);
|
||||||
|
if (ZSTD_isError(result)) {
|
||||||
|
return CompressionError(
|
||||||
|
"Setting parameter failed", "ERR_ZSTD_PARAM_SET_FAILED", -1);
|
||||||
|
}
|
||||||
|
return {};
|
||||||
|
}
|
||||||
|
|
||||||
|
CompressionError ZstdDecompressContext::Init(uint64_t pledged_src_size) {
|
||||||
|
dctx_.reset(ZSTD_createDCtx());
|
||||||
|
if (!dctx_) {
|
||||||
|
return CompressionError("Could not initialize zstd instance",
|
||||||
|
"ERR_ZLIB_INITIALIZATION_FAILED",
|
||||||
|
-1);
|
||||||
|
}
|
||||||
|
return {};
|
||||||
|
}
|
||||||
|
|
||||||
|
CompressionError ZstdDecompressContext::ResetStream() {
|
||||||
|
// We pass ZSTD_CONTENTSIZE_UNKNOWN because the argument is ignored for
|
||||||
|
// decompression.
|
||||||
|
return Init(ZSTD_CONTENTSIZE_UNKNOWN);
|
||||||
|
}
|
||||||
|
|
||||||
|
void ZstdDecompressContext::DoThreadPoolWork() {
|
||||||
|
size_t const ret = ZSTD_decompressStream(dctx_.get(), &output_, &input_);
|
||||||
|
if (ZSTD_isError(ret)) {
|
||||||
|
error_ = ZSTD_getErrorCode(ret);
|
||||||
|
error_code_string_ = ZstdStrerror(error_);
|
||||||
|
error_string_ = ZSTD_getErrorString(error_);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
template <typename Stream>
|
template <typename Stream>
|
||||||
struct MakeClass {
|
struct MakeClass {
|
||||||
@ -1332,6 +1642,8 @@ void Initialize(Local<Object> target,
|
|||||||
MakeClass<ZlibStream>::Make(env, target, "Zlib");
|
MakeClass<ZlibStream>::Make(env, target, "Zlib");
|
||||||
MakeClass<BrotliEncoderStream>::Make(env, target, "BrotliEncoder");
|
MakeClass<BrotliEncoderStream>::Make(env, target, "BrotliEncoder");
|
||||||
MakeClass<BrotliDecoderStream>::Make(env, target, "BrotliDecoder");
|
MakeClass<BrotliDecoderStream>::Make(env, target, "BrotliDecoder");
|
||||||
|
MakeClass<ZstdCompressStream>::Make(env, target, "ZstdCompress");
|
||||||
|
MakeClass<ZstdDecompressStream>::Make(env, target, "ZstdDecompress");
|
||||||
|
|
||||||
SetMethod(context, target, "crc32", CRC32);
|
SetMethod(context, target, "crc32", CRC32);
|
||||||
target->Set(env->context(),
|
target->Set(env->context(),
|
||||||
@ -1343,6 +1655,8 @@ void RegisterExternalReferences(ExternalReferenceRegistry* registry) {
|
|||||||
MakeClass<ZlibStream>::Make(registry);
|
MakeClass<ZlibStream>::Make(registry);
|
||||||
MakeClass<BrotliEncoderStream>::Make(registry);
|
MakeClass<BrotliEncoderStream>::Make(registry);
|
||||||
MakeClass<BrotliDecoderStream>::Make(registry);
|
MakeClass<BrotliDecoderStream>::Make(registry);
|
||||||
|
MakeClass<ZstdCompressStream>::Make(registry);
|
||||||
|
MakeClass<ZstdDecompressStream>::Make(registry);
|
||||||
registry->Register(CRC32);
|
registry->Register(CRC32);
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -1387,6 +1701,8 @@ void DefineZlibConstants(Local<Object> target) {
|
|||||||
NODE_DEFINE_CONSTANT(target, UNZIP);
|
NODE_DEFINE_CONSTANT(target, UNZIP);
|
||||||
NODE_DEFINE_CONSTANT(target, BROTLI_DECODE);
|
NODE_DEFINE_CONSTANT(target, BROTLI_DECODE);
|
||||||
NODE_DEFINE_CONSTANT(target, BROTLI_ENCODE);
|
NODE_DEFINE_CONSTANT(target, BROTLI_ENCODE);
|
||||||
|
NODE_DEFINE_CONSTANT(target, ZSTD_DECOMPRESS);
|
||||||
|
NODE_DEFINE_CONSTANT(target, ZSTD_COMPRESS);
|
||||||
|
|
||||||
NODE_DEFINE_CONSTANT(target, Z_MIN_WINDOWBITS);
|
NODE_DEFINE_CONSTANT(target, Z_MIN_WINDOWBITS);
|
||||||
NODE_DEFINE_CONSTANT(target, Z_MAX_WINDOWBITS);
|
NODE_DEFINE_CONSTANT(target, Z_MAX_WINDOWBITS);
|
||||||
@ -1466,6 +1782,70 @@ void DefineZlibConstants(Local<Object> target) {
|
|||||||
NODE_DEFINE_CONSTANT(target, BROTLI_DECODER_ERROR_ALLOC_RING_BUFFER_2);
|
NODE_DEFINE_CONSTANT(target, BROTLI_DECODER_ERROR_ALLOC_RING_BUFFER_2);
|
||||||
NODE_DEFINE_CONSTANT(target, BROTLI_DECODER_ERROR_ALLOC_BLOCK_TYPE_TREES);
|
NODE_DEFINE_CONSTANT(target, BROTLI_DECODER_ERROR_ALLOC_BLOCK_TYPE_TREES);
|
||||||
NODE_DEFINE_CONSTANT(target, BROTLI_DECODER_ERROR_UNREACHABLE);
|
NODE_DEFINE_CONSTANT(target, BROTLI_DECODER_ERROR_UNREACHABLE);
|
||||||
|
|
||||||
|
// Zstd constants
|
||||||
|
NODE_DEFINE_CONSTANT(target, ZSTD_e_continue);
|
||||||
|
NODE_DEFINE_CONSTANT(target, ZSTD_e_flush);
|
||||||
|
NODE_DEFINE_CONSTANT(target, ZSTD_e_end);
|
||||||
|
NODE_DEFINE_CONSTANT(target, ZSTD_fast);
|
||||||
|
NODE_DEFINE_CONSTANT(target, ZSTD_dfast);
|
||||||
|
NODE_DEFINE_CONSTANT(target, ZSTD_greedy);
|
||||||
|
NODE_DEFINE_CONSTANT(target, ZSTD_lazy);
|
||||||
|
NODE_DEFINE_CONSTANT(target, ZSTD_lazy2);
|
||||||
|
NODE_DEFINE_CONSTANT(target, ZSTD_btlazy2);
|
||||||
|
NODE_DEFINE_CONSTANT(target, ZSTD_btopt);
|
||||||
|
NODE_DEFINE_CONSTANT(target, ZSTD_btultra);
|
||||||
|
NODE_DEFINE_CONSTANT(target, ZSTD_btultra2);
|
||||||
|
NODE_DEFINE_CONSTANT(target, ZSTD_c_compressionLevel);
|
||||||
|
NODE_DEFINE_CONSTANT(target, ZSTD_c_windowLog);
|
||||||
|
NODE_DEFINE_CONSTANT(target, ZSTD_c_hashLog);
|
||||||
|
NODE_DEFINE_CONSTANT(target, ZSTD_c_chainLog);
|
||||||
|
NODE_DEFINE_CONSTANT(target, ZSTD_c_searchLog);
|
||||||
|
NODE_DEFINE_CONSTANT(target, ZSTD_c_minMatch);
|
||||||
|
NODE_DEFINE_CONSTANT(target, ZSTD_c_targetLength);
|
||||||
|
NODE_DEFINE_CONSTANT(target, ZSTD_c_strategy);
|
||||||
|
NODE_DEFINE_CONSTANT(target, ZSTD_c_enableLongDistanceMatching);
|
||||||
|
NODE_DEFINE_CONSTANT(target, ZSTD_c_ldmHashLog);
|
||||||
|
NODE_DEFINE_CONSTANT(target, ZSTD_c_ldmMinMatch);
|
||||||
|
NODE_DEFINE_CONSTANT(target, ZSTD_c_ldmBucketSizeLog);
|
||||||
|
NODE_DEFINE_CONSTANT(target, ZSTD_c_ldmHashRateLog);
|
||||||
|
NODE_DEFINE_CONSTANT(target, ZSTD_c_contentSizeFlag);
|
||||||
|
NODE_DEFINE_CONSTANT(target, ZSTD_c_checksumFlag);
|
||||||
|
NODE_DEFINE_CONSTANT(target, ZSTD_c_dictIDFlag);
|
||||||
|
NODE_DEFINE_CONSTANT(target, ZSTD_c_nbWorkers);
|
||||||
|
NODE_DEFINE_CONSTANT(target, ZSTD_c_jobSize);
|
||||||
|
NODE_DEFINE_CONSTANT(target, ZSTD_c_overlapLog);
|
||||||
|
NODE_DEFINE_CONSTANT(target, ZSTD_d_windowLogMax);
|
||||||
|
NODE_DEFINE_CONSTANT(target, ZSTD_CLEVEL_DEFAULT);
|
||||||
|
// Error codes
|
||||||
|
NODE_DEFINE_CONSTANT(target, ZSTD_error_no_error);
|
||||||
|
NODE_DEFINE_CONSTANT(target, ZSTD_error_GENERIC);
|
||||||
|
NODE_DEFINE_CONSTANT(target, ZSTD_error_prefix_unknown);
|
||||||
|
NODE_DEFINE_CONSTANT(target, ZSTD_error_version_unsupported);
|
||||||
|
NODE_DEFINE_CONSTANT(target, ZSTD_error_frameParameter_unsupported);
|
||||||
|
NODE_DEFINE_CONSTANT(target, ZSTD_error_frameParameter_windowTooLarge);
|
||||||
|
NODE_DEFINE_CONSTANT(target, ZSTD_error_corruption_detected);
|
||||||
|
NODE_DEFINE_CONSTANT(target, ZSTD_error_checksum_wrong);
|
||||||
|
NODE_DEFINE_CONSTANT(target, ZSTD_error_literals_headerWrong);
|
||||||
|
NODE_DEFINE_CONSTANT(target, ZSTD_error_dictionary_corrupted);
|
||||||
|
NODE_DEFINE_CONSTANT(target, ZSTD_error_dictionary_wrong);
|
||||||
|
NODE_DEFINE_CONSTANT(target, ZSTD_error_dictionaryCreation_failed);
|
||||||
|
NODE_DEFINE_CONSTANT(target, ZSTD_error_parameter_unsupported);
|
||||||
|
NODE_DEFINE_CONSTANT(target, ZSTD_error_parameter_combination_unsupported);
|
||||||
|
NODE_DEFINE_CONSTANT(target, ZSTD_error_parameter_outOfBound);
|
||||||
|
NODE_DEFINE_CONSTANT(target, ZSTD_error_tableLog_tooLarge);
|
||||||
|
NODE_DEFINE_CONSTANT(target, ZSTD_error_maxSymbolValue_tooLarge);
|
||||||
|
NODE_DEFINE_CONSTANT(target, ZSTD_error_maxSymbolValue_tooSmall);
|
||||||
|
NODE_DEFINE_CONSTANT(target, ZSTD_error_stabilityCondition_notRespected);
|
||||||
|
NODE_DEFINE_CONSTANT(target, ZSTD_error_stage_wrong);
|
||||||
|
NODE_DEFINE_CONSTANT(target, ZSTD_error_init_missing);
|
||||||
|
NODE_DEFINE_CONSTANT(target, ZSTD_error_memory_allocation);
|
||||||
|
NODE_DEFINE_CONSTANT(target, ZSTD_error_workSpace_tooSmall);
|
||||||
|
NODE_DEFINE_CONSTANT(target, ZSTD_error_dstSize_tooSmall);
|
||||||
|
NODE_DEFINE_CONSTANT(target, ZSTD_error_srcSize_wrong);
|
||||||
|
NODE_DEFINE_CONSTANT(target, ZSTD_error_dstBuffer_null);
|
||||||
|
NODE_DEFINE_CONSTANT(target, ZSTD_error_noForwardProgress_destFull);
|
||||||
|
NODE_DEFINE_CONSTANT(target, ZSTD_error_noForwardProgress_inputEmpty);
|
||||||
}
|
}
|
||||||
|
|
||||||
} // namespace node
|
} // namespace node
|
||||||
|
BIN
test/fixtures/person.jpg.zst
vendored
Normal file
BIN
test/fixtures/person.jpg.zst
vendored
Normal file
Binary file not shown.
@ -54,6 +54,8 @@ for (const [type, expect] of [
|
|||||||
['deflateRaw', 'inflateRaw', 'DeflateRaw', 'InflateRaw'],
|
['deflateRaw', 'inflateRaw', 'DeflateRaw', 'InflateRaw'],
|
||||||
['brotliCompress', 'brotliDecompress',
|
['brotliCompress', 'brotliDecompress',
|
||||||
'BrotliCompress', 'BrotliDecompress'],
|
'BrotliCompress', 'BrotliDecompress'],
|
||||||
|
['zstdCompress', 'zstdDecompress',
|
||||||
|
'ZstdCompress', 'ZstdDecompress'],
|
||||||
]) {
|
]) {
|
||||||
zlib[method[0]](expect, opts, common.mustCall((err, result) => {
|
zlib[method[0]](expect, opts, common.mustCall((err, result) => {
|
||||||
zlib[method[1]](result, opts, common.mustCall((err, result) => {
|
zlib[method[1]](result, opts, common.mustCall((err, result) => {
|
||||||
|
@ -11,10 +11,12 @@ const emptyBuffer = Buffer.alloc(0);
|
|||||||
[ zlib.deflateSync, zlib.inflateSync, 'deflate sync' ],
|
[ zlib.deflateSync, zlib.inflateSync, 'deflate sync' ],
|
||||||
[ zlib.gzipSync, zlib.gunzipSync, 'gzip sync' ],
|
[ zlib.gzipSync, zlib.gunzipSync, 'gzip sync' ],
|
||||||
[ zlib.brotliCompressSync, zlib.brotliDecompressSync, 'br sync' ],
|
[ zlib.brotliCompressSync, zlib.brotliDecompressSync, 'br sync' ],
|
||||||
|
[ zlib.zstdCompressSync, zlib.zstdDecompressSync, 'zstd sync' ],
|
||||||
[ promisify(zlib.deflateRaw), promisify(zlib.inflateRaw), 'raw' ],
|
[ promisify(zlib.deflateRaw), promisify(zlib.inflateRaw), 'raw' ],
|
||||||
[ promisify(zlib.deflate), promisify(zlib.inflate), 'deflate' ],
|
[ promisify(zlib.deflate), promisify(zlib.inflate), 'deflate' ],
|
||||||
[ promisify(zlib.gzip), promisify(zlib.gunzip), 'gzip' ],
|
[ promisify(zlib.gzip), promisify(zlib.gunzip), 'gzip' ],
|
||||||
[ promisify(zlib.brotliCompress), promisify(zlib.brotliDecompress), 'br' ],
|
[ promisify(zlib.brotliCompress), promisify(zlib.brotliDecompress), 'br' ],
|
||||||
|
[ promisify(zlib.zstdCompress), promisify(zlib.zstdDecompress), 'zstd' ],
|
||||||
]) {
|
]) {
|
||||||
const compressed = await compress(emptyBuffer);
|
const compressed = await compress(emptyBuffer);
|
||||||
const decompressed = await decompress(compressed);
|
const decompressed = await decompress(compressed);
|
||||||
|
@ -40,6 +40,7 @@ const unzips = [
|
|||||||
zlib.Inflate(),
|
zlib.Inflate(),
|
||||||
zlib.InflateRaw(),
|
zlib.InflateRaw(),
|
||||||
zlib.BrotliDecompress(),
|
zlib.BrotliDecompress(),
|
||||||
|
zlib.ZstdDecompress(),
|
||||||
];
|
];
|
||||||
|
|
||||||
nonStringInputs.forEach(common.mustCall((input) => {
|
nonStringInputs.forEach(common.mustCall((input) => {
|
||||||
|
@ -144,6 +144,7 @@ class HashStream extends Stream {
|
|||||||
for (const [ createCompress, createDecompress ] of [
|
for (const [ createCompress, createDecompress ] of [
|
||||||
[ zlib.createGzip, zlib.createGunzip ],
|
[ zlib.createGzip, zlib.createGunzip ],
|
||||||
[ zlib.createBrotliCompress, zlib.createBrotliDecompress ],
|
[ zlib.createBrotliCompress, zlib.createBrotliDecompress ],
|
||||||
|
[ zlib.createZstdCompress, zlib.createZstdDecompress ],
|
||||||
]) {
|
]) {
|
||||||
const inp = new RandomReadStream({ total: 1024, block: 256, jitter: 16 });
|
const inp = new RandomReadStream({ total: 1024, block: 256, jitter: 16 });
|
||||||
const out = new HashStream();
|
const out = new HashStream();
|
||||||
|
@ -31,6 +31,7 @@ test('zlib should accept writing after flush', async () => {
|
|||||||
for (const [ createCompress, createDecompress ] of [
|
for (const [ createCompress, createDecompress ] of [
|
||||||
[ zlib.createGzip, zlib.createGunzip ],
|
[ zlib.createGzip, zlib.createGunzip ],
|
||||||
[ zlib.createBrotliCompress, zlib.createBrotliDecompress ],
|
[ zlib.createBrotliCompress, zlib.createBrotliDecompress ],
|
||||||
|
[ zlib.createZstdCompress, zlib.createZstdDecompress ],
|
||||||
]) {
|
]) {
|
||||||
const { promise, resolve, reject } = Promise.withResolvers();
|
const { promise, resolve, reject } = Promise.withResolvers();
|
||||||
const gzip = createCompress();
|
const gzip = createCompress();
|
||||||
|
@ -28,7 +28,13 @@ const zlib = require('node:zlib');
|
|||||||
const { test } = require('node:test');
|
const { test } = require('node:test');
|
||||||
|
|
||||||
test('zlib should properly handle zero byte input', async () => {
|
test('zlib should properly handle zero byte input', async () => {
|
||||||
for (const Compressor of [zlib.Gzip, zlib.BrotliCompress]) {
|
const compressors = [
|
||||||
|
[zlib.Gzip, 20],
|
||||||
|
[zlib.BrotliCompress, 1],
|
||||||
|
[zlib.ZstdCompress, 9],
|
||||||
|
];
|
||||||
|
|
||||||
|
for (const [Compressor, expected] of compressors) {
|
||||||
const { promise, resolve, reject } = Promise.withResolvers();
|
const { promise, resolve, reject } = Promise.withResolvers();
|
||||||
const gz = Compressor();
|
const gz = Compressor();
|
||||||
const emptyBuffer = Buffer.alloc(0);
|
const emptyBuffer = Buffer.alloc(0);
|
||||||
@ -38,7 +44,6 @@ test('zlib should properly handle zero byte input', async () => {
|
|||||||
});
|
});
|
||||||
gz.on('error', reject);
|
gz.on('error', reject);
|
||||||
gz.on('end', function() {
|
gz.on('end', function() {
|
||||||
const expected = Compressor === zlib.Gzip ? 20 : 1;
|
|
||||||
assert.strictEqual(received, expected,
|
assert.strictEqual(received, expected,
|
||||||
`${received}, ${expected}, ${Compressor.name}`);
|
`${received}, ${expected}, ${Compressor.name}`);
|
||||||
resolve();
|
resolve();
|
||||||
|
28
test/parallel/test-zlib-zstd-flush.js
Normal file
28
test/parallel/test-zlib-zstd-flush.js
Normal file
@ -0,0 +1,28 @@
|
|||||||
|
'use strict';
|
||||||
|
require('../common');
|
||||||
|
const assert = require('assert');
|
||||||
|
const zlib = require('zlib');
|
||||||
|
const fixtures = require('../common/fixtures');
|
||||||
|
|
||||||
|
const file = fixtures.readSync('person.jpg');
|
||||||
|
const chunkSize = 16;
|
||||||
|
const compress = new zlib.ZstdCompress();
|
||||||
|
|
||||||
|
const chunk = file.slice(0, chunkSize);
|
||||||
|
const expectedFull = Buffer.from('KLUv/QBYgAAA/9j/4AAQSkZJRgABAQEASA==', 'base64');
|
||||||
|
let actualFull;
|
||||||
|
|
||||||
|
compress.write(chunk, function() {
|
||||||
|
compress.flush(function() {
|
||||||
|
const bufs = [];
|
||||||
|
let buf;
|
||||||
|
while ((buf = compress.read()) !== null)
|
||||||
|
bufs.push(buf);
|
||||||
|
actualFull = Buffer.concat(bufs);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
process.once('exit', function() {
|
||||||
|
assert.deepStrictEqual(actualFull.toString('base64'), expectedFull.toString('base64'));
|
||||||
|
assert.deepStrictEqual(actualFull, expectedFull);
|
||||||
|
});
|
38
test/parallel/test-zlib-zstd-from-string.js
Normal file
38
test/parallel/test-zlib-zstd-from-string.js
Normal file
@ -0,0 +1,38 @@
|
|||||||
|
'use strict';
|
||||||
|
// Test compressing and uncompressing a string with zstd
|
||||||
|
|
||||||
|
const common = require('../common');
|
||||||
|
const assert = require('assert');
|
||||||
|
const zlib = require('zlib');
|
||||||
|
|
||||||
|
const inputString = 'ΩΩLorem ipsum dolor sit amet, consectetur adipiscing eli' +
|
||||||
|
't. Morbi faucibus, purus at gravida dictum, libero arcu ' +
|
||||||
|
'convallis lacus, in commodo libero metus eu nisi. Nullam' +
|
||||||
|
' commodo, neque nec porta placerat, nisi est fermentum a' +
|
||||||
|
'ugue, vitae gravida tellus sapien sit amet tellus. Aenea' +
|
||||||
|
'n non diam orci. Proin quis elit turpis. Suspendisse non' +
|
||||||
|
' diam ipsum. Suspendisse nec ullamcorper odio. Vestibulu' +
|
||||||
|
'm arcu mi, sodales non suscipit id, ultrices ut massa. S' +
|
||||||
|
'ed ac sem sit amet arcu malesuada fermentum. Nunc sed. ';
|
||||||
|
const compressedString = 'KLUv/QRYRQkA9tc9H6AlhTb/z/7/gbTI3kaWLKnbCtkZu/hXm0j' +
|
||||||
|
'FpNz/VQM2ADMANQBHTuQOpIYzfVv7XGwXrpoIfgXNAB98xW4wV3' +
|
||||||
|
'vnCF2bjcvWZF2wIZ1vr1mSHHvPHU0TgMGBwUFrF0xqReWcWPO8z' +
|
||||||
|
'Ny6wMwFUilN+Lg987Zvs2GSRMy6uYvtovK9Uuhgst6l9FQrXLnA' +
|
||||||
|
'5gpZL7PdI8bO9sDH3tHm73XBzaUK+LjSPNKRmzQ3ZMYEPozdof1' +
|
||||||
|
'2KcZGfIcLa0PTsdkYqhGcAx/E9mWa8EGEeq0Qou2LTmzgg3YJz/' +
|
||||||
|
'21OuXSF+TOd662d60Qyb04xC5dOF4b8JFH8mpHAxAAELu3tg1oa' +
|
||||||
|
'bBEIWaRHdE0l/+0RdEWWIVMAku8TgbiX/4bU+OpLo4UuY1FKDR8' +
|
||||||
|
'RgBc';
|
||||||
|
|
||||||
|
zlib.zstdCompress(inputString, common.mustCall((err, buffer) => {
|
||||||
|
assert(inputString.length > buffer.length);
|
||||||
|
|
||||||
|
zlib.zstdDecompress(buffer, common.mustCall((err, buffer) => {
|
||||||
|
assert.strictEqual(buffer.toString(), inputString);
|
||||||
|
}));
|
||||||
|
}));
|
||||||
|
|
||||||
|
const buffer = Buffer.from(compressedString, 'base64');
|
||||||
|
zlib.zstdDecompress(buffer, common.mustCall((err, buffer) => {
|
||||||
|
assert.strictEqual(buffer.toString(), inputString);
|
||||||
|
}));
|
34
test/parallel/test-zlib-zstd-from-zstd.js
Normal file
34
test/parallel/test-zlib-zstd-from-zstd.js
Normal file
@ -0,0 +1,34 @@
|
|||||||
|
'use strict';
|
||||||
|
// Test unzipping a file that was created with a non-node zstd lib,
|
||||||
|
// piped in as fast as possible.
|
||||||
|
//
|
||||||
|
// The compressed fixture was created using the reference CLI:
|
||||||
|
// $ zstd -19 test/fixtures/person.jpg -o test/fixtures/person.jpg.zst
|
||||||
|
|
||||||
|
const common = require('../common');
|
||||||
|
const assert = require('assert');
|
||||||
|
const zlib = require('zlib');
|
||||||
|
const fixtures = require('../common/fixtures');
|
||||||
|
|
||||||
|
const tmpdir = require('../common/tmpdir');
|
||||||
|
tmpdir.refresh();
|
||||||
|
|
||||||
|
const decompress = new zlib.ZstdDecompress();
|
||||||
|
|
||||||
|
const fs = require('fs');
|
||||||
|
|
||||||
|
const fixture = fixtures.path('person.jpg.zst');
|
||||||
|
const unzippedFixture = fixtures.path('person.jpg');
|
||||||
|
const outputFile = tmpdir.resolve('person.jpg');
|
||||||
|
const expect = fs.readFileSync(unzippedFixture);
|
||||||
|
const inp = fs.createReadStream(fixture);
|
||||||
|
const out = fs.createWriteStream(outputFile);
|
||||||
|
|
||||||
|
inp.pipe(decompress).pipe(out);
|
||||||
|
out.on('close', common.mustCall(() => {
|
||||||
|
const actual = fs.readFileSync(outputFile);
|
||||||
|
assert.strictEqual(actual.length, expect.length);
|
||||||
|
for (let i = 0, l = actual.length; i < l; i++) {
|
||||||
|
assert.strictEqual(actual[i], expect[i], `byte[${i}]`);
|
||||||
|
}
|
||||||
|
}));
|
29
test/parallel/test-zlib-zstd-kmaxlength-rangeerror.js
Normal file
29
test/parallel/test-zlib-zstd-kmaxlength-rangeerror.js
Normal file
@ -0,0 +1,29 @@
|
|||||||
|
'use strict';
|
||||||
|
require('../common');
|
||||||
|
|
||||||
|
// This test ensures that zlib throws a RangeError if the final buffer needs to
|
||||||
|
// be larger than kMaxLength and concatenation fails.
|
||||||
|
// https://github.com/nodejs/node/pull/1811
|
||||||
|
|
||||||
|
const assert = require('assert');
|
||||||
|
|
||||||
|
// Change kMaxLength for zlib to trigger the error without having to allocate
|
||||||
|
// large Buffers.
|
||||||
|
const buffer = require('buffer');
|
||||||
|
const oldkMaxLength = buffer.kMaxLength;
|
||||||
|
buffer.kMaxLength = 64;
|
||||||
|
const zlib = require('zlib');
|
||||||
|
buffer.kMaxLength = oldkMaxLength;
|
||||||
|
|
||||||
|
// "a".repeat(128), compressed using zstd.
|
||||||
|
const encoded = Buffer.from('KLUv/SCARQAAEGFhAQA7BVg=', 'base64');
|
||||||
|
|
||||||
|
// Async
|
||||||
|
zlib.zstdDecompress(encoded, function(err) {
|
||||||
|
assert.ok(err instanceof RangeError);
|
||||||
|
});
|
||||||
|
|
||||||
|
// Sync
|
||||||
|
assert.throws(function() {
|
||||||
|
zlib.zstdDecompressSync(encoded);
|
||||||
|
}, RangeError);
|
37
test/parallel/test-zlib-zstd-pledged-src-size.js
Normal file
37
test/parallel/test-zlib-zstd-pledged-src-size.js
Normal file
@ -0,0 +1,37 @@
|
|||||||
|
'use strict';
|
||||||
|
const common = require('../common');
|
||||||
|
const assert = require('assert');
|
||||||
|
const zlib = require('zlib');
|
||||||
|
|
||||||
|
function compressWithPledgedSrcSize({ pledgedSrcSize, actualSrcSize }) {
|
||||||
|
return new Promise((resolve, reject) => {
|
||||||
|
const compressor = zlib.createZstdCompress({ pledgedSrcSize });
|
||||||
|
compressor.on('error', (e) => {
|
||||||
|
reject(e);
|
||||||
|
});
|
||||||
|
compressor.on('end', resolve);
|
||||||
|
compressor.write('x'.repeat(actualSrcSize), () => {
|
||||||
|
compressor.end();
|
||||||
|
compressor.resume();
|
||||||
|
});
|
||||||
|
}).then(() => {
|
||||||
|
// Compression should only succeed if sizes match
|
||||||
|
assert.strictEqual(pledgedSrcSize, actualSrcSize);
|
||||||
|
}, (error) => {
|
||||||
|
assert.strictEqual(error.code, 'ZSTD_error_srcSize_wrong');
|
||||||
|
// Size error should only happen when sizes do not match
|
||||||
|
assert.notStrictEqual(pledgedSrcSize, actualSrcSize);
|
||||||
|
}).then(common.mustCall());
|
||||||
|
}
|
||||||
|
|
||||||
|
compressWithPledgedSrcSize({ pledgedSrcSize: 0, actualSrcSize: 0 });
|
||||||
|
|
||||||
|
compressWithPledgedSrcSize({ pledgedSrcSize: 0, actualSrcSize: 42 });
|
||||||
|
|
||||||
|
compressWithPledgedSrcSize({ pledgedSrcSize: 13, actualSrcSize: 42 });
|
||||||
|
|
||||||
|
compressWithPledgedSrcSize({ pledgedSrcSize: 42, actualSrcSize: 0 });
|
||||||
|
|
||||||
|
compressWithPledgedSrcSize({ pledgedSrcSize: 42, actualSrcSize: 13 });
|
||||||
|
|
||||||
|
compressWithPledgedSrcSize({ pledgedSrcSize: 42, actualSrcSize: 42 });
|
134
test/parallel/test-zlib-zstd.js
Normal file
134
test/parallel/test-zlib-zstd.js
Normal file
@ -0,0 +1,134 @@
|
|||||||
|
'use strict';
|
||||||
|
require('../common');
|
||||||
|
const fixtures = require('../common/fixtures');
|
||||||
|
const assert = require('assert');
|
||||||
|
const zlib = require('zlib');
|
||||||
|
|
||||||
|
// Test some zstd-specific properties of the zstd streams that can not
|
||||||
|
// be easily covered through expanding zlib-only tests.
|
||||||
|
|
||||||
|
const sampleBuffer = fixtures.readSync('/pss-vectors.json');
|
||||||
|
|
||||||
|
{
|
||||||
|
// Test setting the quality parameter at stream creation:
|
||||||
|
const sizes = [];
|
||||||
|
for (let quality = 1;
|
||||||
|
quality <= 22;
|
||||||
|
quality++) {
|
||||||
|
const encoded = zlib.zstdCompressSync(sampleBuffer, {
|
||||||
|
params: {
|
||||||
|
[zlib.constants.ZSTD_c_compressionLevel]: quality
|
||||||
|
}
|
||||||
|
});
|
||||||
|
sizes.push(encoded.length);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Increasing quality should roughly correspond to decreasing compressed size:
|
||||||
|
for (let i = 0; i < sizes.length - 1; i++) {
|
||||||
|
assert(sizes[i + 1] <= sizes[i] * 1.05, sizes); // 5 % margin of error.
|
||||||
|
}
|
||||||
|
assert(sizes[0] > sizes[sizes.length - 1], sizes);
|
||||||
|
}
|
||||||
|
|
||||||
|
{
|
||||||
|
// Test that setting out-of-bounds option values or keys fails.
|
||||||
|
assert.throws(() => {
|
||||||
|
zlib.createZstdCompress({
|
||||||
|
params: {
|
||||||
|
10000: 0
|
||||||
|
}
|
||||||
|
});
|
||||||
|
}, {
|
||||||
|
code: 'ERR_ZSTD_INVALID_PARAM',
|
||||||
|
name: 'RangeError',
|
||||||
|
message: '10000 is not a valid zstd parameter'
|
||||||
|
});
|
||||||
|
|
||||||
|
// Test that accidentally using duplicate keys fails.
|
||||||
|
assert.throws(() => {
|
||||||
|
zlib.createZstdCompress({
|
||||||
|
params: {
|
||||||
|
'0': 0,
|
||||||
|
'00': 0
|
||||||
|
}
|
||||||
|
});
|
||||||
|
}, {
|
||||||
|
code: 'ERR_ZSTD_INVALID_PARAM',
|
||||||
|
name: 'RangeError',
|
||||||
|
message: '00 is not a valid zstd parameter'
|
||||||
|
});
|
||||||
|
|
||||||
|
assert.throws(() => {
|
||||||
|
zlib.createZstdCompress({
|
||||||
|
params: {
|
||||||
|
// This param must be a valid ZSTD_strategy value.
|
||||||
|
[zlib.constants.ZSTD_c_strategy]: 130
|
||||||
|
}
|
||||||
|
});
|
||||||
|
}, {
|
||||||
|
code: 'ERR_ZLIB_INITIALIZATION_FAILED',
|
||||||
|
name: 'Error',
|
||||||
|
message: 'Setting parameter failed'
|
||||||
|
});
|
||||||
|
|
||||||
|
// Test that setting out-of-bounds option values or keys fails.
|
||||||
|
assert.throws(() => {
|
||||||
|
zlib.createZstdDecompress({
|
||||||
|
params: {
|
||||||
|
10000: 0
|
||||||
|
}
|
||||||
|
});
|
||||||
|
}, {
|
||||||
|
code: 'ERR_ZSTD_INVALID_PARAM',
|
||||||
|
name: 'RangeError',
|
||||||
|
message: '10000 is not a valid zstd parameter'
|
||||||
|
});
|
||||||
|
|
||||||
|
// Test that accidentally using duplicate keys fails.
|
||||||
|
assert.throws(() => {
|
||||||
|
zlib.createZstdDecompress({
|
||||||
|
params: {
|
||||||
|
'0': 0,
|
||||||
|
'00': 0
|
||||||
|
}
|
||||||
|
});
|
||||||
|
}, {
|
||||||
|
code: 'ERR_ZSTD_INVALID_PARAM',
|
||||||
|
name: 'RangeError',
|
||||||
|
message: '00 is not a valid zstd parameter'
|
||||||
|
});
|
||||||
|
|
||||||
|
assert.throws(() => {
|
||||||
|
zlib.createZstdDecompress({
|
||||||
|
params: {
|
||||||
|
// This param must be >= 10 (ZSTD_WINDOWLOG_ABSOLUTEMIN).
|
||||||
|
[zlib.constants.ZSTD_d_windowLogMax]: 1
|
||||||
|
}
|
||||||
|
});
|
||||||
|
}, {
|
||||||
|
code: 'ERR_ZLIB_INITIALIZATION_FAILED',
|
||||||
|
name: 'Error',
|
||||||
|
message: 'Setting parameter failed'
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
{
|
||||||
|
// Test options.flush range
|
||||||
|
assert.throws(() => {
|
||||||
|
zlib.zstdCompressSync('', { flush: zlib.constants.Z_FINISH });
|
||||||
|
}, {
|
||||||
|
code: 'ERR_OUT_OF_RANGE',
|
||||||
|
name: 'RangeError',
|
||||||
|
message: 'The value of "options.flush" is out of range. It must be >= 0 ' +
|
||||||
|
'and <= 2. Received 4',
|
||||||
|
});
|
||||||
|
|
||||||
|
assert.throws(() => {
|
||||||
|
zlib.zstdCompressSync('', { finishFlush: zlib.constants.Z_FINISH });
|
||||||
|
}, {
|
||||||
|
code: 'ERR_OUT_OF_RANGE',
|
||||||
|
name: 'RangeError',
|
||||||
|
message: 'The value of "options.finishFlush" is out of range. It must be ' +
|
||||||
|
'>= 0 and <= 2. Received 4',
|
||||||
|
});
|
||||||
|
}
|
@ -43,6 +43,7 @@ let zlibPairs = [
|
|||||||
[zlib.Gzip, zlib.Unzip],
|
[zlib.Gzip, zlib.Unzip],
|
||||||
[zlib.DeflateRaw, zlib.InflateRaw],
|
[zlib.DeflateRaw, zlib.InflateRaw],
|
||||||
[zlib.BrotliCompress, zlib.BrotliDecompress],
|
[zlib.BrotliCompress, zlib.BrotliDecompress],
|
||||||
|
[zlib.ZstdCompress, zlib.ZstdDecompress],
|
||||||
];
|
];
|
||||||
|
|
||||||
// How fast to trickle through the slowstream
|
// How fast to trickle through the slowstream
|
||||||
|
@ -252,6 +252,7 @@ const customTypesMap = {
|
|||||||
'X509Certificate': 'crypto.html#class-x509certificate',
|
'X509Certificate': 'crypto.html#class-x509certificate',
|
||||||
|
|
||||||
'zlib options': 'zlib.html#class-options',
|
'zlib options': 'zlib.html#class-options',
|
||||||
|
'zstd options': 'zlib.html#class-zstdoptions',
|
||||||
|
|
||||||
'ReadableStream':
|
'ReadableStream':
|
||||||
'webstreams.html#class-readablestream',
|
'webstreams.html#class-readablestream',
|
||||||
|
Loading…
Reference in New Issue
Block a user