zlib: add zstd support
Some checks are pending
Coverage Linux (without intl) / coverage-linux-without-intl (push) Waiting to run
Coverage Linux / coverage-linux (push) Waiting to run
Coverage Windows / coverage-windows (push) Waiting to run
Test and upload documentation to artifacts / build-docs (push) Waiting to run
Linters / lint-addon-docs (push) Waiting to run
Linters / lint-cpp (push) Waiting to run
Linters / format-cpp (push) Waiting to run
Linters / lint-js-and-md (push) Waiting to run
Linters / lint-py (push) Waiting to run
Linters / lint-yaml (push) Waiting to run
Linters / lint-sh (push) Waiting to run
Linters / lint-codeowners (push) Waiting to run
Linters / lint-pr-url (push) Waiting to run
Linters / lint-readme (push) Waiting to run
Notify on Push / Notify on Force Push on `main` (push) Waiting to run
Notify on Push / Notify on Push on `main` that lacks metadata (push) Waiting to run
Scorecard supply-chain security / Scorecard analysis (push) Waiting to run

Fixes: https://github.com/nodejs/node/issues/48412
PR-URL: https://github.com/nodejs/node/pull/52100
Reviewed-By: Yagiz Nizipli <yagiz@nizipli.com>
This commit is contained in:
Jan Krems 2024-03-15 14:54:30 -07:00 committed by Node.js GitHub Bot
parent f5353100be
commit bf12d72faa
22 changed files with 997 additions and 21 deletions

View File

@ -5,7 +5,7 @@ const zlib = require('zlib');
const bench = common.createBenchmark(main, {
type: [
'Deflate', 'DeflateRaw', 'Inflate', 'InflateRaw', 'Gzip', 'Gunzip', 'Unzip',
'BrotliCompress', 'BrotliDecompress',
'BrotliCompress', 'BrotliDecompress', 'ZstdCompress', 'ZstdDecompress',
],
options: ['true', 'false'],
n: [5e5],

View File

@ -7,7 +7,7 @@ const bench = common.createBenchmark(main, {
inputLen: [1024],
duration: [5],
type: ['string', 'buffer'],
algorithm: ['gzip', 'brotli'],
algorithm: ['gzip', 'brotli', 'zstd'],
}, {
test: {
inputLen: 1024,
@ -15,14 +15,19 @@ const bench = common.createBenchmark(main, {
},
});
const algorithms = {
'gzip': [zlib.createGzip, zlib.createGunzip],
'brotli': [zlib.createBrotliCompress, zlib.createBrotliDecompress],
'zstd': [zlib.createZstdCompress, zlib.createZstdDecompress],
};
function main({ inputLen, duration, type, algorithm }) {
const buffer = Buffer.alloc(inputLen, fs.readFileSync(__filename));
const chunk = type === 'buffer' ? buffer : buffer.toString('utf8');
const input = algorithm === 'gzip' ?
zlib.createGzip() : zlib.createBrotliCompress();
const output = algorithm === 'gzip' ?
zlib.createGunzip() : zlib.createBrotliDecompress();
const [createCompress, createUncompress] = algorithms[algorithm];
const input = createCompress();
const output = createUncompress();
let readFromOutput = 0;
input.pipe(output);

View File

@ -3334,6 +3334,12 @@ The requested functionality is not supported in worker threads.
Creation of a [`zlib`][] object failed due to incorrect configuration.
<a id="ERR_ZSTD_INVALID_PARAM"></a>
### `ERR_ZSTD_INVALID_PARAM`
An invalid parameter key was passed during construction of a Zstd stream.
<a id="HPE_CHUNK_EXTENSIONS_OVERFLOW"></a>
### `HPE_CHUNK_EXTENSIONS_OVERFLOW`

View File

@ -7,7 +7,7 @@
<!-- source_link=lib/zlib.js -->
The `node:zlib` module provides compression functionality implemented using
Gzip, Deflate/Inflate, and Brotli.
Gzip, Deflate/Inflate, Brotli, and Zstd.
To access it:
@ -220,8 +220,8 @@ operations be cached to avoid duplication of effort.
## Compressing HTTP requests and responses
The `node:zlib` module can be used to implement support for the `gzip`, `deflate`
and `br` content-encoding mechanisms defined by
The `node:zlib` module can be used to implement support for the `gzip`, `deflate`,
`br`, and `zstd` content-encoding mechanisms defined by
[HTTP](https://tools.ietf.org/html/rfc7230#section-4.2).
The HTTP [`Accept-Encoding`][] header is used within an HTTP request to identify
@ -284,7 +284,7 @@ const { pipeline } = require('node:stream');
const request = http.get({ host: 'example.com',
path: '/',
port: 80,
headers: { 'Accept-Encoding': 'br,gzip,deflate' } });
headers: { 'Accept-Encoding': 'br,gzip,deflate,zstd' } });
request.on('response', (response) => {
const output = fs.createWriteStream('example.com_index.html');
@ -306,6 +306,9 @@ request.on('response', (response) => {
case 'deflate':
pipeline(response, zlib.createInflate(), output, onError);
break;
case 'zstd':
pipeline(response, zlib.createZstdDecompress(), output, onError);
break;
default:
pipeline(response, output, onError);
break;
@ -396,6 +399,9 @@ http.createServer((request, response) => {
} else if (/\bbr\b/.test(acceptEncoding)) {
response.writeHead(200, { 'Content-Encoding': 'br' });
pipeline(raw, zlib.createBrotliCompress(), response, onError);
} else if (/\bzstd\b/.test(acceptEncoding)) {
response.writeHead(200, { 'Content-Encoding': 'zstd' });
pipeline(raw, zlib.createZstdCompress(), response, onError);
} else {
response.writeHead(200, {});
pipeline(raw, response, onError);
@ -416,6 +422,7 @@ const buffer = Buffer.from('eJzT0yMA', 'base64');
zlib.unzip(
buffer,
// For Brotli, the equivalent is zlib.constants.BROTLI_OPERATION_FLUSH.
// For Zstd, the equivalent is zlib.constants.ZSTD_e_flush.
{ finishFlush: zlib.constants.Z_SYNC_FLUSH },
(err, buffer) => {
if (err) {
@ -487,6 +494,16 @@ these options have different ranges than the zlib ones:
See [below][Brotli parameters] for more details on Brotli-specific options.
### For Zstd-based streams
There are equivalents to the zlib options for Zstd-based streams, although
these options have different ranges than the zlib ones:
* zlib's `level` option matches Zstd's `ZSTD_c_compressionLevel` option.
* zlib's `windowBits` option matches Zstd's `ZSTD_c_windowLog` option.
See [below][Zstd parameters] for more details on Zstd-specific options.
## Flushing
Calling [`.flush()`][] on a compression stream will make `zlib` return as much
@ -701,6 +718,50 @@ These advanced options are available for controlling decompression:
* Boolean flag enabling “Large Window Brotli” mode (not compatible with the
Brotli format as standardized in [RFC 7932][]).
### Zstd constants
<!-- YAML
added: REPLACEME
-->
There are several options and other constants available for Zstd-based
streams:
#### Flush operations
The following values are valid flush operations for Zstd-based streams:
* `zlib.constants.ZSTD_e_continue` (default for all operations)
* `zlib.constants.ZSTD_e_flush` (default when calling `.flush()`)
* `zlib.constants.ZSTD_e_end` (default for the last chunk)
#### Compressor options
There are several options that can be set on Zstd encoders, affecting
compression efficiency and speed. Both the keys and the values can be accessed
as properties of the `zlib.constants` object.
The most important options are:
* `ZSTD_c_compressionLevel`
* Set compression parameters according to pre-defined cLevel table. Default
level is ZSTD\_CLEVEL\_DEFAULT==3.
#### Pledged Source Size
It's possible to specify the expected total size of the uncompressed input via
`opts.pledgedSrcSize`. If the size doesn't match at the end of the input,
compression will fail with the code `ZSTD_error_srcSize_wrong`.
#### Decompressor options
These advanced options are available for controlling decompression:
* `ZSTD_d_windowLogMax`
* Select a size limit (in power of 2) beyond which the streaming API will
refuse to allocate memory buffer in order to protect the host from
unreasonable memory requirements.
## Class: `Options`
<!-- YAML
@ -962,6 +1023,51 @@ added: v0.7.0
Reset the compressor/decompressor to factory defaults. Only applicable to
the inflate and deflate algorithms.
## Class: `ZstdOptions`
<!-- YAML
added: REPLACEME
-->
<!--type=misc-->
Each Zstd-based class takes an `options` object. All options are optional.
* `flush` {integer} **Default:** `zlib.constants.ZSTD_e_continue`
* `finishFlush` {integer} **Default:** `zlib.constants.ZSTD_e_end`
* `chunkSize` {integer} **Default:** `16 * 1024`
* `params` {Object} Key-value object containing indexed [Zstd parameters][].
* `maxOutputLength` {integer} Limits output size when using
[convenience methods][]. **Default:** [`buffer.kMaxLength`][]
For example:
```js
const stream = zlib.createZstdCompress({
chunkSize: 32 * 1024,
params: {
[zlib.constants.ZSTD_c_compressionLevel]: 10,
[zlib.constants.ZSTD_c_checksumFlag]: 1,
},
});
```
## Class: `zlib.ZstdCompress`
<!-- YAML
added: REPLACEME
-->
Compress data using the Zstd algorithm.
## Class: `zlib.ZstdDecompress`
<!-- YAML
added: REPLACEME
-->
Decompress data using the Zstd algorithm.
## `zlib.constants`
<!-- YAML
@ -1135,6 +1241,26 @@ added: v0.5.8
Creates and returns a new [`Unzip`][] object.
## `zlib.createZstdCompress([options])`
<!-- YAML
added: REPLACEME
-->
* `options` {zstd options}
Creates and returns a new [`ZstdCompress`][] object.
## `zlib.createZstdDecompress([options])`
<!-- YAML
added: REPLACEME
-->
* `options` {zstd options}
Creates and returns a new [`ZstdDecompress`][] object.
## Convenience methods
<!--type=misc-->
@ -1481,11 +1607,54 @@ changes:
Decompress a chunk of data with [`Unzip`][].
### `zlib.zstdCompress(buffer[, options], callback)`
<!-- YAML
added: REPLACEME
-->
* `buffer` {Buffer|TypedArray|DataView|ArrayBuffer|string}
* `options` {zstd options}
* `callback` {Function}
### `zlib.zstdCompressSync(buffer[, options])`
<!-- YAML
added: REPLACEME
-->
* `buffer` {Buffer|TypedArray|DataView|ArrayBuffer|string}
* `options` {zstd options}
Compress a chunk of data with [`ZstdCompress`][].
### `zlib.zstdDecompress(buffer[, options], callback)`
<!-- YAML
added: REPLACEME
-->
* `buffer` {Buffer|TypedArray|DataView|ArrayBuffer|string}
* `options` {zstd options}
* `callback` {Function}
### `zlib.zstdDecompressSync(buffer[, options])`
<!-- YAML
added: REPLACEME
-->
* `buffer` {Buffer|TypedArray|DataView|ArrayBuffer|string}
* `options` {zstd options}
Decompress a chunk of data with [`ZstdDecompress`][].
[Brotli parameters]: #brotli-constants
[Cyclic redundancy check]: https://en.wikipedia.org/wiki/Cyclic_redundancy_check
[Memory usage tuning]: #memory-usage-tuning
[RFC 7932]: https://www.rfc-editor.org/rfc/rfc7932.txt
[Streams API]: stream.md
[Zstd parameters]: #zstd-constants
[`.flush()`]: #zlibflushkind-callback
[`Accept-Encoding`]: https://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.3
[`BrotliCompress`]: #class-zlibbrotlicompress
@ -1498,6 +1667,8 @@ Decompress a chunk of data with [`Unzip`][].
[`InflateRaw`]: #class-zlibinflateraw
[`Inflate`]: #class-zlibinflate
[`Unzip`]: #class-zlibunzip
[`ZstdCompress`]: #class-zlibzstdcompress
[`ZstdDecompress`]: #class-zlibzstddecompress
[`buffer.kMaxLength`]: buffer.md#bufferkmaxlength
[`deflateInit2` and `inflateInit2`]: https://zlib.net/manual.html#Advanced
[`stream.Transform`]: stream.md#class-streamtransform

View File

@ -1889,3 +1889,4 @@ E('ERR_WORKER_UNSERIALIZABLE_ERROR',
'Serializing an uncaught exception failed', Error);
E('ERR_WORKER_UNSUPPORTED_OPERATION',
'%s is not supported in workers', TypeError);
E('ERR_ZSTD_INVALID_PARAM', '%s is not a valid zstd parameter', RangeError);

View File

@ -42,6 +42,7 @@ const {
ERR_BUFFER_TOO_LARGE,
ERR_INVALID_ARG_TYPE,
ERR_OUT_OF_RANGE,
ERR_ZSTD_INVALID_PARAM,
},
genericNodeError,
} = require('internal/errors');
@ -83,9 +84,12 @@ const {
// Node's compression stream modes (node_zlib_mode)
DEFLATE, DEFLATERAW, INFLATE, INFLATERAW, GZIP, GUNZIP, UNZIP,
BROTLI_DECODE, BROTLI_ENCODE,
ZSTD_COMPRESS, ZSTD_DECOMPRESS,
// Brotli operations (~flush levels)
BROTLI_OPERATION_PROCESS, BROTLI_OPERATION_FLUSH,
BROTLI_OPERATION_FINISH, BROTLI_OPERATION_EMIT_METADATA,
// Zstd end directives (~flush levels)
ZSTD_e_continue, ZSTD_e_flush, ZSTD_e_end,
} = constants;
// Translation table for return codes.
@ -192,9 +196,11 @@ function zlibOnError(message, errno, code) {
const FLUSH_BOUND = [
[ Z_NO_FLUSH, Z_BLOCK ],
[ BROTLI_OPERATION_PROCESS, BROTLI_OPERATION_EMIT_METADATA ],
[ ZSTD_e_continue, ZSTD_e_end ],
];
const FLUSH_BOUND_IDX_NORMAL = 0;
const FLUSH_BOUND_IDX_BROTLI = 1;
const FLUSH_BOUND_IDX_ZSTD = 2;
// The base class for all Zlib-style streams.
function ZlibBase(opts, mode, handle, { flush, finishFlush, fullFlush }) {
@ -203,13 +209,15 @@ function ZlibBase(opts, mode, handle, { flush, finishFlush, fullFlush }) {
// The ZlibBase class is not exported to user land, the mode should only be
// passed in by us.
assert(typeof mode === 'number');
assert(mode >= DEFLATE && mode <= BROTLI_ENCODE);
assert(mode >= DEFLATE && mode <= ZSTD_DECOMPRESS);
let flushBoundIdx;
if (mode !== BROTLI_ENCODE && mode !== BROTLI_DECODE) {
flushBoundIdx = FLUSH_BOUND_IDX_NORMAL;
} else {
if (mode === BROTLI_ENCODE || mode === BROTLI_DECODE) {
flushBoundIdx = FLUSH_BOUND_IDX_BROTLI;
} else if (mode === ZSTD_COMPRESS || mode === ZSTD_DECOMPRESS) {
flushBoundIdx = FLUSH_BOUND_IDX_ZSTD;
} else {
flushBoundIdx = FLUSH_BOUND_IDX_NORMAL;
}
if (opts) {
@ -829,6 +837,89 @@ ObjectSetPrototypeOf(BrotliDecompress.prototype, Brotli.prototype);
ObjectSetPrototypeOf(BrotliDecompress, Brotli);
const zstdDefaultOpts = {
flush: ZSTD_e_continue,
finishFlush: ZSTD_e_end,
fullFlush: ZSTD_e_flush,
};
function Zstd(opts, mode, initParamsArray, maxParam) {
assert(mode === ZSTD_COMPRESS || mode === ZSTD_DECOMPRESS);
initParamsArray.fill(-1);
if (opts?.params) {
ObjectKeys(opts.params).forEach((origKey) => {
const key = +origKey;
if (NumberIsNaN(key) || key < 0 || key > maxParam ||
(initParamsArray[key] | 0) !== -1) {
throw new ERR_ZSTD_INVALID_PARAM(origKey);
}
const value = opts.params[origKey];
if (typeof value !== 'number' && typeof value !== 'boolean') {
throw new ERR_INVALID_ARG_TYPE('options.params[key]',
'number', opts.params[origKey]);
}
initParamsArray[key] = value;
});
}
const handle = mode === ZSTD_COMPRESS ?
new binding.ZstdCompress() : new binding.ZstdDecompress();
const pledgedSrcSize = opts?.pledgedSrcSize ?? undefined;
this._writeState = new Uint32Array(2);
handle.init(
initParamsArray,
pledgedSrcSize,
this._writeState,
processCallback,
);
ReflectApply(ZlibBase, this, [opts, mode, handle, zstdDefaultOpts]);
}
ObjectSetPrototypeOf(Zstd.prototype, ZlibBase.prototype);
ObjectSetPrototypeOf(Zstd, ZlibBase);
const kMaxZstdCParam = MathMax(...ObjectKeys(constants).map(
(key) => (key.startsWith('ZSTD_c_') ?
constants[key] :
0),
));
const zstdInitCParamsArray = new Uint32Array(kMaxZstdCParam + 1);
function ZstdCompress(opts) {
if (!(this instanceof ZstdCompress))
return new ZstdCompress(opts);
ReflectApply(Zstd, this,
[opts, ZSTD_COMPRESS, zstdInitCParamsArray, kMaxZstdCParam]);
}
ObjectSetPrototypeOf(ZstdCompress.prototype, Zstd.prototype);
ObjectSetPrototypeOf(ZstdCompress, Zstd);
const kMaxZstdDParam = MathMax(...ObjectKeys(constants).map(
(key) => (key.startsWith('ZSTD_d_') ?
constants[key] :
0),
));
const zstdInitDParamsArray = new Uint32Array(kMaxZstdDParam + 1);
function ZstdDecompress(opts) {
if (!(this instanceof ZstdDecompress))
return new ZstdDecompress(opts);
ReflectApply(Zstd, this,
[opts, ZSTD_DECOMPRESS, zstdInitDParamsArray, kMaxZstdDParam]);
}
ObjectSetPrototypeOf(ZstdDecompress.prototype, Zstd.prototype);
ObjectSetPrototypeOf(ZstdDecompress, Zstd);
function createProperty(ctor) {
return {
__proto__: null,
@ -867,6 +958,8 @@ module.exports = {
Unzip,
BrotliCompress,
BrotliDecompress,
ZstdCompress,
ZstdDecompress,
// Convenience methods.
// compress/decompress a string or buffer in one step.
@ -888,6 +981,10 @@ module.exports = {
brotliCompressSync: createConvenienceMethod(BrotliCompress, true),
brotliDecompress: createConvenienceMethod(BrotliDecompress, false),
brotliDecompressSync: createConvenienceMethod(BrotliDecompress, true),
zstdCompress: createConvenienceMethod(ZstdCompress, false),
zstdCompressSync: createConvenienceMethod(ZstdCompress, true),
zstdDecompress: createConvenienceMethod(ZstdDecompress, false),
zstdDecompressSync: createConvenienceMethod(ZstdDecompress, true),
};
ObjectDefineProperties(module.exports, {
@ -900,6 +997,8 @@ ObjectDefineProperties(module.exports, {
createUnzip: createProperty(Unzip),
createBrotliCompress: createProperty(BrotliCompress),
createBrotliDecompress: createProperty(BrotliDecompress),
createZstdCompress: createProperty(ZstdCompress),
createZstdDecompress: createProperty(ZstdDecompress),
constants: {
__proto__: null,
configurable: false,

View File

@ -32,9 +32,11 @@
#include "v8.h"
#include "brotli/encode.h"
#include "brotli/decode.h"
#include "brotli/encode.h"
#include "zlib.h"
#include "zstd.h"
#include "zstd_errors.h"
#include <sys/types.h>
@ -95,6 +97,44 @@ inline const char* ZlibStrerror(int err) {
return "Z_UNKNOWN_ERROR";
}
#define ZSTD_ERROR_CODES(V) \
V(ZSTD_error_no_error) \
V(ZSTD_error_GENERIC) \
V(ZSTD_error_prefix_unknown) \
V(ZSTD_error_version_unsupported) \
V(ZSTD_error_frameParameter_unsupported) \
V(ZSTD_error_frameParameter_windowTooLarge) \
V(ZSTD_error_corruption_detected) \
V(ZSTD_error_checksum_wrong) \
V(ZSTD_error_literals_headerWrong) \
V(ZSTD_error_dictionary_corrupted) \
V(ZSTD_error_dictionary_wrong) \
V(ZSTD_error_dictionaryCreation_failed) \
V(ZSTD_error_parameter_unsupported) \
V(ZSTD_error_parameter_combination_unsupported) \
V(ZSTD_error_parameter_outOfBound) \
V(ZSTD_error_tableLog_tooLarge) \
V(ZSTD_error_maxSymbolValue_tooLarge) \
V(ZSTD_error_maxSymbolValue_tooSmall) \
V(ZSTD_error_stabilityCondition_notRespected) \
V(ZSTD_error_stage_wrong) \
V(ZSTD_error_init_missing) \
V(ZSTD_error_memory_allocation) \
V(ZSTD_error_workSpace_tooSmall) \
V(ZSTD_error_dstSize_tooSmall) \
V(ZSTD_error_srcSize_wrong) \
V(ZSTD_error_dstBuffer_null) \
V(ZSTD_error_noForwardProgress_destFull) \
V(ZSTD_error_noForwardProgress_inputEmpty)
inline const char* ZstdStrerror(int err) {
#define V(code) \
if (err == code) return #code;
ZSTD_ERROR_CODES(V)
#undef V
return "ZSTD_error_GENERIC";
}
enum node_zlib_mode {
NONE,
DEFLATE,
@ -105,7 +145,9 @@ enum node_zlib_mode {
INFLATERAW,
UNZIP,
BROTLI_DECODE,
BROTLI_ENCODE
BROTLI_ENCODE,
ZSTD_COMPRESS,
ZSTD_DECOMPRESS
};
constexpr uint8_t GZIP_HEADER_ID1 = 0x1f;
@ -249,6 +291,79 @@ class BrotliDecoderContext final : public BrotliContext {
DeleteFnPtr<BrotliDecoderState, BrotliDecoderDestroyInstance> state_;
};
class ZstdContext : public MemoryRetainer {
public:
ZstdContext() = default;
// Streaming-related, should be available for all compression libraries:
void Close();
void SetBuffers(const char* in, uint32_t in_len, char* out, uint32_t out_len);
void SetFlush(int flush);
void GetAfterWriteOffsets(uint32_t* avail_in, uint32_t* avail_out) const;
CompressionError GetErrorInfo() const;
ZstdContext(const ZstdContext&) = delete;
ZstdContext& operator=(const ZstdContext&) = delete;
protected:
ZSTD_EndDirective flush_ = ZSTD_e_continue;
ZSTD_inBuffer input_ = {nullptr, 0, 0};
ZSTD_outBuffer output_ = {nullptr, 0, 0};
ZSTD_ErrorCode error_ = ZSTD_error_no_error;
std::string error_string_;
std::string error_code_string_;
};
class ZstdCompressContext final : public ZstdContext {
public:
ZstdCompressContext() = default;
// Streaming-related, should be available for all compression libraries:
void DoThreadPoolWork();
CompressionError ResetStream();
// Zstd specific:
CompressionError Init(uint64_t pledged_src_size);
CompressionError SetParameter(int key, int value);
// Wrap ZSTD_freeCCtx to remove the return type.
static void FreeZstd(ZSTD_CCtx* cctx) { ZSTD_freeCCtx(cctx); }
SET_MEMORY_INFO_NAME(ZstdCompressContext)
SET_SELF_SIZE(ZstdCompressContext)
SET_NO_MEMORY_INFO()
private:
DeleteFnPtr<ZSTD_CCtx, ZstdCompressContext::FreeZstd> cctx_;
uint64_t pledged_src_size_ = ZSTD_CONTENTSIZE_UNKNOWN;
};
class ZstdDecompressContext final : public ZstdContext {
public:
ZstdDecompressContext() = default;
// Streaming-related, should be available for all compression libraries:
void DoThreadPoolWork();
CompressionError ResetStream();
// Zstd specific:
CompressionError Init(uint64_t pledged_src_size);
CompressionError SetParameter(int key, int value);
// Wrap ZSTD_freeDCtx to remove the return type.
static void FreeZstd(ZSTD_DCtx* dctx) { ZSTD_freeDCtx(dctx); }
SET_MEMORY_INFO_NAME(ZstdDecompressContext)
SET_SELF_SIZE(ZstdDecompressContext)
SET_NO_MEMORY_INFO()
private:
DeleteFnPtr<ZSTD_DCtx, ZstdDecompressContext::FreeZstd> dctx_;
};
template <typename CompressionContext>
class CompressionStream : public AsyncWrap, public ThreadPoolWork {
public:
@ -740,6 +855,92 @@ class BrotliCompressionStream final :
using BrotliEncoderStream = BrotliCompressionStream<BrotliEncoderContext>;
using BrotliDecoderStream = BrotliCompressionStream<BrotliDecoderContext>;
template <typename CompressionContext>
class ZstdStream final : public CompressionStream<CompressionContext> {
public:
ZstdStream(Environment* env, Local<Object> wrap)
: CompressionStream<CompressionContext>(env, wrap) {}
inline CompressionContext* context() {
return this->CompressionStream<CompressionContext>::context();
}
typedef typename CompressionStream<CompressionContext>::AllocScope AllocScope;
static void New(const FunctionCallbackInfo<Value>& args) {
Environment* env = Environment::GetCurrent(args);
new ZstdStream(env, args.This());
}
static void Init(const FunctionCallbackInfo<Value>& args) {
Environment* env = Environment::GetCurrent(args);
Local<Context> context = env->context();
CHECK(args.Length() == 4 &&
"init(params, pledgedSrcSize, writeResult, writeCallback)");
ZstdStream* wrap;
ASSIGN_OR_RETURN_UNWRAP(&wrap, args.Holder());
CHECK(args[2]->IsUint32Array());
uint32_t* write_result = reinterpret_cast<uint32_t*>(Buffer::Data(args[2]));
CHECK(args[3]->IsFunction());
Local<Function> write_js_callback = args[3].As<Function>();
wrap->InitStream(write_result, write_js_callback);
uint64_t pledged_src_size = ZSTD_CONTENTSIZE_UNKNOWN;
if (args[1]->IsNumber()) {
int64_t signed_pledged_src_size;
if (!args[1]->IntegerValue(context).To(&signed_pledged_src_size)) {
THROW_ERR_INVALID_ARG_VALUE(wrap->env(),
"pledgedSrcSize should be an integer");
return;
}
if (signed_pledged_src_size < 0) {
THROW_ERR_INVALID_ARG_VALUE(wrap->env(),
"pledgedSrcSize may not be negative");
return;
}
pledged_src_size = signed_pledged_src_size;
}
AllocScope alloc_scope(wrap);
CompressionError err = wrap->context()->Init(pledged_src_size);
if (err.IsError()) {
wrap->EmitError(err);
THROW_ERR_ZLIB_INITIALIZATION_FAILED(wrap->env(), err.message);
return;
}
CHECK(args[0]->IsUint32Array());
const uint32_t* data = reinterpret_cast<uint32_t*>(Buffer::Data(args[0]));
size_t len = args[0].As<Uint32Array>()->Length();
for (int i = 0; static_cast<size_t>(i) < len; i++) {
if (data[i] == static_cast<uint32_t>(-1)) continue;
CompressionError err = wrap->context()->SetParameter(i, data[i]);
if (err.IsError()) {
wrap->EmitError(err);
THROW_ERR_ZLIB_INITIALIZATION_FAILED(wrap->env(), err.message);
return;
}
}
}
static void Params(const FunctionCallbackInfo<Value>& args) {
// Currently a no-op, and not accessed from JS land.
// At some point zstd may support changing parameters on the fly,
// in which case we can implement this and a JS equivalent similar to
// the zlib Params() function.
}
SET_MEMORY_INFO_NAME(ZstdStream)
SET_SELF_SIZE(ZstdStream)
};
using ZstdCompressStream = ZstdStream<ZstdCompressContext>;
using ZstdDecompressStream = ZstdStream<ZstdDecompressContext>;
void ZlibContext::Close() {
{
Mutex::ScopedLock lock(mutex_);
@ -1263,6 +1464,115 @@ CompressionError BrotliDecoderContext::GetErrorInfo() const {
}
}
void ZstdContext::Close() {}
void ZstdContext::SetBuffers(const char* in,
uint32_t in_len,
char* out,
uint32_t out_len) {
input_.src = reinterpret_cast<const uint8_t*>(in);
input_.size = in_len;
input_.pos = 0;
output_.dst = reinterpret_cast<uint8_t*>(out);
output_.size = out_len;
output_.pos = 0;
}
void ZstdContext::SetFlush(int flush) {
flush_ = static_cast<ZSTD_EndDirective>(flush);
}
void ZstdContext::GetAfterWriteOffsets(uint32_t* avail_in,
uint32_t* avail_out) const {
*avail_in = input_.size - input_.pos;
*avail_out = output_.size - output_.pos;
}
CompressionError ZstdContext::GetErrorInfo() const {
if (error_ != ZSTD_error_no_error) {
return CompressionError(error_string_.c_str(),
error_code_string_.c_str(),
static_cast<int>(error_));
} else {
return {};
}
}
CompressionError ZstdCompressContext::SetParameter(int key, int value) {
size_t result = ZSTD_CCtx_setParameter(
cctx_.get(), static_cast<ZSTD_cParameter>(key), value);
if (ZSTD_isError(result)) {
return CompressionError(
"Setting parameter failed", "ERR_ZSTD_PARAM_SET_FAILED", -1);
}
return {};
}
CompressionError ZstdCompressContext::Init(uint64_t pledged_src_size) {
pledged_src_size_ = pledged_src_size;
cctx_.reset(ZSTD_createCCtx());
if (!cctx_) {
return CompressionError("Could not initialize zstd instance",
"ERR_ZLIB_INITIALIZATION_FAILED",
-1);
}
size_t result = ZSTD_CCtx_setPledgedSrcSize(cctx_.get(), pledged_src_size);
if (ZSTD_isError(result)) {
return CompressionError(
"Could not set pledged src size", "ERR_ZLIB_INITIALIZATION_FAILED", -1);
}
return {};
}
CompressionError ZstdCompressContext::ResetStream() {
return Init(pledged_src_size_);
}
void ZstdCompressContext::DoThreadPoolWork() {
size_t const remaining =
ZSTD_compressStream2(cctx_.get(), &output_, &input_, flush_);
if (ZSTD_isError(remaining)) {
error_ = ZSTD_getErrorCode(remaining);
error_code_string_ = ZstdStrerror(error_);
error_string_ = ZSTD_getErrorString(error_);
}
}
CompressionError ZstdDecompressContext::SetParameter(int key, int value) {
size_t result = ZSTD_DCtx_setParameter(
dctx_.get(), static_cast<ZSTD_dParameter>(key), value);
if (ZSTD_isError(result)) {
return CompressionError(
"Setting parameter failed", "ERR_ZSTD_PARAM_SET_FAILED", -1);
}
return {};
}
CompressionError ZstdDecompressContext::Init(uint64_t pledged_src_size) {
dctx_.reset(ZSTD_createDCtx());
if (!dctx_) {
return CompressionError("Could not initialize zstd instance",
"ERR_ZLIB_INITIALIZATION_FAILED",
-1);
}
return {};
}
CompressionError ZstdDecompressContext::ResetStream() {
// We pass ZSTD_CONTENTSIZE_UNKNOWN because the argument is ignored for
// decompression.
return Init(ZSTD_CONTENTSIZE_UNKNOWN);
}
void ZstdDecompressContext::DoThreadPoolWork() {
size_t const ret = ZSTD_decompressStream(dctx_.get(), &output_, &input_);
if (ZSTD_isError(ret)) {
error_ = ZSTD_getErrorCode(ret);
error_code_string_ = ZstdStrerror(error_);
error_string_ = ZSTD_getErrorString(error_);
}
}
template <typename Stream>
struct MakeClass {
@ -1332,6 +1642,8 @@ void Initialize(Local<Object> target,
MakeClass<ZlibStream>::Make(env, target, "Zlib");
MakeClass<BrotliEncoderStream>::Make(env, target, "BrotliEncoder");
MakeClass<BrotliDecoderStream>::Make(env, target, "BrotliDecoder");
MakeClass<ZstdCompressStream>::Make(env, target, "ZstdCompress");
MakeClass<ZstdDecompressStream>::Make(env, target, "ZstdDecompress");
SetMethod(context, target, "crc32", CRC32);
target->Set(env->context(),
@ -1343,6 +1655,8 @@ void RegisterExternalReferences(ExternalReferenceRegistry* registry) {
MakeClass<ZlibStream>::Make(registry);
MakeClass<BrotliEncoderStream>::Make(registry);
MakeClass<BrotliDecoderStream>::Make(registry);
MakeClass<ZstdCompressStream>::Make(registry);
MakeClass<ZstdDecompressStream>::Make(registry);
registry->Register(CRC32);
}
@ -1387,6 +1701,8 @@ void DefineZlibConstants(Local<Object> target) {
NODE_DEFINE_CONSTANT(target, UNZIP);
NODE_DEFINE_CONSTANT(target, BROTLI_DECODE);
NODE_DEFINE_CONSTANT(target, BROTLI_ENCODE);
NODE_DEFINE_CONSTANT(target, ZSTD_DECOMPRESS);
NODE_DEFINE_CONSTANT(target, ZSTD_COMPRESS);
NODE_DEFINE_CONSTANT(target, Z_MIN_WINDOWBITS);
NODE_DEFINE_CONSTANT(target, Z_MAX_WINDOWBITS);
@ -1466,6 +1782,70 @@ void DefineZlibConstants(Local<Object> target) {
NODE_DEFINE_CONSTANT(target, BROTLI_DECODER_ERROR_ALLOC_RING_BUFFER_2);
NODE_DEFINE_CONSTANT(target, BROTLI_DECODER_ERROR_ALLOC_BLOCK_TYPE_TREES);
NODE_DEFINE_CONSTANT(target, BROTLI_DECODER_ERROR_UNREACHABLE);
// Zstd constants
NODE_DEFINE_CONSTANT(target, ZSTD_e_continue);
NODE_DEFINE_CONSTANT(target, ZSTD_e_flush);
NODE_DEFINE_CONSTANT(target, ZSTD_e_end);
NODE_DEFINE_CONSTANT(target, ZSTD_fast);
NODE_DEFINE_CONSTANT(target, ZSTD_dfast);
NODE_DEFINE_CONSTANT(target, ZSTD_greedy);
NODE_DEFINE_CONSTANT(target, ZSTD_lazy);
NODE_DEFINE_CONSTANT(target, ZSTD_lazy2);
NODE_DEFINE_CONSTANT(target, ZSTD_btlazy2);
NODE_DEFINE_CONSTANT(target, ZSTD_btopt);
NODE_DEFINE_CONSTANT(target, ZSTD_btultra);
NODE_DEFINE_CONSTANT(target, ZSTD_btultra2);
NODE_DEFINE_CONSTANT(target, ZSTD_c_compressionLevel);
NODE_DEFINE_CONSTANT(target, ZSTD_c_windowLog);
NODE_DEFINE_CONSTANT(target, ZSTD_c_hashLog);
NODE_DEFINE_CONSTANT(target, ZSTD_c_chainLog);
NODE_DEFINE_CONSTANT(target, ZSTD_c_searchLog);
NODE_DEFINE_CONSTANT(target, ZSTD_c_minMatch);
NODE_DEFINE_CONSTANT(target, ZSTD_c_targetLength);
NODE_DEFINE_CONSTANT(target, ZSTD_c_strategy);
NODE_DEFINE_CONSTANT(target, ZSTD_c_enableLongDistanceMatching);
NODE_DEFINE_CONSTANT(target, ZSTD_c_ldmHashLog);
NODE_DEFINE_CONSTANT(target, ZSTD_c_ldmMinMatch);
NODE_DEFINE_CONSTANT(target, ZSTD_c_ldmBucketSizeLog);
NODE_DEFINE_CONSTANT(target, ZSTD_c_ldmHashRateLog);
NODE_DEFINE_CONSTANT(target, ZSTD_c_contentSizeFlag);
NODE_DEFINE_CONSTANT(target, ZSTD_c_checksumFlag);
NODE_DEFINE_CONSTANT(target, ZSTD_c_dictIDFlag);
NODE_DEFINE_CONSTANT(target, ZSTD_c_nbWorkers);
NODE_DEFINE_CONSTANT(target, ZSTD_c_jobSize);
NODE_DEFINE_CONSTANT(target, ZSTD_c_overlapLog);
NODE_DEFINE_CONSTANT(target, ZSTD_d_windowLogMax);
NODE_DEFINE_CONSTANT(target, ZSTD_CLEVEL_DEFAULT);
// Error codes
NODE_DEFINE_CONSTANT(target, ZSTD_error_no_error);
NODE_DEFINE_CONSTANT(target, ZSTD_error_GENERIC);
NODE_DEFINE_CONSTANT(target, ZSTD_error_prefix_unknown);
NODE_DEFINE_CONSTANT(target, ZSTD_error_version_unsupported);
NODE_DEFINE_CONSTANT(target, ZSTD_error_frameParameter_unsupported);
NODE_DEFINE_CONSTANT(target, ZSTD_error_frameParameter_windowTooLarge);
NODE_DEFINE_CONSTANT(target, ZSTD_error_corruption_detected);
NODE_DEFINE_CONSTANT(target, ZSTD_error_checksum_wrong);
NODE_DEFINE_CONSTANT(target, ZSTD_error_literals_headerWrong);
NODE_DEFINE_CONSTANT(target, ZSTD_error_dictionary_corrupted);
NODE_DEFINE_CONSTANT(target, ZSTD_error_dictionary_wrong);
NODE_DEFINE_CONSTANT(target, ZSTD_error_dictionaryCreation_failed);
NODE_DEFINE_CONSTANT(target, ZSTD_error_parameter_unsupported);
NODE_DEFINE_CONSTANT(target, ZSTD_error_parameter_combination_unsupported);
NODE_DEFINE_CONSTANT(target, ZSTD_error_parameter_outOfBound);
NODE_DEFINE_CONSTANT(target, ZSTD_error_tableLog_tooLarge);
NODE_DEFINE_CONSTANT(target, ZSTD_error_maxSymbolValue_tooLarge);
NODE_DEFINE_CONSTANT(target, ZSTD_error_maxSymbolValue_tooSmall);
NODE_DEFINE_CONSTANT(target, ZSTD_error_stabilityCondition_notRespected);
NODE_DEFINE_CONSTANT(target, ZSTD_error_stage_wrong);
NODE_DEFINE_CONSTANT(target, ZSTD_error_init_missing);
NODE_DEFINE_CONSTANT(target, ZSTD_error_memory_allocation);
NODE_DEFINE_CONSTANT(target, ZSTD_error_workSpace_tooSmall);
NODE_DEFINE_CONSTANT(target, ZSTD_error_dstSize_tooSmall);
NODE_DEFINE_CONSTANT(target, ZSTD_error_srcSize_wrong);
NODE_DEFINE_CONSTANT(target, ZSTD_error_dstBuffer_null);
NODE_DEFINE_CONSTANT(target, ZSTD_error_noForwardProgress_destFull);
NODE_DEFINE_CONSTANT(target, ZSTD_error_noForwardProgress_inputEmpty);
}
} // namespace node

BIN
test/fixtures/person.jpg.zst vendored Normal file

Binary file not shown.

View File

@ -54,6 +54,8 @@ for (const [type, expect] of [
['deflateRaw', 'inflateRaw', 'DeflateRaw', 'InflateRaw'],
['brotliCompress', 'brotliDecompress',
'BrotliCompress', 'BrotliDecompress'],
['zstdCompress', 'zstdDecompress',
'ZstdCompress', 'ZstdDecompress'],
]) {
zlib[method[0]](expect, opts, common.mustCall((err, result) => {
zlib[method[1]](result, opts, common.mustCall((err, result) => {

View File

@ -11,10 +11,12 @@ const emptyBuffer = Buffer.alloc(0);
[ zlib.deflateSync, zlib.inflateSync, 'deflate sync' ],
[ zlib.gzipSync, zlib.gunzipSync, 'gzip sync' ],
[ zlib.brotliCompressSync, zlib.brotliDecompressSync, 'br sync' ],
[ zlib.zstdCompressSync, zlib.zstdDecompressSync, 'zstd sync' ],
[ promisify(zlib.deflateRaw), promisify(zlib.inflateRaw), 'raw' ],
[ promisify(zlib.deflate), promisify(zlib.inflate), 'deflate' ],
[ promisify(zlib.gzip), promisify(zlib.gunzip), 'gzip' ],
[ promisify(zlib.brotliCompress), promisify(zlib.brotliDecompress), 'br' ],
[ promisify(zlib.zstdCompress), promisify(zlib.zstdDecompress), 'zstd' ],
]) {
const compressed = await compress(emptyBuffer);
const decompressed = await decompress(compressed);

View File

@ -40,6 +40,7 @@ const unzips = [
zlib.Inflate(),
zlib.InflateRaw(),
zlib.BrotliDecompress(),
zlib.ZstdDecompress(),
];
nonStringInputs.forEach(common.mustCall((input) => {

View File

@ -144,6 +144,7 @@ class HashStream extends Stream {
for (const [ createCompress, createDecompress ] of [
[ zlib.createGzip, zlib.createGunzip ],
[ zlib.createBrotliCompress, zlib.createBrotliDecompress ],
[ zlib.createZstdCompress, zlib.createZstdDecompress ],
]) {
const inp = new RandomReadStream({ total: 1024, block: 256, jitter: 16 });
const out = new HashStream();

View File

@ -28,9 +28,10 @@ const zlib = require('node:zlib');
const { test } = require('node:test');
test('zlib should accept writing after flush', async () => {
for (const [createCompress, createDecompress] of [
[zlib.createGzip, zlib.createGunzip],
[zlib.createBrotliCompress, zlib.createBrotliDecompress],
for (const [ createCompress, createDecompress ] of [
[ zlib.createGzip, zlib.createGunzip ],
[ zlib.createBrotliCompress, zlib.createBrotliDecompress ],
[ zlib.createZstdCompress, zlib.createZstdDecompress ],
]) {
const { promise, resolve, reject } = Promise.withResolvers();
const gzip = createCompress();

View File

@ -28,7 +28,13 @@ const zlib = require('node:zlib');
const { test } = require('node:test');
test('zlib should properly handle zero byte input', async () => {
for (const Compressor of [zlib.Gzip, zlib.BrotliCompress]) {
const compressors = [
[zlib.Gzip, 20],
[zlib.BrotliCompress, 1],
[zlib.ZstdCompress, 9],
];
for (const [Compressor, expected] of compressors) {
const { promise, resolve, reject } = Promise.withResolvers();
const gz = Compressor();
const emptyBuffer = Buffer.alloc(0);
@ -38,7 +44,6 @@ test('zlib should properly handle zero byte input', async () => {
});
gz.on('error', reject);
gz.on('end', function() {
const expected = Compressor === zlib.Gzip ? 20 : 1;
assert.strictEqual(received, expected,
`${received}, ${expected}, ${Compressor.name}`);
resolve();

View File

@ -0,0 +1,28 @@
'use strict';
require('../common');
const assert = require('assert');
const zlib = require('zlib');
const fixtures = require('../common/fixtures');
const file = fixtures.readSync('person.jpg');
const chunkSize = 16;
const compress = new zlib.ZstdCompress();
const chunk = file.slice(0, chunkSize);
const expectedFull = Buffer.from('KLUv/QBYgAAA/9j/4AAQSkZJRgABAQEASA==', 'base64');
let actualFull;
compress.write(chunk, function() {
compress.flush(function() {
const bufs = [];
let buf;
while ((buf = compress.read()) !== null)
bufs.push(buf);
actualFull = Buffer.concat(bufs);
});
});
process.once('exit', function() {
assert.deepStrictEqual(actualFull.toString('base64'), expectedFull.toString('base64'));
assert.deepStrictEqual(actualFull, expectedFull);
});

View File

@ -0,0 +1,38 @@
'use strict';
// Test compressing and uncompressing a string with zstd
const common = require('../common');
const assert = require('assert');
const zlib = require('zlib');
const inputString = 'ΩΩLorem ipsum dolor sit amet, consectetur adipiscing eli' +
't. Morbi faucibus, purus at gravida dictum, libero arcu ' +
'convallis lacus, in commodo libero metus eu nisi. Nullam' +
' commodo, neque nec porta placerat, nisi est fermentum a' +
'ugue, vitae gravida tellus sapien sit amet tellus. Aenea' +
'n non diam orci. Proin quis elit turpis. Suspendisse non' +
' diam ipsum. Suspendisse nec ullamcorper odio. Vestibulu' +
'm arcu mi, sodales non suscipit id, ultrices ut massa. S' +
'ed ac sem sit amet arcu malesuada fermentum. Nunc sed. ';
const compressedString = 'KLUv/QRYRQkA9tc9H6AlhTb/z/7/gbTI3kaWLKnbCtkZu/hXm0j' +
'FpNz/VQM2ADMANQBHTuQOpIYzfVv7XGwXrpoIfgXNAB98xW4wV3' +
'vnCF2bjcvWZF2wIZ1vr1mSHHvPHU0TgMGBwUFrF0xqReWcWPO8z' +
'Ny6wMwFUilN+Lg987Zvs2GSRMy6uYvtovK9Uuhgst6l9FQrXLnA' +
'5gpZL7PdI8bO9sDH3tHm73XBzaUK+LjSPNKRmzQ3ZMYEPozdof1' +
'2KcZGfIcLa0PTsdkYqhGcAx/E9mWa8EGEeq0Qou2LTmzgg3YJz/' +
'21OuXSF+TOd662d60Qyb04xC5dOF4b8JFH8mpHAxAAELu3tg1oa' +
'bBEIWaRHdE0l/+0RdEWWIVMAku8TgbiX/4bU+OpLo4UuY1FKDR8' +
'RgBc';
zlib.zstdCompress(inputString, common.mustCall((err, buffer) => {
assert(inputString.length > buffer.length);
zlib.zstdDecompress(buffer, common.mustCall((err, buffer) => {
assert.strictEqual(buffer.toString(), inputString);
}));
}));
const buffer = Buffer.from(compressedString, 'base64');
zlib.zstdDecompress(buffer, common.mustCall((err, buffer) => {
assert.strictEqual(buffer.toString(), inputString);
}));

View File

@ -0,0 +1,34 @@
'use strict';
// Test unzipping a file that was created with a non-node zstd lib,
// piped in as fast as possible.
//
// The compressed fixture was created using the reference CLI:
// $ zstd -19 test/fixtures/person.jpg -o test/fixtures/person.jpg.zst
const common = require('../common');
const assert = require('assert');
const zlib = require('zlib');
const fixtures = require('../common/fixtures');
const tmpdir = require('../common/tmpdir');
tmpdir.refresh();
const decompress = new zlib.ZstdDecompress();
const fs = require('fs');
const fixture = fixtures.path('person.jpg.zst');
const unzippedFixture = fixtures.path('person.jpg');
const outputFile = tmpdir.resolve('person.jpg');
const expect = fs.readFileSync(unzippedFixture);
const inp = fs.createReadStream(fixture);
const out = fs.createWriteStream(outputFile);
inp.pipe(decompress).pipe(out);
out.on('close', common.mustCall(() => {
const actual = fs.readFileSync(outputFile);
assert.strictEqual(actual.length, expect.length);
for (let i = 0, l = actual.length; i < l; i++) {
assert.strictEqual(actual[i], expect[i], `byte[${i}]`);
}
}));

View File

@ -0,0 +1,29 @@
'use strict';
require('../common');
// This test ensures that zlib throws a RangeError if the final buffer needs to
// be larger than kMaxLength and concatenation fails.
// https://github.com/nodejs/node/pull/1811
const assert = require('assert');
// Change kMaxLength for zlib to trigger the error without having to allocate
// large Buffers.
const buffer = require('buffer');
const oldkMaxLength = buffer.kMaxLength;
buffer.kMaxLength = 64;
const zlib = require('zlib');
buffer.kMaxLength = oldkMaxLength;
// "a".repeat(128), compressed using zstd.
const encoded = Buffer.from('KLUv/SCARQAAEGFhAQA7BVg=', 'base64');
// Async
zlib.zstdDecompress(encoded, function(err) {
assert.ok(err instanceof RangeError);
});
// Sync
assert.throws(function() {
zlib.zstdDecompressSync(encoded);
}, RangeError);

View File

@ -0,0 +1,37 @@
'use strict';
const common = require('../common');
const assert = require('assert');
const zlib = require('zlib');
function compressWithPledgedSrcSize({ pledgedSrcSize, actualSrcSize }) {
return new Promise((resolve, reject) => {
const compressor = zlib.createZstdCompress({ pledgedSrcSize });
compressor.on('error', (e) => {
reject(e);
});
compressor.on('end', resolve);
compressor.write('x'.repeat(actualSrcSize), () => {
compressor.end();
compressor.resume();
});
}).then(() => {
// Compression should only succeed if sizes match
assert.strictEqual(pledgedSrcSize, actualSrcSize);
}, (error) => {
assert.strictEqual(error.code, 'ZSTD_error_srcSize_wrong');
// Size error should only happen when sizes do not match
assert.notStrictEqual(pledgedSrcSize, actualSrcSize);
}).then(common.mustCall());
}
compressWithPledgedSrcSize({ pledgedSrcSize: 0, actualSrcSize: 0 });
compressWithPledgedSrcSize({ pledgedSrcSize: 0, actualSrcSize: 42 });
compressWithPledgedSrcSize({ pledgedSrcSize: 13, actualSrcSize: 42 });
compressWithPledgedSrcSize({ pledgedSrcSize: 42, actualSrcSize: 0 });
compressWithPledgedSrcSize({ pledgedSrcSize: 42, actualSrcSize: 13 });
compressWithPledgedSrcSize({ pledgedSrcSize: 42, actualSrcSize: 42 });

View File

@ -0,0 +1,134 @@
'use strict';
require('../common');
const fixtures = require('../common/fixtures');
const assert = require('assert');
const zlib = require('zlib');
// Test some zstd-specific properties of the zstd streams that can not
// be easily covered through expanding zlib-only tests.
const sampleBuffer = fixtures.readSync('/pss-vectors.json');
{
// Test setting the quality parameter at stream creation:
const sizes = [];
for (let quality = 1;
quality <= 22;
quality++) {
const encoded = zlib.zstdCompressSync(sampleBuffer, {
params: {
[zlib.constants.ZSTD_c_compressionLevel]: quality
}
});
sizes.push(encoded.length);
}
// Increasing quality should roughly correspond to decreasing compressed size:
for (let i = 0; i < sizes.length - 1; i++) {
assert(sizes[i + 1] <= sizes[i] * 1.05, sizes); // 5 % margin of error.
}
assert(sizes[0] > sizes[sizes.length - 1], sizes);
}
{
// Test that setting out-of-bounds option values or keys fails.
assert.throws(() => {
zlib.createZstdCompress({
params: {
10000: 0
}
});
}, {
code: 'ERR_ZSTD_INVALID_PARAM',
name: 'RangeError',
message: '10000 is not a valid zstd parameter'
});
// Test that accidentally using duplicate keys fails.
assert.throws(() => {
zlib.createZstdCompress({
params: {
'0': 0,
'00': 0
}
});
}, {
code: 'ERR_ZSTD_INVALID_PARAM',
name: 'RangeError',
message: '00 is not a valid zstd parameter'
});
assert.throws(() => {
zlib.createZstdCompress({
params: {
// This param must be a valid ZSTD_strategy value.
[zlib.constants.ZSTD_c_strategy]: 130
}
});
}, {
code: 'ERR_ZLIB_INITIALIZATION_FAILED',
name: 'Error',
message: 'Setting parameter failed'
});
// Test that setting out-of-bounds option values or keys fails.
assert.throws(() => {
zlib.createZstdDecompress({
params: {
10000: 0
}
});
}, {
code: 'ERR_ZSTD_INVALID_PARAM',
name: 'RangeError',
message: '10000 is not a valid zstd parameter'
});
// Test that accidentally using duplicate keys fails.
assert.throws(() => {
zlib.createZstdDecompress({
params: {
'0': 0,
'00': 0
}
});
}, {
code: 'ERR_ZSTD_INVALID_PARAM',
name: 'RangeError',
message: '00 is not a valid zstd parameter'
});
assert.throws(() => {
zlib.createZstdDecompress({
params: {
// This param must be >= 10 (ZSTD_WINDOWLOG_ABSOLUTEMIN).
[zlib.constants.ZSTD_d_windowLogMax]: 1
}
});
}, {
code: 'ERR_ZLIB_INITIALIZATION_FAILED',
name: 'Error',
message: 'Setting parameter failed'
});
}
{
// Test options.flush range
assert.throws(() => {
zlib.zstdCompressSync('', { flush: zlib.constants.Z_FINISH });
}, {
code: 'ERR_OUT_OF_RANGE',
name: 'RangeError',
message: 'The value of "options.flush" is out of range. It must be >= 0 ' +
'and <= 2. Received 4',
});
assert.throws(() => {
zlib.zstdCompressSync('', { finishFlush: zlib.constants.Z_FINISH });
}, {
code: 'ERR_OUT_OF_RANGE',
name: 'RangeError',
message: 'The value of "options.finishFlush" is out of range. It must be ' +
'>= 0 and <= 2. Received 4',
});
}

View File

@ -43,6 +43,7 @@ let zlibPairs = [
[zlib.Gzip, zlib.Unzip],
[zlib.DeflateRaw, zlib.InflateRaw],
[zlib.BrotliCompress, zlib.BrotliDecompress],
[zlib.ZstdCompress, zlib.ZstdDecompress],
];
// How fast to trickle through the slowstream

View File

@ -252,6 +252,7 @@ const customTypesMap = {
'X509Certificate': 'crypto.html#class-x509certificate',
'zlib options': 'zlib.html#class-options',
'zstd options': 'zlib.html#class-zstdoptions',
'ReadableStream':
'webstreams.html#class-readablestream',