Skip to content

Commit 7d258db

Browse files
IlyasShabiIlyasShabi
authored
stream: support typed arrays
PR-URL: #51866 Reviewed-By: Matteo Collina <matteo.collina@gmail.com> Reviewed-By: Raz Luvaton <rluvaton@gmail.com> Reviewed-By: Paolo Insogna <paolo@cowtech.it> Reviewed-By: Benjamin Gruenbaum <benjamingr@gmail.com>
1 parent d62ab3a commit 7d258db

File tree

8 files changed

+250
-35
lines changed

8 files changed

+250
-35
lines changed

‎benchmark/streams/readable-uint8array.js‎

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,36 @@
1+
'use strict';
2+
const common = require('../common.js');
3+
const { Readable } = require('stream');
4+
5+
const bench = common.createBenchmark(main, {
6+
n: [1e6],
7+
kind: ['read', 'encoding'],
8+
});
9+
const ABC = new Uint8Array([0x41, 0x42, 0x43]);
10+
11+
function main({ n, kind }) {
12+
switch (kind) {
13+
case 'read': {
14+
bench.start();
15+
const readable = new Readable({
16+
read() {},
17+
});
18+
for (let i = 0; i < n; ++i) readable.push(ABC);
19+
bench.end(n);
20+
break;
21+
}
22+
23+
case 'encoding': {
24+
bench.start();
25+
const readable = new Readable({
26+
read() {},
27+
});
28+
readable.setEncoding('utf8');
29+
for (let i = 0; i < n; ++i) readable.push(ABC);
30+
bench.end(n);
31+
break;
32+
}
33+
default:
34+
throw new Error('Invalid kind');
35+
}
36+
}

‎benchmark/streams/writable-uint8array.js‎

Lines changed: 51 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,51 @@
1+
'use strict';
2+
const common = require('../common.js');
3+
const { Writable } = require('stream');
4+
5+
const bench = common.createBenchmark(main, {
6+
n: [50e6],
7+
kind: ['write', 'object-mode', 'writev'],
8+
});
9+
const ABC = new Uint8Array([0x41, 0x42, 0x43]);
10+
11+
function main({ n, kind }) {
12+
switch (kind) {
13+
case 'write': {
14+
bench.start();
15+
const wr = new Writable({
16+
write(chunk, encoding, cb) {
17+
cb();
18+
},
19+
});
20+
for (let i = 0; i < n; ++i) wr.write(ABC);
21+
bench.end(n);
22+
break;
23+
}
24+
25+
case 'object-mode': {
26+
bench.start();
27+
const wr = new Writable({
28+
objectMode: true,
29+
write(chunk, encoding, cb) {
30+
cb();
31+
},
32+
});
33+
for (let i = 0; i < n; ++i) wr.write(ABC);
34+
bench.end(n);
35+
break;
36+
}
37+
case 'writev': {
38+
bench.start();
39+
const wr = new Writable({
40+
writev(chunks, cb) {
41+
cb();
42+
},
43+
});
44+
for (let i = 0; i < n; ++i) wr.write(ABC);
45+
bench.end(n);
46+
break;
47+
}
48+
default:
49+
throw new Error('Invalid kind');
50+
}
51+
}

‎doc/api/stream.md‎

Lines changed: 48 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -282,11 +282,19 @@ The `finished` API also provides a [callback version][stream-finished].
282282

283283
### Object mode
284284

285-
All streams created by Node.js APIs operate exclusively on strings and `Buffer`
286-
(or `Uint8Array`) objects. It is possible, however, for stream implementations
287-
to work with other types of JavaScript values (with the exception of `null`,
288-
which serves a special purpose within streams). Such streams are considered to
289-
operate in "object mode".
285+
All streams created by Node.js APIs operate exclusively on strings, {Buffer},
286+
{TypedArray} and {DataView} objects:
287+
288+
* `Strings` and `Buffers` are the most common types used with streams.
289+
* `TypedArray` and `DataView` lets you handle binary data with types like
290+
`Int32Array` or `Uint8Array`. When you write a TypedArray or DataView to a
291+
stream, Node.js processes
292+
the raw bytes.
293+
294+
It is possible, however, for stream
295+
implementations to work with other types of JavaScript values (with the
296+
exception of `null`, which serves a special purpose within streams).
297+
Such streams are considered to operate in "object mode".
290298

291299
Stream instances are switched into object mode using the `objectMode` option
292300
when the stream is created. Attempting to switch an existing stream into
@@ -712,6 +720,9 @@ console.log(myStream.destroyed); // true
712720
<!-- YAML
713721
added: v0.9.4
714722
changes:
723+
- version: REPLACEME
724+
pr-url: https://github.com/nodejs/node/pull/51866
725+
description: The `chunk` argument can now be a `TypedArray` or `DataView` instance.
715726
- version: v15.0.0
716727
pr-url: https://github.com/nodejs/node/pull/34101
717728
description: The `callback` is invoked before 'finish' or on error.
@@ -726,10 +737,10 @@ changes:
726737
description: The `chunk` argument can now be a `Uint8Array` instance.
727738
-->
728739

729-
* `chunk` {string|Buffer|Uint8Array|any} Optional data to write. For streams
730-
not operating in object mode, `chunk` must be a string, `Buffer` or
731-
`Uint8Array`. For object mode streams, `chunk` may be any JavaScript value
732-
other than `null`.
740+
* `chunk` {string|Buffer|TypedArray|DataView|any} Optional data to write. For
741+
streams not operating in object mode, `chunk` must be a {string}, {Buffer},
742+
{TypedArray} or {DataView}. For object mode streams, `chunk` may be any
743+
JavaScript value other than `null`.
733744
* `encoding` {string} The encoding if `chunk` is a string
734745
* `callback` {Function} Callback for when the stream is finished.
735746
* Returns: {this}
@@ -926,6 +937,9 @@ Getter for the property `objectMode` of a given `Writable` stream.
926937
<!-- YAML
927938
added: v0.9.4
928939
changes:
940+
- version: REPLACEME
941+
pr-url: https://github.com/nodejs/node/pull/51866
942+
description: The `chunk` argument can now be a `TypedArray` or `DataView` instance.
929943
- version: v8.0.0
930944
pr-url: https://github.com/nodejs/node/pull/11608
931945
description: The `chunk` argument can now be a `Uint8Array` instance.
@@ -935,10 +949,10 @@ changes:
935949
considered invalid now, even in object mode.
936950
-->
937951

938-
* `chunk` {string|Buffer|Uint8Array|any} Optional data to write. For streams
939-
not operating in object mode, `chunk` must be a string, `Buffer` or
940-
`Uint8Array`. For object mode streams, `chunk` may be any JavaScript value
941-
other than `null`.
952+
* `chunk` {string|Buffer|TypedArray|DataView|any} Optional data to write. For
953+
streams not operating in object mode, `chunk` must be a {string}, {Buffer},
954+
{TypedArray} or {DataView}. For object mode streams, `chunk` may be any
955+
JavaScript value other than `null`.
942956
* `encoding` {string|null} The encoding, if `chunk` is a string. **Default:** `'utf8'`
943957
* `callback` {Function} Callback for when this chunk of data is flushed.
944958
* Returns: {boolean} `false` if the stream wishes for the calling code to
@@ -1763,15 +1777,18 @@ setTimeout(() => {
17631777
<!-- YAML
17641778
added: v0.9.11
17651779
changes:
1780+
- version: REPLACEME
1781+
pr-url: https://github.com/nodejs/node/pull/51866
1782+
description: The `chunk` argument can now be a `TypedArray` or `DataView` instance.
17661783
- version: v8.0.0
17671784
pr-url: https://github.com/nodejs/node/pull/11608
17681785
description: The `chunk` argument can now be a `Uint8Array` instance.
17691786
-->
17701787

1771-
* `chunk` {Buffer|Uint8Array|string|null|any} Chunk of data to unshift onto the
1772-
read queue. For streams not operating in object mode, `chunk` must be a
1773-
string, `Buffer`, `Uint8Array`, or `null`. For object mode streams, `chunk`
1774-
may be any JavaScript value.
1788+
* `chunk` {Buffer|TypedArray|DataView|string|null|any} Chunk of data to unshift
1789+
onto the read queue. For streams not operating in object mode, `chunk` must
1790+
be a {string}, {Buffer}, {TypedArray}, {DataView} or `null`.
1791+
For object mode streams, `chunk` may be any JavaScript value.
17751792
* `encoding` {string} Encoding of string chunks. Must be a valid
17761793
`Buffer` encoding, such as `'utf8'` or `'ascii'`.
17771794

@@ -3515,8 +3532,8 @@ changes:
35153532
**Default:** `'utf8'`.
35163533
* `objectMode` {boolean} Whether or not the
35173534
[`stream.write(anyObj)`][stream-write] is a valid operation. When set,
3518-
it becomes possible to write JavaScript values other than string,
3519-
`Buffer` or `Uint8Array` if supported by the stream implementation.
3535+
it becomes possible to write JavaScript values other than string, {Buffer},
3536+
{TypedArray} or {DataView} if supported by the stream implementation.
35203537
**Default:** `false`.
35213538
* `emitClose` {boolean} Whether or not the stream should emit `'close'`
35223539
after it has been destroyed. **Default:** `true`.
@@ -4068,22 +4085,25 @@ It can be overridden by child classes but it **must not** be called directly.
40684085

40694086
<!-- YAML
40704087
changes:
4088+
- version: REPLACEME
4089+
pr-url: https://github.com/nodejs/node/pull/51866
4090+
description: The `chunk` argument can now be a `TypedArray` or `DataView` instance.
40714091
- version: v8.0.0
40724092
pr-url: https://github.com/nodejs/node/pull/11608
40734093
description: The `chunk` argument can now be a `Uint8Array` instance.
40744094
-->
40754095

4076-
* `chunk` {Buffer|Uint8Array|string|null|any} Chunk of data to push into the
4077-
read queue. For streams not operating in object mode, `chunk` must be a
4078-
string, `Buffer` or `Uint8Array`. For object mode streams, `chunk` may be
4079-
any JavaScript value.
4096+
* `chunk` {Buffer|TypedArray|DataView|string|null|any} Chunk of data to push
4097+
into the read queue. For streams not operating in object mode, `chunk` must
4098+
be a {string}, {Buffer}, {TypedArray} or {DataView}. For object mode streams,
4099+
`chunk` may be any JavaScript value.
40804100
* `encoding` {string} Encoding of string chunks. Must be a valid
40814101
`Buffer` encoding, such as `'utf8'` or `'ascii'`.
40824102
* Returns: {boolean} `true` if additional chunks of data may continue to be
40834103
pushed; `false` otherwise.
40844104

4085-
When `chunk` is a `Buffer`, `Uint8Array`, or `string`, the `chunk` of data will
4086-
be added to the internal queue for users of the stream to consume.
4105+
When `chunk` is a {Buffer}, {TypedArray}, {DataView} or {string}, the `chunk`
4106+
of data will be added to the internal queue for users of the stream to consume.
40874107
Passing `chunk` as `null` signals the end of the stream (EOF), after which no
40884108
more data can be written.
40894109

@@ -4758,8 +4778,9 @@ situations within Node.js where this is done, particularly in the
47584778

47594779
Use of `readable.push('')` is not recommended.
47604780

4761-
Pushing a zero-byte string, `Buffer`, or `Uint8Array` to a stream that is not in
4762-
object mode has an interesting side effect. Because it _is_ a call to
4781+
Pushing a zero-byte {string}, {Buffer}, {TypedArray} or {DataView} to a stream
4782+
that is not in object mode has an interesting side effect.
4783+
Because it _is_ a call to
47634784
[`readable.push()`][stream-push], the call will end the reading process.
47644785
However, because the argument is an empty string, no data is added to the
47654786
readable buffer so there is nothing for a user to consume.

‎lib/internal/streams/readable.js‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -420,11 +420,11 @@ function readableAddChunkUnshiftByteMode(stream, state, chunk, encoding) {
420420
chunk = Buffer.from(chunk, encoding);
421421
}
422422
}
423-
} else if (Stream._isUint8Array(chunk)) {
423+
} else if (Stream._isArrayBufferView(chunk)) {
424424
chunk = Stream._uint8ArrayToBuffer(chunk);
425425
} else if (chunk !== undefined && !(chunk instanceof Buffer)) {
426426
errorOrDestroy(stream, new ERR_INVALID_ARG_TYPE(
427-
'chunk', ['string', 'Buffer', 'Uint8Array'], chunk));
427+
'chunk', ['string', 'Buffer', 'TypedArray', 'DataView'], chunk));
428428
return false;
429429
}
430430

@@ -473,12 +473,12 @@ function readableAddChunkPushByteMode(stream, state, chunk, encoding) {
473473
}
474474
} else if (chunk instanceof Buffer) {
475475
encoding = '';
476-
} else if (Stream._isUint8Array(chunk)) {
476+
} else if (Stream._isArrayBufferView(chunk)) {
477477
chunk = Stream._uint8ArrayToBuffer(chunk);
478478
encoding = '';
479479
} else if (chunk !== undefined) {
480480
errorOrDestroy(stream, new ERR_INVALID_ARG_TYPE(
481-
'chunk', ['string', 'Buffer', 'Uint8Array'], chunk));
481+
'chunk', ['string', 'Buffer', 'TypedArray', 'DataView'], chunk));
482482
return false;
483483
}
484484

‎lib/internal/streams/writable.js‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -467,12 +467,12 @@ function _write(stream, chunk, encoding, cb) {
467467
}
468468
} else if (chunk instanceof Buffer) {
469469
encoding = 'buffer';
470-
} else if (Stream._isUint8Array(chunk)) {
470+
} else if (Stream._isArrayBufferView(chunk)) {
471471
chunk = Stream._uint8ArrayToBuffer(chunk);
472472
encoding = 'buffer';
473473
} else {
474474
throw new ERR_INVALID_ARG_TYPE(
475-
'chunk', ['string', 'Buffer', 'Uint8Array'], chunk);
475+
'chunk', ['string', 'Buffer', 'TypedArray', 'DataView'], chunk);
476476
}
477477
}
478478

‎lib/stream.js‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -50,6 +50,7 @@ const internalBuffer = require('internal/buffer');
5050

5151
const promises = require('stream/promises');
5252
const utils = require('internal/streams/utils');
53+
const { isArrayBufferView, isUint8Array } = require('internal/util/types');
5354

5455
const Stream = module.exports = require('internal/streams/legacy').Stream;
5556

@@ -137,7 +138,8 @@ ObjectDefineProperty(eos, customPromisify, {
137138
// Backwards-compat with node 0.4.x
138139
Stream.Stream = Stream;
139140

140-
Stream._isUint8Array = require('internal/util/types').isUint8Array;
141+
Stream._isArrayBufferView = isArrayBufferView;
142+
Stream._isUint8Array = isUint8Array;
141143
Stream._uint8ArrayToBuffer = function _uint8ArrayToBuffer(chunk) {
142144
return new internalBuffer.FastBuffer(chunk.buffer,
143145
chunk.byteOffset,

‎test/parallel/test-net-write-arguments.js‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,6 @@ assert.throws(() => {
3434
code: 'ERR_INVALID_ARG_TYPE',
3535
name: 'TypeError',
3636
message: 'The "chunk" argument must be of type string or an instance of ' +
37-
`Buffer or Uint8Array.${common.invalidArgTypeHelper(value)}`
37+
`Buffer, TypedArray, or DataView.${common.invalidArgTypeHelper(value)}`
3838
});
3939
});

0 commit comments

Comments
 (0)