Skip to content

Commit d033ea6

Browse files
inoway46inoway46
committed
buffer: search utf16le at every byte offset
UTF-16LE searches treated Buffer contents as 16-bit values. This rounded odd offsets down and ignored matches at odd byte positions, even though a Buffer can contain UTF-16LE data beginning at any byte offset. Search encoded string needles byte-by-byte at every offset. Buffer and Uint8Array needles are now compared byte-for-byte in their entirety. This is a breaking change because a match spanning UTF-16 code unit boundaries can be returned before an aligned match. UCS2 searches using Buffer or Uint8Array needles can also return matches that were previously ignored or truncated. Signed-off-by: inoway46 <inoueyuya416@gmail.com>
1 parent b9dacd4 commit d033ea6

4 files changed

Lines changed: 104 additions & 46 deletions

File tree

‎doc/api/buffer.md‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2092,6 +2092,10 @@ console.log(buf.fill('zz', 'hex'));
20922092
<!-- YAML
20932093
added: v5.3.0
20942094
changes:
2095+
- version: REPLACEME
2096+
pr-url: https://github.com/nodejs/node/pull/64917
2097+
description: UTF-16LE and UCS2 searches now inspect every byte offset.
2098+
Buffer and Uint8Array values are compared in their entirety.
20952099
- version: v26.1.0
20962100
pr-url: https://github.com/nodejs/node/pull/62390
20972101
description: Added the `end` parameter.
@@ -2160,6 +2164,10 @@ console.log(buf.includes('this', 4));
21602164
<!-- YAML
21612165
added: v1.5.0
21622166
changes:
2167+
- version: REPLACEME
2168+
pr-url: https://github.com/nodejs/node/pull/64917
2169+
description: UTF-16LE and UCS2 searches now inspect every byte offset.
2170+
Buffer and Uint8Array values are compared in their entirety.
21632171
- version: v26.1.0
21642172
pr-url: https://github.com/nodejs/node/pull/62390
21652173
description: Added the `end` parameter.
@@ -2343,6 +2351,10 @@ for (const key of buf.keys()) {
23432351
<!-- YAML
23442352
added: v6.0.0
23452353
changes:
2354+
- version: REPLACEME
2355+
pr-url: https://github.com/nodejs/node/pull/64917
2356+
description: UTF-16LE and UCS2 searches now inspect every byte offset.
2357+
Buffer and Uint8Array values are compared in their entirety.
23462358
- version: v26.1.0
23472359
pr-url: https://github.com/nodejs/node/pull/62390
23482360
description: Added the `end` parameter.

‎src/node_buffer.cc‎

Lines changed: 21 additions & 41 deletions
Original file line numberDiff line numberDiff line change
@@ -1045,17 +1045,14 @@ void IndexOfString(const FunctionCallbackInfo<Value>& args) {
10451045
int64_t end_i64 = args[5].As<Integer>()->Value();
10461046

10471047
const char* haystack = buffer.data();
1048-
// Round down to the nearest multiple of 2 in case of UCS2.
1049-
const size_t haystack_length = (enc == UCS2) ?
1050-
buffer.length() &~ 1 : buffer.length(); // NOLINT(whitespace/operators)
1048+
const size_t haystack_length = buffer.length();
10511049

10521050
size_t needle_length;
10531051
if (!StringBytes::Size(isolate, needle, enc).To(&needle_length)) return;
10541052

10551053
// search_end is the exclusive upper bound of the search range.
10561054
size_t search_end = static_cast<size_t>(std::min(
10571055
std::max(end_i64, int64_t{0}), static_cast<int64_t>(haystack_length)));
1058-
if (enc == UCS2) search_end &= ~static_cast<size_t>(1);
10591056

10601057
int64_t opt_offset = IndexOfOffset(haystack_length,
10611058
offset_i64,
@@ -1102,27 +1099,27 @@ void IndexOfString(const FunctionCallbackInfo<Value>& args) {
11021099
if constexpr (IsBigEndian()) {
11031100
StringBytes::InlineDecoder decoder;
11041101
if (decoder.Decode(env, needle, enc).IsNothing()) return;
1105-
const uint16_t* decoded_string =
1106-
reinterpret_cast<const uint16_t*>(decoder.out());
1102+
const uint8_t* decoded_string =
1103+
reinterpret_cast<const uint8_t*>(decoder.out());
11071104

11081105
if (decoded_string == nullptr)
11091106
return args.GetReturnValue().Set(-1);
11101107

1111-
result = nbytes::SearchString(reinterpret_cast<const uint16_t*>(haystack),
1112-
search_end / 2,
1108+
result = nbytes::SearchString(reinterpret_cast<const uint8_t*>(haystack),
1109+
search_end,
11131110
decoded_string,
1114-
decoder.size() / 2,
1115-
offset / 2,
1111+
decoder.size(),
1112+
offset,
11161113
is_forward);
11171114
} else {
1118-
result = nbytes::SearchString(reinterpret_cast<const uint16_t*>(haystack),
1119-
search_end / 2,
1120-
needle_value.out(),
1121-
needle_value.length(),
1122-
offset / 2,
1123-
is_forward);
1115+
result = nbytes::SearchString(
1116+
reinterpret_cast<const uint8_t*>(haystack),
1117+
search_end,
1118+
reinterpret_cast<const uint8_t*>(needle_value.out()),
1119+
needle_length,
1120+
offset,
1121+
is_forward);
11241122
}
1125-
result *= 2;
11261123
} else if (enc == UTF8) {
11271124
Utf8Value needle_value(isolate, needle);
11281125
if (*needle_value == nullptr)
@@ -1163,8 +1160,6 @@ void IndexOfBuffer(const FunctionCallbackInfo<Value>& args) {
11631160
CHECK(args[4]->IsBoolean());
11641161
CHECK(args[5]->IsNumber());
11651162

1166-
enum encoding enc = static_cast<enum encoding>(args[3].As<Int32>()->Value());
1167-
11681163
Environment* env = Environment::GetCurrent(args);
11691164
THROW_AND_RETURN_UNLESS_BUFFER(env, args[0]);
11701165
THROW_AND_RETURN_UNLESS_BUFFER(env, args[1]);
@@ -1182,7 +1177,6 @@ void IndexOfBuffer(const FunctionCallbackInfo<Value>& args) {
11821177
// search_end is the exclusive upper bound of the search range.
11831178
size_t search_end = static_cast<size_t>(std::min(
11841179
std::max(end_i64, int64_t{0}), static_cast<int64_t>(haystack_length)));
1185-
if (enc == UCS2) search_end &= ~static_cast<size_t>(1);
11861180

11871181
int64_t opt_offset = IndexOfOffset(haystack_length,
11881182
offset_i64,
@@ -1218,27 +1212,13 @@ void IndexOfBuffer(const FunctionCallbackInfo<Value>& args) {
12181212
return args.GetReturnValue().Set(-1);
12191213
}
12201214

1221-
size_t result = search_end;
1222-
1223-
if (enc == UCS2) {
1224-
if (search_end < 2 || needle_length < 2) {
1225-
return args.GetReturnValue().Set(-1);
1226-
}
1227-
result = nbytes::SearchString(reinterpret_cast<const uint16_t*>(haystack),
1228-
search_end / 2,
1229-
reinterpret_cast<const uint16_t*>(needle),
1230-
needle_length / 2,
1231-
offset / 2,
1232-
is_forward);
1233-
result *= 2;
1234-
} else {
1235-
result = nbytes::SearchString(reinterpret_cast<const uint8_t*>(haystack),
1236-
search_end,
1237-
reinterpret_cast<const uint8_t*>(needle),
1238-
needle_length,
1239-
offset,
1240-
is_forward);
1241-
}
1215+
size_t result =
1216+
nbytes::SearchString(reinterpret_cast<const uint8_t*>(haystack),
1217+
search_end,
1218+
reinterpret_cast<const uint8_t*>(needle),
1219+
needle_length,
1220+
offset,
1221+
is_forward);
12421222

12431223
args.GetReturnValue().Set(result >= search_end ? -1
12441224
: static_cast<int>(result));

‎test/parallel/test-buffer-includes.js‎

Lines changed: 12 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -74,7 +74,18 @@ assert(b.includes(Buffer.from('f'), 5));
7474
assert(b.includes(Buffer.from('f'), -1));
7575
assert(!b.includes(Buffer.from('f'), 6));
7676

77-
assert(!Buffer.from('ff').includes(Buffer.from('f'), 1, 'ucs2'));
77+
assert(Buffer.from('ff').includes(Buffer.from('f'), 1, 'ucs2'));
78+
79+
{
80+
const oddIndexBuffer = Buffer.from('00aaaa', 'hex');
81+
const oddIndexNeedle = Buffer.from('\uaaaa', 'utf16le');
82+
assert(oddIndexBuffer.includes('\uaaaa', 0, 'utf16le'));
83+
assert(!oddIndexBuffer.includes('\uaaaa', 0, 2, 'utf16le'));
84+
assert(oddIndexBuffer.includes('\uaaaa', 0, 3, 'utf16le'));
85+
assert(oddIndexBuffer.includes(oddIndexNeedle, 0, 'utf16le'));
86+
assert(oddIndexBuffer.includes(
87+
new Uint8Array(oddIndexNeedle), 0, 'utf16le'));
88+
}
7889

7990
// test hex encoding
8091
assert.strictEqual(

‎test/parallel/test-buffer-indexof.js‎

Lines changed: 59 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -81,7 +81,7 @@ assert.strictEqual(b.indexOf(Buffer.from('f'), 5), 5);
8181
assert.strictEqual(b.indexOf(Buffer.from('f'), -1), 5);
8282
assert.strictEqual(b.indexOf(Buffer.from('f'), 6), -1);
8383

84-
assert.strictEqual(Buffer.from('ff').indexOf(Buffer.from('f'), 1, 'ucs2'), -1);
84+
assert.strictEqual(Buffer.from('ff').indexOf(Buffer.from('f'), 1, 'ucs2'), 1);
8585

8686
// Test invalid and uppercase encoding
8787
assert.strictEqual(b.indexOf('b', 'utf8'), 1);
@@ -192,7 +192,7 @@ assert.strictEqual(Buffer.from('aaaa0').indexOf('30', 'hex'), 4);
192192
assert.strictEqual(Buffer.from('aaaa00a').indexOf('3030', 'hex'), 4);
193193

194194
{
195-
// Test usc2 and utf16le encoding
195+
// Test ucs2 and utf16le encodings.
196196
['ucs2', 'utf16le'].forEach((encoding) => {
197197
const twoByteString = Buffer.from(
198198
'\u039a\u0391\u03a3\u03a3\u0395', encoding);
@@ -308,6 +308,51 @@ assert.strictEqual(Buffer.from('aaaa').indexOf('你好', 'ucs2'), -1);
308308
// Haystack has odd length, but the needle is UCS2.
309309
assert.strictEqual(Buffer.from('aaaaa').indexOf('b', 'ucs2'), -1);
310310

311+
{
312+
// Search UTF-16LE values at byte offsets that are not aligned to the start
313+
// of the Buffer.
314+
const value = '\u6881\u6882\u6881';
315+
const valueBuffer = Buffer.from(value, 'utf16le');
316+
assert.strictEqual(valueBuffer.indexOf('\u6881', 1, 'utf16le'), 4);
317+
assert.strictEqual(valueBuffer.slice(1).indexOf('\u6881', 'utf16le'), 3);
318+
assert.strictEqual(valueBuffer.indexOf('\u6881', -5, 'utf16le'), 4);
319+
assert.strictEqual(valueBuffer.indexOf('\u6881', 1, 4, 'utf16le'), -1);
320+
assert.strictEqual(valueBuffer.indexOf('\u6881', 1, 6, 'utf16le'), 4);
321+
322+
const prefixed = Buffer.alloc(7);
323+
prefixed.write(value, 1, 'utf16le');
324+
assert.strictEqual(prefixed.indexOf(value, 1, 'utf16le'), 1);
325+
326+
// UTF-16LE searches compare bytes at every offset, even when a match spans
327+
// two code units. The same needle also occurs on a code unit boundary at 4.
328+
const crossUnitBuffer = Buffer.from([
329+
0x41, 0x00, 0x01, 0x00, 0x00, 0x01,
330+
]);
331+
assert.strictEqual(crossUnitBuffer.indexOf('\u0100', 0, 'utf16le'), 1);
332+
333+
const oddIndexBuffer = Buffer.from('00aaaa', 'hex');
334+
const oddIndexNeedle = Buffer.from('\uaaaa', 'utf16le');
335+
assert.strictEqual(oddIndexBuffer.indexOf('\uaaaa', 0, 'utf16le'), 1);
336+
assert.strictEqual(oddIndexBuffer.indexOf('\uaaaa', 0, 2, 'utf16le'), -1);
337+
assert.strictEqual(oddIndexBuffer.indexOf('\uaaaa', 0, 3, 'utf16le'), 1);
338+
assert.strictEqual(oddIndexBuffer.indexOf(
339+
oddIndexNeedle, 0, 'utf16le'), 1);
340+
assert.strictEqual(oddIndexBuffer.indexOf(
341+
new Uint8Array(oddIndexNeedle), 0, 'utf16le'), 1);
342+
assert.strictEqual(oddIndexBuffer.lastIndexOf(
343+
'\uaaaa', 2, 3, 'utf16le'), 1);
344+
assert.strictEqual(oddIndexBuffer.lastIndexOf(
345+
oddIndexNeedle, 2, 3, 'utf16le'), 1);
346+
347+
// The encoding argument only applies to string needles. Buffer needles are
348+
// compared byte-for-byte, so an odd-length needle includes its final byte.
349+
const oddLengthNeedle = Buffer.from([0x61, 0x00, 0xff]);
350+
assert.strictEqual(Buffer.from([0x61, 0x00, 0x62, 0x00])
351+
.indexOf(oddLengthNeedle, 0, 'utf16le'), -1);
352+
assert.strictEqual(Buffer.from([0x61, 0x00, 0xff, 0x00])
353+
.indexOf(oddLengthNeedle, 0, 'utf16le'), 0);
354+
}
355+
311356
{
312357
// Find substrings in Utf8.
313358
const lengths = [1, 3, 15]; // Single char, simple and complex.
@@ -349,12 +394,18 @@ assert.strictEqual(Buffer.from('aaaaa').indexOf('b', 'ucs2'), -1);
349394

350395
const patternBufferUcs2 =
351396
allCharsBufferUcs2.slice(index, index + length);
397+
const expectedBufferIndex =
398+
allCharsBufferUcs2.indexOf(patternBufferUcs2);
352399
assert.strictEqual(
353-
index, allCharsBufferUcs2.indexOf(patternBufferUcs2, 0, 'ucs2'));
400+
expectedBufferIndex,
401+
allCharsBufferUcs2.indexOf(patternBufferUcs2, 0, 'ucs2'));
354402

355403
const patternStringUcs2 = patternBufferUcs2.toString('ucs2');
404+
const expectedStringIndex = allCharsBufferUcs2.indexOf(
405+
Buffer.from(patternStringUcs2, 'ucs2'));
356406
assert.strictEqual(
357-
index, allCharsBufferUcs2.indexOf(patternStringUcs2, 0, 'ucs2'));
407+
expectedStringIndex,
408+
allCharsBufferUcs2.indexOf(patternStringUcs2, 0, 'ucs2'));
358409
}
359410
}
360411
}
@@ -722,7 +773,11 @@ assert.strictEqual(reallyLong.lastIndexOf(pattern), 0);
722773
assert.strictEqual(buf.indexOf('', 0, 3), 0);
723774
assert.strictEqual(buf.indexOf('', 5, 3), 3);
724775
assert.strictEqual(buf.indexOf(Buffer.from(''), 5, 3), 3);
776+
assert.strictEqual(buf.indexOf('', 5, 3, 'utf16le'), 3);
777+
assert.strictEqual(buf.indexOf(Buffer.from(''), 5, 3, 'utf16le'), 3);
725778
assert.strictEqual(buf.indexOf('', 0, 0), 0);
726779
assert.strictEqual(buf.lastIndexOf('', 5, 3), 3);
727780
assert.strictEqual(buf.lastIndexOf(Buffer.from(''), 5, 3), 3);
781+
assert.strictEqual(buf.lastIndexOf('', 5, 3, 'utf16le'), 3);
782+
assert.strictEqual(buf.lastIndexOf(Buffer.from(''), 5, 3, 'utf16le'), 3);
728783
}

0 commit comments

Comments
 (0)